commit aa4226b5a472edc630dee16c19806f7d3d0e5939
parent fecbbe0a9ed9fb9e55a0190276e377bc79ba27db
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 25 Sep 2026 18:37:22 -0400
common: no 30 s sleep after a video that fetched no media
runManagedDownloads slept `sleepBetweenDownloadsSeconds` after every video
whatever its outcome. On 2026-09-25 paramount-tactical's download-missing
slept 193 times for 14 archives: each video the download filter declined
paid 30 s for nothing but its metadata prefetch.
The sleep now skips when the video made no media request — every attempt
was the n: 0 prefetch — and it ended skipped-filtered or failed per_video
(members-only, private, removed). A real fetch still sleeps, success or
failure, and so does any failure that is not the video's (a rate limit
above all). runManagedDownloads is exported with a deps seam (downloadOne,
sleep); managedDownloadsSleep.test.ts drives the loop with canned outcomes:
6 cases, 3 of which fail on the old gate.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
2 files changed, 211 insertions(+), 6 deletions(-)
diff --git a/common/ytdlp/managedDownloadsSleep.test.ts b/common/ytdlp/managedDownloadsSleep.test.ts
@@ -0,0 +1,168 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, writeFileSync, mkdirSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import type {
+ DownloadAttempt,
+ DownloadOutcomeRecord,
+ DownloadOutcomeStatus,
+} from "../lib/downloadOutcome";
+import type { DownloadFailureClass } from "../lib/availability";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test ytdlp/managedDownloadsSleep.test.ts
+//
+// THE BETWEEN-DOWNLOADS SLEEP PACES MEDIA FETCHES (release 9, B2). On
+// 2026-09-25 a filtered channel's download-missing slept 30 s after every video
+// the download filter declined — 193 sleeps for 14 archives. The loop is driven
+// here with an injected per-video download (canned outcomes, no yt-dlp) and an
+// injected sleeper that only counts. getPaths() memoizes, so the env is set
+// before anything imports it.
+const ROOT = mkdtempSync(path.join(os.tmpdir(), "managed-sleep-"));
+process.env.TRANSCRIPTS_DIR = ROOT;
+process.env.SETTINGS_FILE = path.join(ROOT, "settings.json");
+writeFileSync(
+ process.env.SETTINGS_FILE,
+ JSON.stringify({ minFreeDiskGB: 0, sleepBetweenDownloadsSeconds: 30 }) + "\n",
+);
+mkdirSync(path.join(ROOT, "channels", "c", "data"), { recursive: true });
+
+const { runManagedDownloads, declinedWithoutMediaFetch } =
+ await import("./runYtdlp");
+const { getPaths } = await import("../lib/paths");
+
+const PREFETCH: DownloadAttempt = {
+ n: 0,
+ kind: "metadata-prefetch",
+ handling: "youtube",
+ usedCookies: false,
+ ytdlpExitCode: 0,
+};
+const PRIMARY: DownloadAttempt = {
+ n: 1,
+ kind: "primary",
+ handling: "youtube",
+ usedCookies: false,
+ ytdlpExitCode: 0,
+};
+
+function outcome(
+ status: DownloadOutcomeStatus,
+ attempts: DownloadAttempt[],
+ failureClass?: DownloadFailureClass,
+): DownloadOutcomeRecord {
+ return {
+ videoId: "x",
+ status,
+ startedAt: "2026-09-25T00:00:00.000Z",
+ finishedAt: "2026-09-25T00:00:01.000Z",
+ attempts,
+ ...(failureClass ? { failureClass } : {}),
+ };
+}
+
+const FETCHED = outcome("ok", [PREFETCH, PRIMARY]);
+const FILTERED = outcome("skipped-filtered", [PREFETCH]);
+const MEMBERS_ONLY = outcome(
+ "failed",
+ [{ ...PREFETCH, ytdlpExitCode: 1, availabilityClass: "members_only" }],
+ "per_video",
+);
+
+// Runs the loop over one URL per outcome and returns how many times it slept.
+async function sleepsFor(outcomes: DownloadOutcomeRecord[]): Promise<number> {
+ let sleeps = 0;
+ let i = 0;
+ const urls = outcomes.map(
+ (_, n) =>
+ `https://www.youtube.com/watch?v=vid${String(n).padStart(8, "0")}`,
+ );
+ const channelConfig = {
+ handling: "youtube",
+ url: "https://www.youtube.com/@c/videos",
+ } as never;
+ await runManagedDownloads(
+ {
+ channelSlug: "c",
+ mode: "download-missing" as never,
+ channelConfig,
+ paths: getPaths(),
+ onLog: () => {},
+ signal: new AbortController().signal,
+ abortOnError: false,
+ },
+ urls,
+ channelConfig,
+ undefined,
+ {
+ downloadOne: async () => outcomes[i++],
+ sleep: async () => {
+ sleeps++;
+ },
+ },
+ );
+ return sleeps;
+}
+
+test("a fetched video sleeps before the next one", async () => {
+ assert.equal(await sleepsFor([FETCHED, FETCHED]), 1);
+});
+
+test("a video the download filter declined does not sleep", async () => {
+ assert.equal(await sleepsFor([FILTERED, FETCHED]), 0);
+ assert.equal(await sleepsFor([FILTERED, FILTERED, FILTERED]), 0);
+});
+
+test("a per-video failure that never reached a media request does not sleep", async () => {
+ assert.equal(await sleepsFor([MEMBERS_ONLY, FETCHED]), 0);
+});
+
+test("the last video never sleeps, whatever it was", async () => {
+ assert.equal(await sleepsFor([FETCHED]), 0);
+ assert.equal(await sleepsFor([FILTERED, FETCHED]), 0);
+});
+
+test("a real fetch that failed still sleeps, per-video or not", async () => {
+ const perVideoAfterFetch = outcome(
+ "failed",
+ [PREFETCH, { ...PRIMARY, ytdlpExitCode: 1 }],
+ "per_video",
+ );
+ const network = outcome(
+ "failed",
+ [{ ...PREFETCH, ytdlpExitCode: 1 }],
+ "network",
+ );
+ assert.equal(await sleepsFor([perVideoAfterFetch, FETCHED]), 1);
+ // A network failure at the prefetch is not the video's fault: keep pacing.
+ assert.equal(await sleepsFor([network, FETCHED]), 1);
+});
+
+test("declinedWithoutMediaFetch: a chat-only pass is a fetch", () => {
+ const chatPass: DownloadAttempt = { ...PRIMARY, kind: "live-chat-only" };
+ // A failed chat pass leaves the status skipped-filtered, but it asked.
+ assert.equal(
+ declinedWithoutMediaFetch(
+ outcome("skipped-filtered", [PREFETCH, chatPass]),
+ undefined,
+ ),
+ false,
+ );
+ assert.equal(
+ declinedWithoutMediaFetch(
+ outcome("chat-only", [PREFETCH, chatPass]),
+ undefined,
+ ),
+ false,
+ );
+ assert.equal(declinedWithoutMediaFetch(FILTERED, undefined), true);
+ assert.equal(declinedWithoutMediaFetch(MEMBERS_ONLY, "per_video"), true);
+ assert.equal(
+ declinedWithoutMediaFetch(
+ outcome("failed", [PREFETCH], "rate_limit"),
+ "rate_limit",
+ ),
+ false,
+ );
+});
diff --git a/common/ytdlp/runYtdlp.ts b/common/ytdlp/runYtdlp.ts
@@ -22,7 +22,9 @@ import {
EXCLUDED_FROM_DOWNLOAD,
classifyDownloadFailure,
type Availability,
+ type DownloadFailureClass,
} from "../lib/availability";
+import type { DownloadOutcomeRecord } from "../lib/downloadOutcome";
import {
alwaysCookies,
resolveCookiePolicy,
@@ -888,6 +890,26 @@ async function downloadPlaylistManaged(
await safeBackfillAvailability(opts);
}
+export type ManagedDownloadsDeps = {
+ downloadOne?: typeof downloadOneManaged;
+ sleep?: (ms: number, signal: AbortSignal) => Promise<void>;
+};
+
+// True when the video ended without a single media request: every attempt it
+// made was the n: 0 metadata prefetch (or its cookie retry), and it ended
+// either declined by an app-level filter or failed for a reason that belongs
+// to the video (members-only, private, removed). Attempts n >= 1 are the real
+// fetches — the download attempts and the chat-only pass — and any of them
+// means the source was asked for media.
+export function declinedWithoutMediaFetch(
+ outcome: DownloadOutcomeRecord,
+ failureClass: DownloadFailureClass | undefined,
+): boolean {
+ if (outcome.attempts.some((a) => a.n >= 1)) return false;
+ if (outcome.status === "skipped-filtered") return true;
+ return outcome.status === "failed" && failureClass === "per_video";
+}
+
type ManagedRunResult = {
okCount: number;
failedCount: number;
@@ -906,7 +928,7 @@ type ManagedRunResult = {
// cost is the price of being able to retry each independently. Honors the
// hard `signal`, the soft `drainSignal`, the per-channel sleep, and the
// per-operation tracker.
-async function runManagedDownloads(
+export async function runManagedDownloads(
opts: RunYtdlpOpts,
urls: ReadonlyArray<string>,
effectiveChannelConfig: ChannelConfig,
@@ -914,7 +936,12 @@ async function runManagedDownloads(
// already resolved it (for their own prefilter) pass it through so the
// forceCookies-without-value warning isn't logged twice.
cookiePolicyOverride?: ResolvedCookiePolicy,
+ // Test seams: the per-video download and the between-downloads sleep. Every
+ // production caller passes neither.
+ deps: ManagedDownloadsDeps = {},
): Promise<ManagedRunResult> {
+ const downloadOne = deps.downloadOne ?? downloadOneManaged;
+ const sleep = deps.sleep ?? abortableSleep;
const settings = getSettings();
const cookiePolicy =
cookiePolicyOverride ??
@@ -987,7 +1014,7 @@ async function runManagedDownloads(
});
let outcome;
try {
- outcome = await downloadOneManaged({
+ outcome = await downloadOne({
channelSlug: opts.channelSlug,
channelConfig: effectiveChannelConfig,
paths: opts.paths,
@@ -1023,12 +1050,14 @@ async function runManagedDownloads(
// kept on disk, but not a usable download. Count as skipped (not ok,
// not a retryable failure) so the batch summary stays honest.
skippedCount++;
- } else if (outcome.status === "failed") {
+ }
+ let failureClass: DownloadFailureClass | undefined;
+ if (outcome.status === "failed") {
failedCount++;
// downloadOneManaged classifies against the full stderr tail; fall
// back to the truncated attempt error only if it's somehow absent.
const lastAttempt = outcome.attempts[outcome.attempts.length - 1];
- const failureClass =
+ failureClass =
outcome.failureClass ??
classifyDownloadFailure(
lastAttempt?.error ?? "",
@@ -1057,14 +1086,22 @@ async function runManagedDownloads(
processedCount++;
const isLast = processedCount >= urls.length;
const willAbortLoop = firstFailure !== null && abortOnError;
+ // THE SLEEP PACES MEDIA FETCHES, so a video that fetched none does not
+ // pay for one (release 9, B2). On 2026-09-25 paramount-tactical's
+ // download-missing slept 193 times for 14 archives: every video the
+ // download filter declined slept 30 s after nothing but its metadata
+ // prefetch. A real fetch — success or failure, or a chat-only pass —
+ // still sleeps, and so does every failure that is not per-video (a
+ // rate limit above all).
if (
sleepSeconds > 0 &&
!isLast &&
!willAbortLoop &&
- !opts.signal.aborted
+ !opts.signal.aborted &&
+ !declinedWithoutMediaFetch(outcome, failureClass)
) {
opts.onLog(`Sleeping ${sleepSeconds}s before next download...\n`);
- await abortableSleep(sleepSeconds * 1000, opts.signal);
+ await sleep(sleepSeconds * 1000, opts.signal);
}
}),
),
