commit 18b08d1104fb9376c14b5743b3f0a853194be620
parent 6df0de3d7dcd280509c72182fadd782673685fb4
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 2 Oct 2026 01:30:49 -0400
common: a subtitle 429 beside another subtitle failure takes that failure's class; a run that asked nothing is not clean; clear-platform-hold is a declared kind (re-review R1, R3, R4)
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
5 files changed, 108 insertions(+), 2 deletions(-)
diff --git a/common/jobs/jobKinds.ts b/common/jobs/jobKinds.ts
@@ -100,6 +100,18 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
queueKeyStrategy: "platform",
needsMedia: true,
},
+ // THE OPERATOR'S "Clear hold" on /operations/download (release 17 slice
+ // RL): one step that drops a held platform's hold, backoff and raised pace,
+ // and says so in its log. Its own queue (`pacing:<platform>`), so a Sync
+ // running on the platform's queue never makes the click wait. Touches no
+ // channel and no file but the auto-queue state.
+ "clear-platform-hold": {
+ kind: "clear-platform-hold",
+ label: "Clear rate-limit hold",
+ drainable: false,
+ replayable: false,
+ queueKeyStrategy: "custom",
+ },
// The two OPERATION lanes' runners. Same shape as the two above — one
// long-lived job per lane on queueKey "", drainable, never replayable — with
// one difference worth stating: their units make NO job record. A digest or a
diff --git a/common/lib/availability.test.ts b/common/lib/availability.test.ts
@@ -199,3 +199,19 @@ test("a subtitle failure that is not a 429 is told apart; a WARNING webpage 429
false,
);
});
+
+test("a subtitle 429 beside another subtitle failure classes as the OTHER failure, never rate_limit (re-review R1)", () => {
+ const w429 = "WARNING: Unable to download video subtitles for 'en': HTTP Error 429: Too Many Requests";
+ const chat404 = "WARNING: Unable to download video subtitles for 'live_chat': HTTP Error 404: Not Found";
+ const sub403 = "WARNING: Unable to download video subtitles for 'en-orig': HTTP Error 403: Forbidden";
+ assert.equal(classifyDownloadFailure(`${w429}\n${chat404}`, "error"), "unknown");
+ assert.equal(classifyDownloadFailure(`${w429}\n${sub403}`, "error"), "network");
+ // A real platform 429 beside them still wins.
+ assert.equal(
+ classifyDownloadFailure(
+ `${w429}\n${chat404}\nERROR: [youtube] abc: Unable to download webpage: HTTP Error 429: Too Many Requests`,
+ "error",
+ ),
+ "rate_limit",
+ );
+});
diff --git a/common/lib/availability.ts b/common/lib/availability.ts
@@ -330,7 +330,14 @@ export function classifyDownloadFailure(
) {
return "per_video";
}
- const s = stderrTail.toLowerCase();
+ // A subtitle 429 beside another subtitle failure is still not a platform
+ // signal (release 17 re-review R1): its lines are dropped before the
+ // generic rate-limit test, so the class is the OTHER failure's.
+ const s = stderrTail
+ .split("\n")
+ .filter((l) => !(SUBTITLE_FAILURE_RE.test(l) && RATE_LIMIT_WORDS_RE.test(l)))
+ .join("\n")
+ .toLowerCase();
if (
/http error 429/.test(s) ||
/too many requests/.test(s) ||
diff --git a/common/ytdlp/platformClean.test.ts b/common/ytdlp/platformClean.test.ts
@@ -0,0 +1,58 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { chmodSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test ytdlp/platformClean.test.ts
+//
+// A CLEAN MANUAL RUN SETTLES THE PLATFORM — ONLY IF IT ASKED THE SOURCE
+// SOMETHING (release 17 slice RL, re-review R3). runYtdlp calls
+// `onPlatformClean` after a run that neither threw nor recorded a backoff; a
+// run that made no request (download-missing with nothing to fetch) must not,
+// or it would lift a due hold on no evidence. getPaths() memoizes, so the env
+// is set before anything imports it.
+const ROOT = mkdtempSync(path.join(os.tmpdir(), "platform-clean-"));
+process.env.TRANSCRIPTS_DIR = ROOT;
+process.env.SETTINGS_FILE = path.join(ROOT, "settings.json");
+writeFileSync(
+ process.env.SETTINGS_FILE,
+ JSON.stringify({ minFreeDiskGB: 0, sleepBetweenDownloadsSeconds: 0 }) + "\n",
+);
+const BIN = path.join(ROOT, "fake-ytdlp.sh");
+writeFileSync(BIN, "#!/bin/sh\nexit 0\n");
+chmodSync(BIN, 0o755);
+process.env.YTDLP_BIN = BIN;
+mkdirSync(path.join(ROOT, "channels", "c", "data"), { recursive: true });
+// An empty saved playlist: download-missing has nothing to fetch.
+writeFileSync(path.join(ROOT, "channels", "c", "playlist"), "");
+
+const { runYtdlp } = await import("./runYtdlp");
+const { getPaths } = await import("../lib/paths");
+
+async function cleanCalls(mode: "download-missing" | "download-one-audio"): Promise<number> {
+ let calls = 0;
+ await runYtdlp({
+ channelSlug: "c",
+ mode,
+ channelConfig: { handling: "transcribe", url: "https://www.youtube.com/@c/videos" } as never,
+ paths: getPaths(),
+ onLog: () => {},
+ signal: new AbortController().signal,
+ singleVideoUrl: "https://www.youtube.com/watch?v=abcdefghijk",
+ onPlatformClean: async () => {
+ calls++;
+ return null;
+ },
+ });
+ return calls;
+}
+
+test("a run that made no request is not a clean run", async () => {
+ assert.equal(await cleanCalls("download-missing"), 0);
+});
+
+test("a run that made a request and recorded no backoff is", async () => {
+ assert.equal(await cleanCalls("download-one-audio"), 1);
+});
diff --git a/common/ytdlp/runYtdlp.ts b/common/ytdlp/runYtdlp.ts
@@ -185,6 +185,11 @@ export type RunYtdlpOpts = {
// backoff and any hold clear; release 17 review H2). Returns a line for the
// log, or null.
onPlatformClean?: () => Promise<string | null>;
+ // Internal: how many requests this run made of the source (a listing, a
+ // per-video download, a raw spawn). Set by runYtdlpMode and shared through
+ // every `{...opts}` copy by reference; a run that asked nothing is not clean
+ // (release 17 re-review R3).
+ requestCounter?: { n: number };
};
// The batch modes whose child writes media into `data/<id>/` — yt-dlp's own
@@ -235,14 +240,19 @@ async function runYtdlpMode(opts: RunYtdlpOpts): Promise<void> {
if (!opts.onPlatformClean) return runYtdlpModeInner(opts);
let backedOff = false;
const inner = opts.onPlatformBackoff;
+ const requestCounter = { n: 0 };
await runYtdlpModeInner({
...opts,
+ requestCounter,
onPlatformBackoff: async (failureClass) => {
backedOff = true;
await inner?.(failureClass);
},
});
- if (backedOff || opts.signal.aborted) return;
+ // A run that asked the source nothing (download-missing with nothing to
+ // fetch, download-missing-subs with every video deferred) says nothing about
+ // the platform, and must not lift a due hold.
+ if (backedOff || opts.signal.aborted || requestCounter.n === 0) return;
try {
const line = await opts.onPlatformClean();
if (line) opts.onLog(line);
@@ -488,6 +498,7 @@ async function enumeratePlaylistUrls(
opts.channelConfig.url!,
);
opts.onLog(`$ ${opts.paths.ytdlpBin} ${args.join(" ")}\n`);
+ if (opts.requestCounter) opts.requestCounter.n++;
const child = execa(opts.paths.ytdlpBin, args, {
cwd: root,
@@ -1125,6 +1136,7 @@ export async function runManagedDownloads(
});
let outcome;
try {
+ if (opts.requestCounter) opts.requestCounter.n++;
outcome = await downloadOne({
channelSlug: opts.channelSlug,
channelConfig: effectiveChannelConfig,
@@ -2120,6 +2132,7 @@ async function runChildAndStream(
args: string[],
): Promise<string> {
opts.onLog(`$ ${opts.paths.ytdlpBin} ${args.join(" ")}\n`);
+ if (opts.requestCounter) opts.requestCounter.n++;
const child = execa(opts.paths.ytdlpBin, args, {
cwd,
cancelSignal: opts.signal,
