commit 5896dd4867315fc3b9ae3375a96eb3a30f50041e
parent 6c7ddce2d596256b52732849f32d009e22422e2f
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Wed, 7 Oct 2026 14:44:24 -0400
Merge fetch-window-rumble-hls (editor clip windows from Rumble retry with -extension_picky 0)
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
2 files changed, 127 insertions(+), 5 deletions(-)
diff --git a/common/ytdlp/fetchWindowManaged.test.ts b/common/ytdlp/fetchWindowManaged.test.ts
@@ -0,0 +1,100 @@
+// fetchWindowManaged against a fake yt-dlp: Rumble's `.tar` HLS segments are
+// refused by ffmpeg until the one retry adds -extension_picky 0, and the option
+// is never passed on a first try (against a progressive URL it is an error).
+//
+// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test ytdlp/fetchWindowManaged.test.ts
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { chmod, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { fetchWindowManaged, HLS_PICKY_RETRY_ARGS } from "./fetchWindowManaged";
+import type { Paths } from "../lib/paths";
+import type { ChannelConfig } from "../lib/channelConfig";
+
+const ROOT = await mkdtemp(path.join(os.tmpdir(), "fetchwindow-"));
+const BIN = path.join(ROOT, "fake-ytdlp.mjs");
+const ARGS_LOG = path.join(ROOT, "argv.jsonl");
+process.env.FAKE_ARGS_LOG = ARGS_LOG;
+
+// FAKE_MODE=rumble: refuses like ffmpeg 8 on Rumble's segments unless the
+// picky option is present. =progressive: fails if the option IS present, as
+// ffmpeg does against a non-HLS input. =broken: always fails, unrelated.
+await writeFile(
+ BIN,
+ `#!/usr/bin/env node
+import { appendFileSync, writeFileSync } from "node:fs";
+const args = process.argv.slice(2);
+appendFileSync(process.env.FAKE_ARGS_LOG, JSON.stringify(args) + "\\n");
+const picky = args.includes("ffmpeg_i:-extension_picky 0");
+const mode = process.env.FAKE_MODE;
+const fail = (msg) => { process.stderr.write(msg + "\\nERROR: ffmpeg exited with code 183\\n"); process.exit(1); };
+if (mode === "broken") fail("ERROR: something else entirely");
+if (mode === "rumble" && !picky) fail("[in#0] URL https://cdn.example/x.tar?r_file=media-0.ts is not in allowed_segment_extensions");
+if (mode === "progressive" && picky) fail("Option extension_picky not found.");
+writeFileSync(args[args.indexOf("-o") + 1], "mp4");
+`,
+);
+await chmod(BIN, 0o755);
+
+async function setup(name: string) {
+ const videoDir = path.join(ROOT, name, "data", "v1");
+ await mkdir(videoDir, { recursive: true });
+ return videoDir;
+}
+
+async function runs(): Promise<string[][]> {
+ return (await readFile(ARGS_LOG, "utf8").catch(() => ""))
+ .trim().split("\n").filter(Boolean).map((l) => JSON.parse(l) as string[]);
+}
+
+function opts(videoDir: string) {
+ return {
+ channelSlug: "c",
+ channelConfig: {} as ChannelConfig,
+ paths: { ytdlpBin: BIN } as unknown as Paths,
+ videoDir,
+ videoId: "v1",
+ videoUrl: "https://rumble.example/v1.html",
+ from: 10,
+ to: 20,
+ provenance: { requestedBy: "test" },
+ onLog: () => {},
+ signal: new AbortController().signal,
+ };
+}
+
+const hasPicky = (argv: string[]) => argv.join(" ").includes(HLS_PICKY_RETRY_ARGS.join(" "));
+
+test("a Rumble window refused for its segment extension is retried once with -extension_picky 0", async () => {
+ process.env.FAKE_MODE = "rumble";
+ const before = (await runs()).length;
+ const videoDir = await setup("rumble");
+ const res = await fetchWindowManaged(opts(videoDir));
+ const mine = (await runs()).slice(before);
+ assert.equal(mine.length, 2);
+ assert.equal(hasPicky(mine[0]), false, "never on the first try");
+ assert.equal(hasPicky(mine[1]), true);
+ assert.equal(res.cached, false);
+ assert.ok(res.provenance?.ytdlp === undefined);
+ assert.equal(await readFile(res.file, "utf8"), "mp4");
+});
+
+test("a progressive source fetches on the first try, without the HLS option", async () => {
+ process.env.FAKE_MODE = "progressive";
+ const before = (await runs()).length;
+ const res = await fetchWindowManaged(opts(await setup("progressive")));
+ const mine = (await runs()).slice(before);
+ assert.equal(mine.length, 1);
+ assert.equal(hasPicky(mine[0]), false);
+ assert.equal(res.cached, false);
+});
+
+test("any other failure is not retried with the HLS option", async () => {
+ process.env.FAKE_MODE = "broken";
+ const before = (await runs()).length;
+ await assert.rejects(fetchWindowManaged(opts(await setup("broken"))), /yt-dlp failed fetching/);
+ const mine = (await runs()).slice(before);
+ assert.equal(mine.length, 1);
+});
diff --git a/common/ytdlp/fetchWindowManaged.ts b/common/ytdlp/fetchWindowManaged.ts
@@ -49,6 +49,16 @@ import { clipFormatSelector } from "./downloadFormat";
// above 720 are thrown away after paying for them.
export const DEFAULT_CLIP_MAX_HEIGHT = 720;
+// Rumble serves HLS whose segments are named `.tar`, and ffmpeg 8 refuses them
+// ("URL … is not in allowed_segment_extensions", exit 183) — every Rumble window
+// failed. `-extension_picky 0` lets them through, but it is an option of the HLS
+// DEMUXER: against a progressive URL (YouTube's googlevideo mp4) ffmpeg aborts
+// with "Option extension_picky not found". So it is a RETRY on exactly that
+// refusal, never a default — umtool's build-video.mjs does the same
+// (umtool/docs/quirks.md).
+export const HLS_EXTENSION_REFUSED = /allowed_segment_extensions|allowed_extensions/;
+export const HLS_PICKY_RETRY_ARGS = ["--downloader-args", "ffmpeg_i:-extension_picky 0"];
+
// The clip format selector lives with the download presets (the "video_720"
// preset is built from it); re-exported so existing importers keep working.
export { clipFormatSelector };
@@ -162,7 +172,7 @@ export async function fetchWindowManaged(
await rm(part, { force: true });
const maxHeight = opts.maxHeight ?? DEFAULT_CLIP_MAX_HEIGHT;
- const argsWith = (cookies: string | undefined): string[] => [
+ const argsWith = (cookies: string | undefined, retryArgs: string[] = []): string[] => [
// The operator's own yt-dlp config redirects output and attaches thumbnail
// and metadata post-processors; without this the window lands elsewhere —
// and a metadata post-processor is exactly what must not run here.
@@ -205,13 +215,14 @@ export async function fetchWindowManaged(
"--no-write-subs",
"--no-write-auto-subs",
"--no-download-archive",
+ ...retryArgs,
"-o",
part,
"--",
opts.videoUrl,
];
- const run = async (cookies: string | undefined) =>
+ const run = async (cookies: string | undefined, retryArgs: string[] = []) =>
runOneYtdlp(
{
ytdlpBin: opts.paths.ytdlpBin,
@@ -219,11 +230,12 @@ export async function fetchWindowManaged(
signal: opts.signal,
},
opts.cwd ?? videoDir,
- argsWith(cookies),
+ argsWith(cookies, retryArgs),
);
- let args = argsWith(alwaysCookies(policy));
- let outcome = await run(alwaysCookies(policy));
+ let cookiesUsed = alwaysCookies(policy);
+ let args = argsWith(cookiesUsed);
+ let outcome = await run(cookiesUsed);
if (outcome.exitCode !== 0) {
const availability = parseUnavailableFromStderr(outcome.stderrTail);
@@ -239,12 +251,22 @@ export async function fetchWindowManaged(
opts.onLog(
`Window fetch failed with ${availability}; retrying once with cookies.\n`,
);
+ cookiesUsed = retryCookies;
args = argsWith(retryCookies);
outcome = await run(retryCookies);
}
}
}
+ if (outcome.exitCode !== 0 && HLS_EXTENSION_REFUSED.test(outcome.stderrTail)) {
+ opts.onLog(
+ `Window fetch: ffmpeg refused the HLS segment extension; retrying once with -extension_picky 0.\n`,
+ );
+ await rm(part, { force: true });
+ args = argsWith(cookiesUsed, HLS_PICKY_RETRY_ARGS);
+ outcome = await run(cookiesUsed, HLS_PICKY_RETRY_ARGS);
+ }
+
if (outcome.exitCode !== 0) {
await rm(part, { force: true });
const tail = outcome.stderrTail.trim().split("\n").slice(-4).join(" / ");
