commit 561dcff167cf4f22747d415693c308a0f59c1fdd
parent f93f22e334e3ac63aeae3e4bd4add19c1dd7910b
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Wed, 7 Oct 2026 20:39:43 -0400
Merge fetch-windows-rumble-gap (Rumble windows 120 s apart, across runs)
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
3 files changed, 92 insertions(+), 22 deletions(-)
diff --git a/common/controller/fetchWindows.test.ts b/common/controller/fetchWindows.test.ts
@@ -12,6 +12,7 @@ import os from "node:os";
import path from "node:path";
import {
CLIP_WINDOW_MIN_GAP_SECONDS,
+ CLIP_WINDOW_PLATFORM_MIN_GAP_SECONDS,
fetchWindows,
type FetchWindowsDeps,
type FetchWindowsItem,
@@ -84,6 +85,8 @@ function harness(over: Partial<FetchWindowsDeps> = {}) {
rec.cleans.push(platform);
return null;
},
+ gapRemainingMs: () => 0,
+ noteGap: () => {},
...over,
};
return { rec, deps };
@@ -239,3 +242,35 @@ test("a duplicated window is fetched once", async () => {
assert.equal(r.fetched.length, 1);
assert.equal((await spawns()) - before, 1);
});
+
+test("rumble windows are further apart, whatever platform their channel names", async () => {
+ const h = harness();
+ const rumble = (id: string): FetchWindowsItem => ({ slug: "c", id, from: 10, to: 20, webpageUrl: `https://rumble.com/${id}-x.html` });
+ const r = await fetchWindows({
+ paths: await corpus(),
+ items: [rumble("v1"), rumble("v2")],
+ provenance: { requestedBy: "test" },
+ deps: h.deps,
+ });
+ assert.equal(r.fetched.length, 2);
+ assert.equal(h.rec.sleeps.length, 1);
+ const floor = CLIP_WINDOW_PLATFORM_MIN_GAP_SECONDS.rumble * 1000;
+ assert.ok(h.rec.sleeps[0] >= floor, `${h.rec.sleeps[0]} is at least rumble's ${floor}`);
+ assert.ok(h.rec.sleeps[0] <= floor * 1.5 + 60_000);
+});
+
+test("a second run on the same platform waits out the gap the first one set", async () => {
+ const next = new Map<string, number>();
+ const shared = {
+ gapRemainingMs: (key: string) => next.get(key) ?? 0,
+ noteGap: (key: string, ms: number) => void next.set(key, ms),
+ };
+ const first = harness(shared);
+ await go(await corpus(), [item("v1", "/ok"), item("v2", "/403")], first);
+ assert.deepEqual(first.rec.sleeps, [1000], "a run's own first fetch owes nothing");
+ assert.equal(next.get("clip-window:youtube"), 1000, "a refused fetch sets the gap too");
+
+ const second = harness(shared);
+ await go(await corpus(), [item("v3", "/ok"), item("v4", "/ok")], second);
+ assert.deepEqual(second.rec.sleeps, [1000, 1000], "the first fetch waits for the earlier run's gap");
+});
diff --git a/common/controller/fetchWindows.ts b/common/controller/fetchWindows.ts
@@ -10,6 +10,7 @@ import { detectPlatform } from "../lib/platform";
import { findContainingClipWindow } from "../lib/clipWindow-server";
import type { JobProgress } from "../jobs/registry";
import { downloadGapMs } from "../jobs/platformBackoff";
+import { notePlatformGap, platformGapRemainingMs } from "../jobs/platformGap";
import {
heldPlatformRefusal,
platformCooldownRemainingMs,
@@ -25,6 +26,7 @@ import {
import {
channelPaceSeconds,
channelPlatform,
+ configForVideoUrl,
} from "../ytdlp/channelArgs";
import {
platformMinGapSeconds,
@@ -79,6 +81,15 @@ import { findVideoSourceUrl } from "./undownloadedVideos";
// a scraper: 20 s apart was enough for YouTube to answer 403 (2026-10-07).
export const CLIP_WINDOW_MIN_GAP_SECONDS = 30;
+// A platform whose windows must be further apart than that. Rumble's
+// Cloudflare puts the whole IP behind a JS challenge ("Just a moment…", 403 to
+// every rumble.com request, impersonated or not) after a handful of requests
+// in a few minutes, and a window costs three (page, embed JSON, HLS manifest):
+// on 2026-10-07 windows 30–45 s apart drew it within four, and it lifted again
+// in about five minutes of quiet.
+export const CLIP_WINDOW_PLATFORM_MIN_GAP_SECONDS: Readonly<Record<string, number>> =
+ Object.freeze({ rumble: 120 });
+
export type FetchWindowsItem = {
slug: string;
id: string;
@@ -153,6 +164,11 @@ export type FetchWindowsDeps = {
failureClass: "rate_limit" | "network",
) => Promise<void>;
recordClean: (platform: string, paths: Paths) => Promise<string | null>;
+ // The gap ACROSS runs (jobs/platformGap.ts, keyed `clip-window:<platform>`):
+ // two batches back to back on one queue — a umtool manifest each — must not
+ // put the second one's first fetch right after the first one's last.
+ gapRemainingMs: (key: string) => number;
+ noteGap: (key: string, gapMs: number) => void;
};
function abortableSleep(ms: number, signal?: AbortSignal): Promise<void> {
@@ -185,6 +201,8 @@ const DEFAULT_DEPS: FetchWindowsDeps = {
recordBackoff: (platform, paths, failureClass) =>
recordDownloadBackoff(platform, paths, failureClass),
recordClean: (platform, paths) => recordPlatformClean(platform, paths),
+ gapRemainingMs: (key) => platformGapRemainingMs(key),
+ noteGap: (key, gapMs) => notePlatformGap(key, gapMs),
};
export function fetchWindowsItemLabel(item: FetchWindowsItem): string {
@@ -349,28 +367,38 @@ export async function fetchWindows({
// The cooldown's key, as the single fetch and every download path use it.
const platform = detectPlatform(url) ?? "unknown";
- if (networkAttempts > 0) {
- const gap =
- gapMs ??
- downloadGapMs(
- config.sleepBetweenDownloadsSeconds ?? settings.sleepBetweenDownloadsSeconds,
- channelPaceSeconds(config),
- staticSleepRequestsSeconds(channelPlatform(config)),
- {
- minSeconds: Math.max(
- CLIP_WINDOW_MIN_GAP_SECONDS,
- platformMinGapSeconds(channelPlatform(config)),
- ),
- },
- );
- if (gap > 0) {
- log(`Sleeping ${Math.round(gap / 1000)}s before the next fetch...`);
- await deps.sleep(gap, signal);
- const after = interrupted();
- if (after) {
- stop(after, i);
- break;
- }
+ // Paced as the platform the URL points at, which a channel with no
+ // platform of its own (community-notes holds Rumble videos) does not name.
+ const paced = configForVideoUrl(config, url);
+ const gap =
+ gapMs ??
+ downloadGapMs(
+ config.sleepBetweenDownloadsSeconds ?? settings.sleepBetweenDownloadsSeconds,
+ channelPaceSeconds(paced),
+ staticSleepRequestsSeconds(channelPlatform(paced)),
+ {
+ minSeconds: Math.max(
+ CLIP_WINDOW_MIN_GAP_SECONDS,
+ platformMinGapSeconds(channelPlatform(paced)),
+ CLIP_WINDOW_PLATFORM_MIN_GAP_SECONDS[platform] ?? 0,
+ ),
+ },
+ );
+ const gapKey = `clip-window:${platform}`;
+ // Between this run's own fetches, the gap; before its first, whatever is
+ // left of the gap the last run on this platform set.
+ const wait = networkAttempts > 0 ? gap : deps.gapRemainingMs(gapKey);
+ if (wait > 0) {
+ log(
+ networkAttempts > 0
+ ? `Sleeping ${Math.round(wait / 1000)}s before the next fetch...`
+ : `${platform} was asked for a window by an earlier run: waiting ${Math.round(wait / 1000)}s.`,
+ );
+ await deps.sleep(wait, signal);
+ const after = interrupted();
+ if (after) {
+ stop(after, i);
+ break;
}
}
@@ -452,6 +480,10 @@ export async function fetchWindows({
networkStreak = 0;
}
continue;
+ } finally {
+ // Settled, refused or not: the next fetch on this platform, in this run
+ // or the next one, starts no sooner than the gap from now.
+ deps.noteGap(gapKey, gap);
}
done += 1;
progress();
diff --git a/umtool/docs/quirks.md b/umtool/docs/quirks.md
@@ -38,6 +38,9 @@ that channel's args they went without `--impersonate` and Cloudflare 403'd them.
editor's `fetch-windows` job) treats one 403 as that clip's failure — a removed
Rumble page answers 403 too — but two in a row back the platform off and stop the
job, as a 429 does at once. The windows are 30–45 s apart; 20 s drew YouTube 403s.
+Rumble's are 120–180 s apart, across jobs too: its Cloudflare puts the whole IP
+behind a JS challenge ("Just a moment…", 403 to everything) after a handful of
+requests in a few minutes, and it lifts after a few quiet minutes.
**`--force-keyframes-at-cuts` matters because the clip IS the citation.** Without
it the cut snaps to the nearest preceding keyframe, which can be seconds early.
