commit 99623e3e75c2837c6267d474321f0a82e00c1959
parent db0387bd840b65ed295c18cf6b9afc145bcac3ab
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 24 Sep 2026 21:58:41 -0400
editor: e2e — a Rumble sweep that 429s mid-listing (sweep429 sentinel, argv recorded)
The fake yt-dlp records every flat-playlist enumeration's whole argv
(`flat-playlist:full|paged argv=…` in fake-ytdlp.invocations) and gains a
`sweep429` URL sentinel: a full enumeration prints three pages of five urls
with RumbleChannel "Downloading page N" lines, then HTTP 429, exit 1; a paged
walk (-I) lists normally. rumble-sweep.spec.ts pins the slice: platform args on
both spawns, job succeeds as a paged walk, the "sweep incomplete" line, the
rumble cooldown in .auto-queue/state.json, no lastFullSweepAt, playlist
untouched, and the next Sync refused for the cooldown.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
2 files changed, 191 insertions(+), 0 deletions(-)
diff --git a/editor/e2e/fixtures/bin/fake-ytdlp.mjs b/editor/e2e/fixtures/bin/fake-ytdlp.mjs
@@ -535,6 +535,31 @@ async function main() {
has("--print") &&
arg("--print") === "url"
) {
+ // Every enumeration records its WHOLE argv, in order, so a spec can assert
+ // the platform args (a rumble.com URL gets `--impersonate chrome
+ // --sleep-requests 1`) and their position before the channel's own.
+ await appendFile(
+ "fake-ytdlp.invocations",
+ `flat-playlist:${has("-I") ? "paged" : "full"} argv=${argv.join(" ")}\n`,
+ );
+ // `sweep429`: a FULL enumeration (no -I range) prints three pages of five
+ // urls, logging each page the way yt-dlp's RumbleChannel extractor does,
+ // then hits HTTP 429 and exits 1 — the 2026-09-24 the-quartering-rumble
+ // sweep in miniature. A paged walk (-I) of the same channel lists normally.
+ if ((lastNonFlag() ?? "").toLowerCase().includes("sweep429") && !has("-I")) {
+ for (let page = 1; page <= 3; page++) {
+ process.stderr.write(`[RumbleChannel] sweep429: Downloading page ${page}\n`);
+ for (let i = 1; i <= 5; i++) {
+ process.stdout.write(
+ `https://rumble.com/vsweep${page}${i}-listed.html\n`,
+ );
+ }
+ }
+ process.stderr.write(
+ "ERROR: sweep429: Unable to download webpage: HTTP Error 429: Too Many Requests (caused by <HTTPError 429: Too Many Requests>)\n",
+ );
+ process.exit(1);
+ }
await modeStorePlaylist();
return;
}
diff --git a/editor/e2e/rumble-sweep.spec.ts b/editor/e2e/rumble-sweep.spec.ts
@@ -0,0 +1,166 @@
+import { readFile, writeFile } from "node:fs/promises";
+import { test, expect } from "@playwright/test";
+import {
+ channelStage,
+ generateReport,
+ readJson,
+ resetData,
+ resolvePath,
+ writeSettings,
+} from "./helpers";
+
+// Release 5 slice R. A Rumble channel's full sweep that hits HTTP 429 part-way
+// through the listing (the 2026-09-24 the-quartering-rumble run, in miniature):
+//
+// - every yt-dlp spawn for a rumble.com URL carries the platform args
+// (`--impersonate chrome --sleep-requests 1`, channelArgs.ts PLATFORM_ARGS);
+// - the partial listing is "incomplete", not a listing and not a failure: the
+// job records the rumble cooldown, logs one line, runs the paged walk
+// instead and SUCCEEDS, and lastFullSweepAt is never stamped;
+// - the next Sync is refused for the cooldown, like any other 429.
+//
+// The fake yt-dlp's `sweep429` URL sentinel prints three pages of five urls
+// with RumbleChannel-style "Downloading page N" lines, then the 429, exit 1 —
+// for a FULL enumeration only; the paged walk (-I) lists normally.
+
+const CHANNEL = "availability-test";
+const FIXTURE = "availability-baseline";
+const channelRoot = `test-transcripts/channels/${CHANNEL}`;
+const RUMBLE_URL = "https://rumble.com/c/sweep429";
+
+const ALL_IDS = [
+ "vidpublic1",
+ "vidpublic2",
+ "viddeleted1",
+ "vidprivate1",
+ "vidmembers1",
+ "vidneedsauth1",
+];
+
+type ChannelConfigFile = {
+ url?: string;
+ lastSyncedAt?: string;
+ lastFullSweepAt?: string;
+};
+
+type AutoQueueStateFile = {
+ download: {
+ platformBackoff: Record<string, { until: number; fails: number }>;
+ };
+};
+
+async function readConfig(): Promise<ChannelConfigFile> {
+ return readJson<ChannelConfigFile>(`${channelRoot}/config.json`);
+}
+
+async function readInvocations(): Promise<string[]> {
+ const raw = await readFile(
+ resolvePath(`${channelRoot}/fake-ytdlp.invocations`),
+ "utf8",
+ ).catch(() => "");
+ return raw.split("\n").filter(Boolean);
+}
+
+test("a Rumble sweep that 429s mid-listing is incomplete: paged walk, cooldown, no sweep stamp", async ({
+ page,
+}) => {
+ await resetData(FIXTURE);
+ // The sweep is off in the e2e default settings; this spec needs it due.
+ await writeSettings({
+ adminTitle: "Test Admin",
+ maxTranscriptPageBytes: 8388608,
+ sleepBetweenDownloadsSeconds: 0,
+ minFreeDiskGB: 0,
+ verifyAvailabilityBeforeClean: false,
+ syncScheduler: {
+ enabled: false,
+ fullSweepIntervalMinutes: 1440,
+ fullSweepConfirmMaxSuspects: 25,
+ fullSweepShrinkGuardPercent: 10,
+ },
+ });
+ // Point the fixture channel at a Rumble URL carrying the sentinel.
+ const cfgPath = resolvePath(`${channelRoot}/config.json`);
+ const cfg = JSON.parse(await readFile(cfgPath, "utf8")) as ChannelConfigFile;
+ await writeFile(cfgPath, JSON.stringify({ ...cfg, url: RUMBLE_URL }, null, 2));
+ // The paged walk's listing: everything already archived, so it stops on its
+ // first page and downloads nothing — the download set is not the point here.
+ await writeFile(
+ resolvePath(`${channelRoot}/.fake-ytdlp-flat-playlist.json`),
+ JSON.stringify({ ids: ALL_IDS }),
+ );
+ await writeFile(
+ resolvePath(`${channelRoot}/archive`),
+ ALL_IDS.map((id) => `youtube ${id}`).join("\n") + "\n",
+ );
+ const storedPlaylist = "https://rumble.com/vstale-stored.html\n";
+ await writeFile(resolvePath(`${channelRoot}/playlist`), storedPlaylist);
+
+ await generateReport(page, CHANNEL);
+ await page.goto(channelStage(CHANNEL, "playlist"));
+ // Retried: a click landing before hydration fires nothing (sync-deep's idiom).
+ // lastSyncedAt advancing is the success signal — a failed sync never stamps it.
+ let synced = false;
+ for (let attempt = 0; attempt < 5 && !synced; attempt++) {
+ await page
+ .getByRole("button", { name: "Sync", exact: true })
+ .click({ timeout: 5_000 })
+ .catch(() => {});
+ for (let i = 0; i < 60; i++) {
+ const c = await readConfig().catch(() => null);
+ if (c?.lastSyncedAt) {
+ synced = true;
+ break;
+ }
+ await new Promise((r) => setTimeout(r, 250));
+ }
+ }
+ expect(synced, "the sync job succeeded (lastSyncedAt stamped)").toBe(true);
+
+ // The one "incomplete" line, naming the page, the count and the platform.
+ const output = page.getByLabel("Sync output");
+ await expect(output).toContainText(
+ "sweep incomplete: 429 at page 3 of the listing, 15 entries",
+ { timeout: 15_000 },
+ );
+ await expect(output).toContainText("rumble cooldown");
+
+ // Platform args on every spawn: the full enumeration AND the paged walk.
+ const inv = await readInvocations();
+ const full = inv.filter((l) => l.startsWith("flat-playlist:full"));
+ const paged = inv.filter((l) => l.startsWith("flat-playlist:paged"));
+ expect(full).toHaveLength(1);
+ expect(paged.length).toBeGreaterThan(0);
+ for (const line of [...full, ...paged]) {
+ expect(line).toContain("--impersonate chrome");
+ expect(line).toContain("--sleep-requests 1");
+ expect(line).toContain(RUMBLE_URL);
+ }
+
+ // The rumble cooldown is recorded where the Sync gate and the runner read it.
+ const state = await readJson<AutoQueueStateFile>(
+ "test-transcripts/.auto-queue/state.json",
+ );
+ expect(state.download.platformBackoff.rumble?.until).toBeGreaterThan(
+ Date.now(),
+ );
+
+ // Not a listing: no sweep stamp, stored playlist untouched.
+ const after = await readConfig();
+ expect(after.lastFullSweepAt).toBeUndefined();
+ expect(await readFile(resolvePath(`${channelRoot}/playlist`), "utf8")).toBe(
+ storedPlaylist,
+ );
+
+ // The next Sync is refused for the cooldown, as a notice.
+ await page.goto(channelStage(CHANNEL, "playlist"));
+ await expect(async () => {
+ await page
+ .getByRole("button", { name: "Sync", exact: true })
+ .click({ timeout: 5_000 });
+ await expect(page.getByLabel("Sync notice")).toContainText(
+ "rumble is in a rate-limit cooldown",
+ { timeout: 3_000 },
+ );
+ }).toPass({ timeout: 30_000 });
+});
