commit 113303d18c87ba7012a8bd897abed3b76a336572
parent ca4fd1ff99a0f8ab8951c2b76d175ceb63cd8881
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 25 Sep 2026 11:09:49 -0400
e2e: pacing.spec — a deferred video is skipped; a live 429 defers and moves on
T1 seeds download.videoDeferrals.a1 and proves the runner picks only a2,
idles "deferred", and /operations/download draws the "Rate-limit cooldown"
region with "Deferred videos" listing alpha/a1. T2 drives a real 429
through the fake's new `dl429` sentinel (managed single-URL download only;
the prefetch still succeeds, the two-spawn shape): youtube backs off
(fails 1), dl429vid1 is deferred ~6 h, and after the real ~60 s cooldown
the next pick is a2.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
2 files changed, 234 insertions(+), 0 deletions(-)
diff --git a/editor/e2e/fixtures/bin/fake-ytdlp.mjs b/editor/e2e/fixtures/bin/fake-ytdlp.mjs
@@ -444,6 +444,17 @@ async function modeYoutubeSingleUrlManaged(url) {
`youtube-single:${url} cookies=${cookieArg()}\n`,
);
if (cookieGateBlocked(url)) failCookieGate(url);
+ // `dl429`: the 2026-09-24 YouTube shape — the metadata prefetch succeeded
+ // (the prefetch branch has no such sentinel), then the real download's
+ // SUBTITLE fetch answers 429. Per video, not IP-wide: availability.ts
+ // classifies it rate_limit, so the runner backs youtube off AND defers this
+ // one video (pacing.spec.ts).
+ if (url.toLowerCase().includes("dl429")) {
+ process.stderr.write(
+ `ERROR: [youtube] ${url}: Unable to download video subtitles for 'en': HTTP Error 429: Too Many Requests\n`,
+ );
+ process.exit(1);
+ }
process.stdout.write(`[fake-ytdlp] managed single-URL ${id}\n`);
// URL sentinels for no-subs-fallback tests. The sentinels live in the
diff --git a/editor/e2e/pacing.spec.ts b/editor/e2e/pacing.spec.ts
@@ -0,0 +1,223 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { test, expect } from "@playwright/test";
+import { readJson, resetData, resolvePath, writeSettings } from "./helpers";
+import { baseUrl } from "./baseUrl";
+
+// YouTube lane pacing (release 7): a rate-limited video is DEFERRED for 6 h as
+// well as backing the platform off, so after the cooldown the auto-download
+// runner moves on to the NEXT video instead of re-picking the same one. The
+// pure decision is unit-tested (common/jobs/unitOutcome.test.ts); this proves
+// the runner's pick honours a persisted deferral, the lane says why it idles,
+// the page shows it, and a live 429 (the fake's `dl429` sentinel) defers.
+
+type Leaf = { id: string; match: { type: string; value?: string } };
+type Group = { id: string; mode: string; children: (Group | Leaf)[] };
+
+const ONE_WORKER = [
+ {
+ id: "w1",
+ name: "W1",
+ kind: "local",
+ enabled: true,
+ priority: 0,
+ appId: "whisper-cpp",
+ config: {},
+ },
+];
+
+const STATE_FILE = "test-transcripts/.auto-queue/state.json";
+
+// Copied from auto-queue.spec.ts: a YouTube channel with undownloaded videos.
+async function makeDownloadChannel(slug: string, ids: string[]) {
+ const root = resolvePath(`test-transcripts/channels/${slug}`);
+ await mkdir(`${root}/data`, { recursive: true });
+ await writeFile(
+ `${root}/config.json`,
+ JSON.stringify({
+ handling: "youtube",
+ name: slug,
+ url: `https://www.youtube.com/@${slug}/videos`,
+ }),
+ );
+ await writeFile(
+ `${root}/playlist`,
+ ids.map((id) => `https://www.youtube.com/watch?v=${id}`).join("\n") + "\n",
+ );
+ await writeFile(
+ `${root}/snapshot.json`,
+ JSON.stringify({
+ generatedAt: "2026-06-01T00:00:00.000Z",
+ totals: { videos: ids.length, transcribed: 0, downloaded: 0 },
+ buckets: {
+ noTranscript: [],
+ downloadedNoTranscript: [],
+ untranscoded: [],
+ multipleAudioFormats: [],
+ transcribedWithAudio: [],
+ untranscribable: [],
+ noMetadata: [],
+ failedListed: [],
+ missingFromArchive: [],
+ duplicateDirs: [],
+ partialDownloads: [],
+ corruptSource: [],
+ nonStandardVtt: [],
+ skippedByFilter: [],
+ },
+ undownloadedIds: ids,
+ }),
+ );
+}
+
+async function setup(ids: string[]) {
+ await resetData(null);
+ await makeDownloadChannel("alpha", ids);
+ const root: Group = {
+ id: "root",
+ mode: "strict",
+ children: [{ id: "leaf-alpha", match: { type: "channel", value: "alpha" } }],
+ };
+ await writeSettings({
+ adminTitle: "Test Admin",
+ maxTranscriptPageBytes: 8388608,
+ sleepBetweenDownloadsSeconds: 0,
+ minFreeDiskGB: 0,
+ workers: ONE_WORKER,
+ autoQueue: {
+ transcription: {},
+ download: { enabled: true, maxWorkers: null, root },
+ },
+ });
+}
+
+type KindStatus = {
+ runner: { running: boolean; idleReason: string | null };
+ picks: { videoId: string }[];
+ deferred: { videoId: string; channelSlug: string; untilMs: number }[];
+};
+type Status = { download: KindStatus };
+
+async function getStatus(
+ request: import("@playwright/test").APIRequestContext,
+): Promise<Status> {
+ const res = await request.get(`${baseUrl}/api/auto-queue/status`);
+ expect(res.ok()).toBeTruthy();
+ return res.json();
+}
+
+// Chronological pick order (the status log is newest-first).
+function pickOrder(status: Status): string[] {
+ return status.download.picks.map((p) => p.videoId).reverse();
+}
+
+async function control(
+ request: import("@playwright/test").APIRequestContext,
+ action: "start" | "stop",
+) {
+ const res = await request.post(`${baseUrl}/api/auto-queue/control`, {
+ data: { kind: "download", action },
+ });
+ if (action === "start") expect(res.ok()).toBeTruthy();
+}
+
+type PersistedDownload = {
+ platformBackoff: Record<string, { until: number; fails: number }>;
+ videoDeferrals?: Record<string, { until: number; channelSlug: string }>;
+};
+
+async function persistedDownload(): Promise<PersistedDownload | null> {
+ try {
+ return (await readJson<{ download: PersistedDownload }>(STATE_FILE))
+ .download;
+ } catch {
+ return null;
+ }
+}
+
+test.afterEach(async ({ request }) => {
+ await control(request, "stop");
+});
+
+test("a deferred video is skipped by auto-download, and the lane says so", async ({
+ page,
+ request,
+}) => {
+ await setup(["a1", "a2"]);
+ // Seed a live deferral on a1 — as if its download had just answered 429.
+ await mkdir(resolvePath("test-transcripts/.auto-queue"), { recursive: true });
+ await writeFile(
+ resolvePath(STATE_FILE),
+ JSON.stringify({
+ transcription: { runtime: {}, picks: [], platformBackoff: {} },
+ download: {
+ runtime: {},
+ picks: [],
+ platformBackoff: {},
+ videoDeferrals: {
+ a1: { until: Date.now() + 60 * 60_000, channelSlug: "alpha" },
+ },
+ },
+ }),
+ );
+
+ await control(request, "start");
+
+ await expect
+ .poll(async () => (await getStatus(request)).download.runner.idleReason, {
+ timeout: 60_000,
+ })
+ .toBe("deferred");
+ const status = await getStatus(request);
+ // a1 is listed first, and was never picked.
+ expect(pickOrder(status)).toEqual(["a2"]);
+ expect(status.download.deferred.map((d) => d.videoId)).toEqual(["a1"]);
+
+ await page.goto("/operations/download");
+ const region = page.getByRole("region", { name: "Rate-limit cooldown" });
+ await expect(region).toBeVisible({ timeout: 20_000 });
+ const list = region.getByRole("list", { name: "Deferred videos" });
+ await expect(list).toContainText("alpha/a1");
+ // Nothing is cooling, so the platform list is not drawn at all.
+ await expect(
+ region.getByRole("list", { name: "Platforms in cooldown" }),
+ ).toHaveCount(0);
+ await expect(page.getByText(/rate-limited recently and is deferred/)).toBeVisible();
+});
+
+test("a live 429 backs youtube off, defers the video, and the next pick is the next video", async ({
+ request,
+}) => {
+ // A REAL cooldown: base 60 s ±10 %. The whole point is what happens when it
+ // lapses — the runner must move on to a2, not re-pick dl429vid1.
+ test.setTimeout(150_000);
+ await setup(["dl429vid1", "a2"]);
+
+ await control(request, "start");
+
+ await expect
+ .poll(
+ async () => {
+ const d = await persistedDownload();
+ return {
+ deferred: Boolean(d?.videoDeferrals?.dl429vid1),
+ fails: d?.platformBackoff.youtube?.fails ?? 0,
+ };
+ },
+ { timeout: 30_000 },
+ )
+ .toEqual({ deferred: true, fails: 1 });
+ const d = await persistedDownload();
+ expect(d?.videoDeferrals?.dl429vid1?.channelSlug).toBe("alpha");
+ // Deferred for ~6 h, not for the cooldown's minute.
+ expect(d!.videoDeferrals!.dl429vid1.until - Date.now()).toBeGreaterThan(
+ 5 * 60 * 60_000,
+ );
+
+ await expect
+ .poll(async () => pickOrder(await getStatus(request)), {
+ timeout: 100_000,
+ intervals: [2_000],
+ })
+ .toContain("a2");
+ expect(pickOrder(await getStatus(request))).toEqual(["dl429vid1", "a2"]);
+});
