import { mkdir, writeFile } from "node:fs/promises"; import { test, expect } from "@playwright/test"; import { readJson, resetData, resolvePath, writeSettings } from "./helpers"; import { baseUrl } from "./baseUrl"; // YouTube lane pacing (release 7): a rate-limited video is DEFERRED for 6 h as // well as backing the platform off, so after the cooldown the auto-download // runner moves on to the NEXT video instead of re-picking the same one. The // pure decision is unit-tested (common/jobs/unitOutcome.test.ts); this proves // the runner's pick honours a persisted deferral, the lane says why it idles, // the page shows it, and a live 429 (the fake's `wp429` sentinel — a refused // watch page; a SUBTITLE 429, `dl429`, no longer backs the platform off since // release 17 slice RL, see rate-limit.spec.ts) defers. type Leaf = { id: string; match: { type: string; value?: string } }; type Group = { id: string; mode: string; children: (Group | Leaf)[] }; const ONE_WORKER = [ { id: "w1", name: "W1", kind: "local", enabled: true, priority: 0, appId: "whisper-cpp", config: {}, }, ]; const STATE_FILE = "test-transcripts/.auto-queue/state.json"; // Copied from auto-queue.spec.ts: a YouTube channel with undownloaded videos. async function makeDownloadChannel(slug: string, ids: string[]) { const root = resolvePath(`test-transcripts/channels/${slug}`); await mkdir(`${root}/data`, { recursive: true }); await writeFile( `${root}/config.json`, JSON.stringify({ handling: "youtube", name: slug, url: `https://www.youtube.com/@${slug}/videos`, }), ); await writeFile( `${root}/playlist`, ids.map((id) => `https://www.youtube.com/watch?v=${id}`).join("\n") + "\n", ); await writeFile( `${root}/snapshot.json`, JSON.stringify({ generatedAt: "2026-06-01T00:00:00.000Z", totals: { videos: ids.length, transcribed: 0, downloaded: 0 }, buckets: { noTranscript: [], downloadedNoTranscript: [], untranscoded: [], multipleAudioFormats: [], transcribedWithAudio: [], untranscribable: [], noMetadata: [], failedListed: [], missingFromArchive: [], duplicateDirs: [], partialDownloads: [], corruptSource: [], nonStandardVtt: [], skippedByFilter: [], }, undownloadedIds: ids, }), ); } async function setup(ids: string[]) { await resetData(null); await makeDownloadChannel("alpha", ids); const root: Group = { id: "root", mode: "strict", children: [{ id: "leaf-alpha", match: { type: "channel", value: "alpha" } }], }; await writeSettings({ adminTitle: "Test Admin", maxTranscriptPageBytes: 8388608, sleepBetweenDownloadsSeconds: 0, minFreeDiskGB: 0, workers: ONE_WORKER, autoQueue: { transcription: {}, download: { enabled: true, maxWorkers: null, root }, }, }); } type KindStatus = { runner: { running: boolean; idleReason: string | null }; picks: { videoId: string }[]; deferred: { videoId: string; channelSlug: string; untilMs: number }[]; }; type Status = { download: KindStatus }; async function getStatus( request: import("@playwright/test").APIRequestContext, ): Promise { const res = await request.get(`${baseUrl}/api/auto-queue/status`); expect(res.ok()).toBeTruthy(); return res.json(); } // Chronological pick order (the status log is newest-first). function pickOrder(status: Status): string[] { return status.download.picks.map((p) => p.videoId).reverse(); } async function control( request: import("@playwright/test").APIRequestContext, action: "start" | "stop", ) { const res = await request.post(`${baseUrl}/api/auto-queue/control`, { // Behind the ops token since release 19 (A3): it starts and stops lanes. headers: { authorization: "Bearer test-worker-token" }, data: { kind: "download", action }, }); if (action === "start") expect(res.ok()).toBeTruthy(); } type PersistedDownload = { platformBackoff: Record; videoDeferrals?: Record; }; async function persistedDownload(): Promise { try { return (await readJson<{ download: PersistedDownload }>(STATE_FILE)) .download; } catch { return null; } } test.afterEach(async ({ request }) => { await control(request, "stop"); }); test("a deferred video is skipped by auto-download, and the lane says so", async ({ page, request, }) => { await setup(["a1", "a2"]); // Seed a live deferral on a1 — as if its download had just answered 429. await mkdir(resolvePath("test-transcripts/.auto-queue"), { recursive: true }); await writeFile( resolvePath(STATE_FILE), JSON.stringify({ transcription: { runtime: {}, picks: [], platformBackoff: {} }, download: { runtime: {}, picks: [], platformBackoff: {}, videoDeferrals: { a1: { until: Date.now() + 60 * 60_000, channelSlug: "alpha" }, }, }, }), ); await control(request, "start"); await expect .poll(async () => (await getStatus(request)).download.runner.idleReason, { timeout: 60_000, }) .toBe("deferred"); const status = await getStatus(request); // a1 is listed first, and was never picked. expect(pickOrder(status)).toEqual(["a2"]); expect(status.download.deferred.map((d) => d.videoId)).toEqual(["a1"]); await page.goto("/operations/download"); const region = page.getByRole("region", { name: "Rate-limit cooldown" }); await expect(region).toBeVisible({ timeout: 20_000 }); const list = region.getByRole("list", { name: "Deferred videos" }); await expect(list).toContainText("alpha/a1"); // Nothing is cooling, so the platform list is not drawn at all. await expect( region.getByRole("list", { name: "Platforms in cooldown" }), ).toHaveCount(0); // The idle sentence (dispatch.ts) — drawn by the rail and by the lane, so // both say it; one visible is the assertion. await expect( page.getByText(/every pending video was rate-limited recently and is deferred/).first(), ).toBeVisible(); }); test("a live 429 backs youtube off, defers the video, and the next pick is the next video", async ({ request, }) => { // A REAL cooldown, on a shorter clock: the base is 60 s ±10 % in production // and 20 s on the test server (E2E_BACKOFF_BASE_MS, playwright.config.ts). // The whole point is what happens when it lapses — the runner must move on to // a2, not re-pick wp429vid1. test.setTimeout(150_000); await setup(["wp429vid1", "a2"]); await control(request, "start"); await expect .poll( async () => { const d = await persistedDownload(); return { deferred: Boolean(d?.videoDeferrals?.wp429vid1), fails: d?.platformBackoff.youtube?.fails ?? 0, }; }, { timeout: 30_000 }, ) .toEqual({ deferred: true, fails: 1 }); const d = await persistedDownload(); expect(d?.videoDeferrals?.wp429vid1?.channelSlug).toBe("alpha"); // Deferred for ~6 h, not for the cooldown's minute. expect(d!.videoDeferrals!.wp429vid1.until - Date.now()).toBeGreaterThan( 5 * 60 * 60_000, ); await expect .poll(async () => pickOrder(await getStatus(request)), { timeout: 100_000, intervals: [2_000], }) .toContain("a2"); expect(pickOrder(await getStatus(request))).toEqual(["wp429vid1", "a2"]); });