import { test, expect } from "@playwright/test"; import { existsSync, readFileSync, statSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; // --------------------------------------------------------------------------- // Fetching a clip window BY ASKING THE EDITOR. // // The operator's rule is that no yt-dlp runs by hand, so the bench's "fetch // more" now posts to the editor and the bytes land in the CORPUS beside the // video — where the next report citing the same stream gets them for free. // // The editor is STUBBED (e2e/fixtures/editor-stub.mjs, a second webServer): // running a real one would arm its runners against whatever corpus they found, // and the endpoint's whole job is to spend somebody's bandwidth. What the stub // records is the part a file on disk cannot prove — that the provenance and the // bearer token actually crossed the wire. // // The strongest assertion here is a NEGATIVE one: the fixture's yt-dlp stub // logs every invocation, and this path must produce none. // --------------------------------------------------------------------------- const HERE = path.dirname(fileURLToPath(import.meta.url)); const FIXTURE = path.join(HERE, "..", ".e2e-song"); const PROJECT = "reports/editor-fetch-fixture"; const MANY = "reports/editor-fetch-many-fixture"; const REUSE = "reports/editor-fetch-reuse-fixture"; const STUB_LOG = path.join(FIXTURE, "editor-stub.requests.json"); const YTDLP_LOG = path.join(FIXTURE, "bin", "yt-dlp.invocations"); const clipsDir = (video: string) => path.join(FIXTURE, "channels", "testchan", "data", video, "clips"); type StubRequest = { channelSlug?: string; videoId?: string; from?: number; to?: number; requestedBy?: string; manifest?: string; clipId?: string; reason?: string; authorization?: string; }; const stubRequests = (): StubRequest[] => { try { return JSON.parse(readFileSync(STUB_LOG, "utf8")) as StubRequest[]; } catch { return []; } }; const ytdlpCalls = (): number => { try { return readFileSync(YTDLP_LOG, "utf8").split("\n").filter(Boolean).length; } catch { return 0; } }; // BY ID. A bare GET answers with `runningJob()`, which is null the moment the // job ends -- so a FAILED job polled a beat late reads as "nothing running", // which is indistinguishable from success. async function waitForJob( request: import("@playwright/test").APIRequestContext, baseURL: string, jobId?: string, ): Promise<{ state: string; error: string | null }> { let last = { state: "running", error: null as string | null }; const url = jobId ? `${baseURL}/api/report/fetch?job=${encodeURIComponent(jobId)}` : `${baseURL}/api/report/fetch`; await expect .poll( async () => { const r = await request.get(url); const j = (await r.json()) as { job: typeof last | null }; if (j.job) last = j.job; return j.job?.state ?? "done"; }, { timeout: 60_000 }, ) .not.toBe("running"); return last; } test("the fetch route asks the editor, carries the provenance, and runs no yt-dlp", async ({ page, request, baseURL, }) => { test.setTimeout(90_000); const ytdlpBefore = ytdlpCalls(); const asksBefore = stubRequests().length; const post = await request.post(`${baseURL}/api/report/fetch`, { data: { project: PROJECT, clip: "e01", padBefore: 2, padAfter: 2 }, }); expect(post.status(), await post.text()).toBe(202); const started = (await post.json()) as { job: { id: string } }; const job = await waitForJob(request, baseURL!, started.job.id); expect(job.state, job.error ?? "").toBe("done"); // WHAT CROSSED THE WIRE. A window with no requester is a window nobody can // account for in six months, which is the reason the field is required. const asks = stubRequests().slice(asksBefore); expect(asks.length).toBe(1); const ask = asks[0]; expect(ask.authorization).toBe("Bearer umtool-e2e-token"); expect(ask.channelSlug).toBe("testchan"); expect(ask.videoId).toBe("vid3"); expect(ask.requestedBy).toBe("umtool"); expect(ask.manifest).toBe("editor-fetch-fixture"); expect(ask.clipId).toBe("e01"); // The clip's own note is the closest thing the manifest has to a reason. expect(ask.reason).toContain("the speaker names the number"); // Two decimals, the clip's window plus the pads it asked for. expect(ask.from).toBe(1); expect(ask.to).toBe(8); // In the CORPUS, not in this project's out/. const file = path.join(clipsDir("vid3"), "1.00-8.00.mp4"); expect(existsSync(file)).toBe(true); expect(statSync(file).size).toBeGreaterThan(0); // AND NO yt-dlp. This is the whole point: the fetch went through the // editor's managed path, not through a process this server spawned. expect(ytdlpCalls()).toBe(ytdlpBefore); // The page reads the corpus window as cached, so the walk stops skipping it. await page.goto(`/browse/${PROJECT}`); await expect(page.locator('[data-entry="e01"][data-kind]')).toHaveAttribute( "data-fetched", "1", ); await expect(page.locator("[data-ready-count]")).toContainText("ready 1 of 1"); // And /api/report/raw serves it: resolveInRoots admits the corpus as a READ // root, which is what keeps a window that is right there from 400ing. const raw = await request.get( `${baseURL}/api/report/raw?project=${encodeURIComponent(PROJECT)}&clip=e01`, ); expect(raw.status()).toBe(200); expect(raw.headers()["x-window"]).toBe("1.00-8.00.mp4"); expect(raw.headers()["x-fetch-start"]).toBe("1"); }); test("the project page fetches the unfetched clips one at a time, and Stop halts it", async ({ page, request, baseURL, }) => { test.setTimeout(120_000); const ytdlpBefore = ytdlpCalls(); await page.goto(`/browse/${MANY}`); await expect(page.locator("[data-ready-count]")).toContainText("ready 0 of 2"); const button = page.getByRole("button", { name: /fetch 2 unfetched clips/i }); await expect(button).toBeVisible(); // STOP ABANDONS, IT DOES NOT KILL: the in-flight fetch is already paid for, // so what Stop means is "start no more". One clip is asked for, the second // never is — which is also what proves the loop is serial rather than a // burst, since a burst would have both requests already on the wire. await button.click(); await page.getByRole("button", { name: /^stop$/i }).click(); await expect(page.locator("[data-fetch-unfetched-msg]")).toContainText( /stopped after 1 of 2/, { timeout: 90_000 }, ); const asks = stubRequests().filter((a) => a.manifest === "editor-fetch-many-fixture"); expect(asks.map((a) => a.clipId)).toEqual(["f01"]); // f01 is 0–3 s with the route's default ±20 s of pad. expect(existsSync(path.join(clipsDir("vid4"), "0.00-23.00.mp4"))).toBe(true); // AND f02 IS NOW FETCHED TOO, without anybody asking for it. A window is // deliberately generous and the predicate is containment, so one fetch is // the next clip's cache — which is the entire argument for putting these // bytes in the corpus instead of in one project's out/. await page.reload(); await expect(page.locator("[data-ready-count]")).toContainText("ready 2 of 2"); await expect(page.locator('[data-entry="f02"][data-kind]')).toHaveAttribute( "data-fetched", "1", ); await expect( page.getByRole("button", { name: /unfetched clip/i }), ).toHaveCount(0); // Still nothing shelled out to yt-dlp. expect(ytdlpCalls()).toBe(ytdlpBefore); // Leave the runner quiet for the next spec. await waitForJob(request, baseURL!); }); test("a clip the previous fetch already covers does not abort the sweep", async ({ page, request, baseURL, }) => { test.setTimeout(120_000); const ytdlpBefore = ytdlpCalls(); // g01 is 5–25 s, so ±20 s of pad makes it 0.00–45.00. g02's padded ask is // [0, 35] — INSIDE that — so the route answers 409 "already cached to −N s" // rather than queueing a job. // // That is the common case, not an edge: a window is deliberately generous // and the predicate is containment, so a sweep down a timeline routinely // finds its next clip already paid for. Treating it as a refusal stopped the // run on its own success and left the counter short. await page.goto(`/browse/${REUSE}`); await expect(page.locator("[data-ready-count]")).toContainText("ready 0 of 2"); await page.getByRole("button", { name: /fetch 2 unfetched clips/i }).click(); await expect(page.locator("[data-fetch-unfetched-msg]")).toContainText( /fetched 2 clips/, { timeout: 90_000 }, ); // ONE ask on the wire for two clips: the second was never fetched, and never // needed to be. const asks = stubRequests().filter( (a) => a.manifest === "editor-fetch-reuse-fixture", ); expect(asks.map((a) => a.clipId)).toEqual(["g01"]); expect(existsSync(path.join(clipsDir("vid5"), "0.00-45.00.mp4"))).toBe(true); await page.reload(); await expect(page.locator("[data-ready-count]")).toContainText("ready 2 of 2"); await expect( page.getByRole("button", { name: /unfetched clip/i }), ).toHaveCount(0); expect(ytdlpCalls()).toBe(ytdlpBefore); await waitForJob(request, baseURL!); });