import { rm, writeFile } from "node:fs/promises"; import { test, expect } from "@playwright/test"; import { channelStage, generateReport, pathExists, readJson, resetData, resolvePath, } from "./helpers"; const AVAILABILITY_CHANNEL = "availability-test"; // The shard inputs are 1-based in the UI ("shard i of N"); the persisted // shardIndex stays 0-based. So filling index "1" → shardIndex 0 on disk. type ShardFile = { totalShards: number; shardIndex: number; items: string[]; createdAt: string; }; test.describe("Availability check sharding", () => { test("shard 1/2 only checks half the videos and persists the slice", async ({ page, }) => { await resetData("availability-baseline"); await generateReport(page, AVAILABILITY_CHANNEL); await page.goto(channelStage(AVAILABILITY_CHANNEL, "diagnostics")); await page.getByLabel("shard availability total").first().fill("2"); await page.getByLabel("shard availability index").first().fill("1"); await page.getByRole("button", { name: "Resume checks" }).click(); const log = page.getByLabel("Resume availability checks output"); await expect(log).toContainText("Shard availability: computed slice 1/2", { timeout: 20_000, }); await expect(log).toContainText("Availability check done", { timeout: 20_000, }); // 6 fixture videos sorted lex; shard 1 of 2 (0-based index 0) takes 3. const cfg = await readJson( `test-transcripts/channels/${AVAILABILITY_CHANNEL}/shard-availability.json`, ); expect(cfg.totalShards).toBe(2); expect(cfg.shardIndex).toBe(0); expect(cfg.items).toHaveLength(3); // Only the items in the slice got availability.json (other 3 did not). let written = 0; for (const id of [ "viddeleted1", "vidmembers1", "vidneedsauth1", "vidprivate1", "vidpublic1", "vidpublic2", ]) { if ( await pathExists( `test-transcripts/channels/${AVAILABILITY_CHANNEL}/data/${id}/availability.json`, ) ) { written++; } } expect(written).toBe(3); }); test("re-running with the same shard inputs reuses the saved slice", async ({ page, }) => { await resetData("availability-baseline"); await generateReport(page, AVAILABILITY_CHANNEL); await page.goto(channelStage(AVAILABILITY_CHANNEL, "diagnostics")); await page.getByLabel("shard availability total").first().fill("2"); await page.getByLabel("shard availability index").first().fill("1"); await page.getByRole("button", { name: "Resume checks" }).click(); await expect( page.getByLabel("Resume availability checks output"), ).toContainText("computed slice 1/2", { timeout: 20_000 }); // Re-click — should report "using saved slice" rather than recompute. await page.getByRole("button", { name: "Resume checks" }).click(); await expect( page.getByLabel("Resume availability checks output"), ).toContainText("using saved slice 1/2", { timeout: 20_000 }); }); test("Clear button removes the saved shard config", async ({ page }) => { await resetData("availability-baseline"); await generateReport(page, AVAILABILITY_CHANNEL); await page.goto(channelStage(AVAILABILITY_CHANNEL, "diagnostics")); await page.getByLabel("shard availability total").first().fill("2"); await page.getByLabel("shard availability index").first().fill("1"); await page.getByRole("button", { name: "Resume checks" }).click(); await expect( page.getByLabel("Resume availability checks output"), ).toContainText("Availability check done", { timeout: 20_000 }); expect( await pathExists( `test-transcripts/channels/${AVAILABILITY_CHANNEL}/shard-availability.json`, ), ).toBe(true); // revalidatePath happens server-side; navigate to pick up the new // existing-shard prop in the React tree so the Clear button becomes enabled. await page.reload(); await page.getByLabel("shard availability clear").first().click(); // After router.refresh inside the control, the saved-shard pill is gone. await expect( page.getByLabel("shard availability empty").first(), ).toBeVisible({ timeout: 10_000 }); expect( await pathExists( `test-transcripts/channels/${AVAILABILITY_CHANNEL}/shard-availability.json`, ), ).toBe(false); }); }); test.describe("Transcribe missing sharding", () => { test("shard 2/2 only transcribes its half and persists the slice", async ({ page, }) => { await resetData("one-transcribe-channel-with-audio"); await generateReport(page, "test-transcribe"); await page.goto(channelStage("test-transcribe", "transcribe")); await page .getByLabel("shard transcribe-missing total") .first() .fill("2"); await page .getByLabel("shard transcribe-missing index") .first() .fill("2"); await page.getByRole("button", { name: "Transcribe missing" }).click(); const log = page.getByLabel("Transcribe missing output"); await expect(log).toContainText("Shard transcribe-missing: computed slice 2/2", { timeout: 30_000, }); // The job's progress total is scoped to the shard subset, not the whole // channel (3 untranscribed videos sorted, shard 2 of 2 → 1 item). await expect(log).toContainText( "Shard transcribe-missing: progress scoped to 1 of 1 shard item(s)", { timeout: 30_000 }, ); await expect(log).toContainText("succeeded", { timeout: 30_000 }); const cfg = await readJson( "test-transcripts/channels/test-transcribe/shard-transcribe-missing.json", ); expect(cfg.totalShards).toBe(2); expect(cfg.shardIndex).toBe(1); // 3 untranscribed videos sorted; shard 2 of 2 picks 1 of them. expect(cfg.items).toHaveLength(1); // The saved-shard pill reflects the just-saved config WITHOUT a manual // page reload (StreamActionLog refreshes server data on completion). await expect( page.getByLabel("shard transcribe-missing saved").first(), ).toBeVisible({ timeout: 10_000 }); // Only the sharded id should have a transcript. let transcribed = 0; for (const id of ["vidA", "vidB", "vidC"]) { if ( await pathExists( `test-transcripts/channels/test-transcribe/data/${id}/transcript.json`, ) ) { transcribed++; } } expect(transcribed).toBe(1); }); test("shards only the remaining (untranscribed) videos, not the whole channel", async ({ page, }) => { await resetData("one-transcribe-channel-with-audio"); // Pre-transcribe vidB so only vidA & vidC remain. A 1-shard slice must then // own 2 items (the remaining set), never 3 (the whole channel). await writeFile( resolvePath( "test-transcripts/channels/test-transcribe/data/vidB/transcript.json", ), JSON.stringify({ cues: [] }), ); await generateReport(page, "test-transcribe"); await page.goto(channelStage("test-transcribe", "transcribe")); await page.getByLabel("shard transcribe-missing total").first().fill("1"); await page.getByLabel("shard transcribe-missing index").first().fill("1"); await page.getByLabel("shard transcribe-missing save").first().click(); await expect( page.getByLabel("shard transcribe-missing saved").first(), ).toBeVisible({ timeout: 10_000 }); const cfg = await readJson( "test-transcripts/channels/test-transcribe/shard-transcribe-missing.json", ); // The remaining set is {vidA, vidC} — vidB is already transcribed. expect(cfg.items).toHaveLength(2); expect(cfg.items).not.toContain("vidB"); }); }); test.describe("Save shard (no run)", () => { test("Save persists the slice and shows the pill without running", async ({ page, }) => { await resetData("one-transcribe-channel-with-audio"); await generateReport(page, "test-transcribe"); await page.goto(channelStage("test-transcribe", "transcribe")); await page.getByLabel("shard transcribe-missing total").first().fill("2"); await page.getByLabel("shard transcribe-missing index").first().fill("2"); // Save-only: persist the slice WITHOUT transcribing. await page.getByLabel("shard transcribe-missing save").first().click(); // The saved-shard pill appears via router.refresh, no manual reload. await expect( page.getByLabel("shard transcribe-missing saved").first(), ).toBeVisible({ timeout: 10_000 }); const cfg = await readJson( "test-transcripts/channels/test-transcribe/shard-transcribe-missing.json", ); expect(cfg.totalShards).toBe(2); expect(cfg.shardIndex).toBe(1); // 3 untranscribed videos sorted; shard 2 of 2 owns exactly 1. expect(cfg.items).toHaveLength(1); // Save-only must NOT have produced any transcript. let transcribed = 0; for (const id of ["vidA", "vidB", "vidC"]) { if ( await pathExists( `test-transcripts/channels/test-transcribe/data/${id}/transcript.json`, ) ) { transcribed++; } } expect(transcribed).toBe(0); }); test("a run after Save reuses the saved slice", async ({ page }) => { await resetData("one-transcribe-channel-with-audio"); await generateReport(page, "test-transcribe"); await page.goto(channelStage("test-transcribe", "transcribe")); await page.getByLabel("shard transcribe-missing total").first().fill("2"); await page.getByLabel("shard transcribe-missing index").first().fill("2"); await page.getByLabel("shard transcribe-missing save").first().click(); await expect( page.getByLabel("shard transcribe-missing saved").first(), ).toBeVisible({ timeout: 10_000 }); // Now actually run — resolveShardItems should report the saved slice. await page.getByRole("button", { name: "Transcribe missing" }).click(); const log = page.getByLabel("Transcribe missing output"); await expect(log).toContainText( "Shard transcribe-missing: using saved slice 2/2", { timeout: 30_000 }, ); await expect(log).toContainText("succeeded", { timeout: 30_000 }); // Only the sharded id should have a transcript. let transcribed = 0; for (const id of ["vidA", "vidB", "vidC"]) { if ( await pathExists( `test-transcripts/channels/test-transcribe/data/${id}/transcript.json`, ) ) { transcribed++; } } expect(transcribed).toBe(1); }); }); test.describe("Download missing sharding", () => { const channelRoot = "test-transcripts/channels/test-transcribe"; // Make all three videos missing (clear their audio) and set a 3-URL playlist. // The shard slices the *missing* set: vidA/vidB/vidC sorted, shard 1 of 2 // (0-based index 0) owns indices 0 and 2 → vidA, vidC. async function setUpMissing() { for (const id of ["vidA", "vidB", "vidC"]) { await rm(resolvePath(`${channelRoot}/data/${id}/audio.m4a`), { force: true, }); await rm(resolvePath(`${channelRoot}/data/${id}/transcript.json`), { force: true, }); } await writeFile( resolvePath(`${channelRoot}/playlist`), [ "https://www.youtube.com/watch?v=vidA", "https://www.youtube.com/watch?v=vidB", "https://www.youtube.com/watch?v=vidC", ].join("\n") + "\n", ); } test("shard 1/2 persists the missing-set slice before fetching and shows it without reload", async ({ page, }) => { await resetData("one-transcribe-channel-with-audio"); await setUpMissing(); await generateReport(page, "test-transcribe"); await page.goto(channelStage("test-transcribe", "download")); await page.getByLabel("shard download-missing total").first().fill("2"); await page.getByLabel("shard download-missing index").first().fill("1"); await page.getByRole("button", { name: "Download videos" }).click(); const log = page.getByLabel("Download videos output"); // The slice is computed + persisted before any download starts. await expect(log).toContainText( "Shard download-missing: computed slice 1/2", { timeout: 30_000 }, ); // The job's progress total is scoped to the shard subset. await expect(log).toContainText( "Shard download-missing: progress scoped to 2 shard item(s)", { timeout: 30_000 }, ); // The shard config is on disk, snapshotting this shard's slice of the // missing set (vidA, vidC) so a resume reuses it rather than re-slicing. const cfg = await readJson( `${channelRoot}/shard-download-missing.json`, ); expect(cfg.totalShards).toBe(2); expect(cfg.shardIndex).toBe(0); expect(cfg.items).toHaveLength(2); expect(cfg.items.some((u) => u.includes("vidA"))).toBe(true); expect(cfg.items.some((u) => u.includes("vidC"))).toBe(true); expect(cfg.items.some((u) => u.includes("vidB"))).toBe(false); // The saved-shard pill appears without a manual reload. await expect( page.getByLabel("shard download-missing saved").first(), ).toBeVisible({ timeout: 10_000 }); }); test("re-running with the same shard inputs reuses the saved slice", async ({ page, }) => { await resetData("one-transcribe-channel-with-audio"); await setUpMissing(); await generateReport(page, "test-transcribe"); await page.goto(channelStage("test-transcribe", "download")); await page.getByLabel("shard download-missing total").first().fill("2"); await page.getByLabel("shard download-missing index").first().fill("1"); await page.getByRole("button", { name: "Download videos" }).click(); await expect( page.getByLabel("Download videos output"), ).toContainText("computed slice 1/2", { timeout: 30_000 }); // Wait for the first run to fully finish — the saved-shard pill appears via // the post-run refresh — then start a fresh run. (Reload to get a clean // run rather than racing a second click against the still-streaming first // button; the no-reload guarantee is covered by the test above.) After // reload the shard inputs are pre-filled from the saved config. await expect( page.getByLabel("shard download-missing saved").first(), ).toBeVisible({ timeout: 30_000 }); await page.reload(); await page.getByRole("button", { name: "Download videos" }).click(); // resolveShardItems reports the saved slice (resume), proving a cancelled // run picks up exactly the videos this shard still owes rather than // re-slicing the now-smaller missing set. await expect( page.getByLabel("Download videos output"), ).toContainText("using saved slice 1/2", { timeout: 30_000 }); }); });