import path, { dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { readFile, writeFile } from "node:fs/promises"; import { test, expect } from "@playwright/test"; import { channelStage, generateReport, pathExists, readJson, resetData, resolvePath, writeSettings, } from "./helpers"; import { baseUrl } from "./baseUrl"; // The transcribed-audio sweep deletes source audio irreversibly, so it first // checks each candidate is still available upstream and pins (do-not-clean) // anything permanently gone. These tests pin down all three tiers of that gate // plus the global opt-out. // // The availability-baseline fixture is purpose-built for it: each data dir's // metadata.info.json carries a webpage_url whose id doubles as the sentinel that // fake-ytdlp matches on to produce a deterministic availability bucket. const here = dirname(fileURLToPath(import.meta.url)); const CHANNEL = "availability-test"; const FIXTURE = "availability-baseline"; const channelRoot = `test-transcripts/channels/${CHANNEL}`; function dataRel(id: string, file: string): string { return `${channelRoot}/data/${id}/${file}`; } async function defaultSettings(): Promise> { const raw = await readFile( path.join(here, "fixtures", "test-settings.default.json"), "utf8", ); return JSON.parse(raw) as Record; } // Make a video cleanable: a whisper transcript plus audio still on disk. async function seedCleanable(id: string): Promise { await writeFile( resolvePath(dataRel(id, "transcript.json")), '{"transcription":[]}\n', ); await writeFile(resolvePath(dataRel(id, "audio.m4a")), "fake-audio-bytes\n"); } // Write the sidecar controlling which ids fake-ytdlp's flat-playlist branch // emits. It lives in the channel cwd, where availability spawns run. async function setFreshPlaylist(ids: string[]): Promise { await writeFile( resolvePath(`${channelRoot}/.fake-ytdlp-flat-playlist.json`), JSON.stringify({ ids }), ); } async function readInvocations(): Promise { try { return await readFile( resolvePath(`${channelRoot}/fake-ytdlp.invocations`), "utf8", ); } catch { return ""; } } // The five videos each test makes cleanable. vidpublic2 stays untranscribed. const CLEANABLE = [ "vidpublic1", "viddeleted1", "vidprivate1", "vidneedsauth1", "vidmembers1", ]; async function seedAll(): Promise { for (const id of CLEANABLE) await seedCleanable(id); // vidmembers1 is already known members_only. A members-only video stays // listed in its channel's flat playlist, so tier B never flags it — tier A's // cached verdict is the only thing that can catch it, with zero spawns. await writeFile( resolvePath(dataRel("vidmembers1", "availability.json")), JSON.stringify({ checkedAt: new Date().toISOString(), availability: "members_only", }), ); await fetch(`${baseUrl}/api/test/invalidate-cache`).catch(() => {}); } async function runCleanAudio(page: import("@playwright/test").Page) { await generateReport(page, CHANNEL); await page.goto(channelStage(CHANNEL, "cleanup")); const button = page.getByRole("button", { name: "Clean audio", exact: true }); await expect(button).toBeEnabled(); await button.click(); return page.getByLabel("Clean audio output"); } test("cleanup pins videos gone from the source and leaves unresolvable ones alone", async ({ page, }) => { test.setTimeout(120_000); await resetData(FIXTURE); await seedAll(); await writeSettings({ ...(await defaultSettings()), verifyAvailabilityBeforeClean: true, }); // The fresh listing keeps the public videos and the members-only one; the // deleted / private / age-gated videos have dropped out of it. await setFreshPlaylist(["vidpublic1", "vidpublic2", "vidmembers1"]); const log = await runCleanAudio(page); await expect(log).toContainText( "Cleaned 1 audio file(s) from 1 of 6 video dir(s). Skipped 4 (0 protected, 3 gone-from-source pinned, 1 unverified).", { timeout: 60_000 }, ); // Still listed and still public — cleaned. expect(await pathExists(dataRel("vidpublic1", "audio.m4a"))).toBe(false); expect(await pathExists(dataRel("vidpublic1", "transcript.json"))).toBe(true); // Gone from the source — audio kept and pinned, with a note saying why. for (const [id, availability] of [ ["viddeleted1", "deleted"], ["vidprivate1", "private"], ["vidmembers1", "members_only"], ] as const) { expect(await pathExists(dataRel(id, "audio.m4a"))).toBe(true); const marker = await readJson<{ note?: string }>( dataRel(id, "do-not-clean.json"), ); expect(marker.note).toContain(availability); expect(marker.note).toContain("pre-clean check"); } // The fail-safe case: an age-gated video probes as needs_auth, which resolves // nothing. Audio is kept but NO marker is written, so the next sweep retries. expect(await pathExists(dataRel("vidneedsauth1", "audio.m4a"))).toBe(true); expect(await pathExists(dataRel("vidneedsauth1", "do-not-clean.json"))).toBe( false, ); // Tier A resolved the members-only video from its cached availability, so it // never cost a probe. The three suspects did. const invocations = await readInvocations(); expect(invocations).not.toContain("dump-json:https://www.youtube.com/watch?v=vidmembers1"); expect(invocations).toContain("dump-json:https://www.youtube.com/watch?v=viddeleted1"); expect(invocations).toContain("dump-json:https://www.youtube.com/watch?v=vidprivate1"); expect(invocations).toContain("dump-json:https://www.youtube.com/watch?v=vidneedsauth1"); }); test("an unverified video is retried on the next sweep", async ({ page }) => { test.setTimeout(120_000); await resetData(FIXTURE); await seedAll(); await writeSettings({ ...(await defaultSettings()), verifyAvailabilityBeforeClean: true, }); await setFreshPlaylist(["vidpublic1", "vidpublic2", "vidmembers1"]); const log = await runCleanAudio(page); await expect(log).toContainText("1 unverified", { timeout: 60_000 }); // Second sweep: the unpinned, unverified video is probed again (a pinned one // would have been filtered out as do-not-clean before the gate even ran). const secondLog = await runCleanAudio(page); await expect(secondLog).toContainText("1 unverified", { timeout: 60_000 }); const probes = (await readInvocations()) .split("\n") .filter((line) => line.startsWith("dump-json:https://www.youtube.com/watch?v=vidneedsauth1"), ); expect(probes.length).toBe(2); expect(await pathExists(dataRel("vidneedsauth1", "audio.m4a"))).toBe(true); }); test("the availability check can be turned off", async ({ page }) => { test.setTimeout(120_000); await resetData(FIXTURE); await seedAll(); await writeSettings({ ...(await defaultSettings()), verifyAvailabilityBeforeClean: false, }); await setFreshPlaylist(["vidpublic1", "vidpublic2", "vidmembers1"]); const log = await runCleanAudio(page); await expect(log).toContainText( "Cleaned 5 audio file(s) from 5 of 6 video dir(s).", { timeout: 60_000 }, ); // Everything cleanable is cleaned, including videos known to be gone, and no // marker is written — the pre-check is the only thing that pins. for (const id of CLEANABLE) { expect(await pathExists(dataRel(id, "audio.m4a"))).toBe(false); expect(await pathExists(dataRel(id, "do-not-clean.json"))).toBe(false); } // No probes at all: neither the flat-playlist listing nor any per-video probe. expect(await readInvocations()).not.toContain("dump-json:"); });