import { test } from "node:test"; import assert from "node:assert/strict"; import path from "node:path"; import { mkdtemp, mkdir, rm, writeFile, utimes } from "node:fs/promises"; import { tmpdir } from "node:os"; import { isCuesJsonFresh, CUES_FILE_VERSION, } from "./normalizeTranscript"; import { normalizeChannelTranscripts } from "./normalizeAll"; import { CUES_JSON_FILENAME, META_FILENAME, VTT_FILENAME, WHISPER_FILENAME, } from "../lib/videoStatus"; import type { Paths } from "../lib/paths"; // These cover the two things the digest lane's `deferred` bucket turned out to // be, which the code had conflated into one: // // missing — no transcript.cues.json at all. NOTHING WRITES ONE AUTOMATICALLY // for a channel whose subtitles are downloaded rather than // transcribed, so this is permanent until the normalize pass runs. // 1,942 videos on the live corpus. // stale — a cues.json superseded by a rewrite of its inputs. 47 videos. // // The distinction is about COPY and about whether an operator has anything to // do, not about dispatch — both still classify `deferred` — so it lives on // isCuesJsonFresh's reason rather than in a new OperationState. const CUES_BODY = JSON.stringify({ version: CUES_FILE_VERSION, id: "vid1", title: "A video", cues: [{ start: 0, end: 5, text: "hello" }], }); // mtimes are set EXPLICITLY, never left to write order: freshness here is pure // mtime math, and two files written in the same millisecond compare equal on // filesystems with coarse timestamps. async function stampAscending( dir: string, order: Array<[file: string, offsetSec: number]>, ): Promise { const base = Date.now() / 1000 - 1000; for (const [file, offset] of order) { await utimes(path.join(dir, file), base + offset, base + offset); } } async function videoFixture(files: { meta?: boolean; vtt?: boolean; whisper?: boolean; cues?: "fresh" | "stale" | false; }): Promise<{ dir: string; cleanup: () => Promise }> { const dir = await mkdtemp(path.join(tmpdir(), "normalize-cues-")); const stamps: Array<[string, number]> = []; if (files.meta !== false) { await writeFile( path.join(dir, META_FILENAME), JSON.stringify({ id: "vid1", title: "A video", duration: 60 }), ); stamps.push([META_FILENAME, 10]); } if (files.vtt) { await writeFile( path.join(dir, VTT_FILENAME), "WEBVTT\n\n00:00:00.000 --> 00:00:05.000\nhello\n", ); stamps.push([VTT_FILENAME, 10]); } if (files.whisper) { await writeFile( path.join(dir, WHISPER_FILENAME), JSON.stringify({ transcription: [ { offsets: { from: 0, to: 5000 }, text: "hello" }, ], }), ); stamps.push([WHISPER_FILENAME, 10]); } if (files.cues) { await writeFile(path.join(dir, CUES_JSON_FILENAME), CUES_BODY); // stale = OLDER than its inputs, which is the whole definition. stamps.push([CUES_JSON_FILENAME, files.cues === "stale" ? 5 : 20]); } await stampAscending(dir, stamps); return { dir, cleanup: () => rm(dir, { recursive: true, force: true }) }; } test("isCuesJsonFresh reports `missing` when there is no cues.json at all", async () => { const { dir, cleanup } = await videoFixture({ vtt: true, cues: false }); try { const r = await isCuesJsonFresh(dir); assert.equal(r.fresh, false); // The 97.6% case. Naming it is the point: it is the one an operator can fix // and the one nothing fixes on its own. assert.equal(r.reason, "missing"); } finally { await cleanup(); } }); test("isCuesJsonFresh reports `stale` when cues.json predates its raw transcript", async () => { const { dir, cleanup } = await videoFixture({ vtt: true, cues: "stale" }); try { const r = await isCuesJsonFresh(dir); assert.equal(r.fresh, false); assert.equal(r.reason, "stale"); } finally { await cleanup(); } }); test("isCuesJsonFresh reports `no-raw` when nothing was ever transcribed", async () => { // Neither reason an operator can act on: normalizeTranscript would return // `skipped`, and the digest lane counts these as `blocked` on transcription // rather than deferred. const { dir, cleanup } = await videoFixture({ cues: false }); try { const r = await isCuesJsonFresh(dir); assert.equal(r.fresh, false); assert.equal(r.reason, "no-raw"); } finally { await cleanup(); } }); test("isCuesJsonFresh reports `no-meta` before it looks at the transcript", async () => { const { dir, cleanup } = await videoFixture({ meta: false, vtt: true, cues: "fresh", }); try { const r = await isCuesJsonFresh(dir); assert.equal(r.fresh, false); assert.equal(r.reason, "no-meta"); } finally { await cleanup(); } }); test("isCuesJsonFresh still says fresh — and says so with a reason", async () => { const { dir, cleanup } = await videoFixture({ vtt: true, cues: "fresh" }); try { const r = await isCuesJsonFresh(dir); assert.equal(r.fresh, true); assert.equal(r.reason, "fresh"); } finally { await cleanup(); } }); // --------------------------------------------------------------------------- // The channel-scoped runner // --------------------------------------------------------------------------- async function corpusFixture(): Promise<{ paths: Paths; cleanup: () => Promise; }> { const root = await mkdtemp(path.join(tmpdir(), "normalize-channel-")); const channelsDir = path.join(root, "channels"); const video = async ( slug: string, id: string, files: { vtt?: boolean; cues?: "fresh" | false }, ): Promise => { const dir = path.join(channelsDir, slug, "data", id); await mkdir(dir, { recursive: true }); const stamps: Array<[string, number]> = []; await writeFile( path.join(dir, META_FILENAME), JSON.stringify({ id, title: `Video ${id}`, duration: 60 }), ); stamps.push([META_FILENAME, 10]); if (files.vtt) { await writeFile( path.join(dir, VTT_FILENAME), "WEBVTT\n\n00:00:00.000 --> 00:00:05.000\nhello\n", ); stamps.push([VTT_FILENAME, 10]); } if (files.cues) { await writeFile(path.join(dir, CUES_JSON_FILENAME), CUES_BODY); stamps.push([CUES_JSON_FILENAME, 20]); } await stampAscending(dir, stamps); }; for (const slug of ["target", "other"]) { await mkdir(path.join(channelsDir, slug), { recursive: true }); await writeFile( path.join(channelsDir, slug, "config.json"), JSON.stringify({ // REQUIRED — parseChannelConfig returns null without it, and a channel // with no parseable config is silently dropped from the walk. A fixture // missing this makes every assertion below pass vacuously, which is // exactly how the first draft of this file went green. handling: "youtube", name: slug, url: `https://example.com/${slug}`, }), ); } // One of each outcome, which is what makes the three counters meaningful. await video("target", "needs-normalizing", { vtt: true, cues: false }); await video("target", "already-current", { vtt: true, cues: "fresh" }); await video("target", "never-transcribed", {}); // The channel that must NOT be touched — the entire reason for the scope. await video("other", "also-needs-normalizing", { vtt: true, cues: false }); return { paths: { channelsDir } as Paths, cleanup: () => rm(root, { recursive: true, force: true }), }; } test("normalizeChannelTranscripts reports wrote/fresh/skipped separately", async () => { const { paths, cleanup } = await corpusFixture(); try { const result = await normalizeChannelTranscripts({ paths, channelSlug: "target", onLog: () => {}, }); assert.equal(result.wrote, 1, "the video with a VTT and no cues.json"); assert.equal(result.fresh, 1, "the one already current — no rework"); // `skipped` is the honest floor, not a failure: normalizing cannot help a // video that was never transcribed. Those are `blocked` on the digest card, // a different number from `deferred`. assert.equal(result.skipped, 1, "the never-transcribed one"); assert.equal(result.failed, 0); } finally { await cleanup(); } }); test("normalizeChannelTranscripts leaves every other channel alone", async () => { const { paths, cleanup } = await corpusFixture(); try { const result = await normalizeChannelTranscripts({ paths, channelSlug: "target", onLog: () => {}, }); // GUARD AGAINST A VACUOUS PASS. An unparseable fixture config drops the // channel from the walk entirely, and then "the other channel is untouched" // is true because NOTHING ran. Assert the run did its work first. assert.equal(result.wrote, 1, "the run must actually have done something"); // Scope is the whole point: the corpus-wide button walks ~79,000 dirs, and // the digest card's fix must cost one channel. const other = await isCuesJsonFresh( path.join(paths.channelsDir, "other", "data", "also-needs-normalizing"), ); assert.equal(other.reason, "missing"); } finally { await cleanup(); } }); test("normalizing turns a `missing` video into a fresh one", async () => { const { paths, cleanup } = await corpusFixture(); try { const dir = path.join( paths.channelsDir, "target", "data", "needs-normalizing", ); assert.equal((await isCuesJsonFresh(dir)).reason, "missing"); await normalizeChannelTranscripts({ paths, channelSlug: "target", onLog: () => {}, }); // The end-to-end claim this whole change rests on: a deferred video becomes // reachable digest work. assert.equal((await isCuesJsonFresh(dir)).fresh, true); } finally { await cleanup(); } });