// Integration: a VOD mirror's recorded dates (release 20 D1) through the REAL // buildIndex over a temp corpus — derived from the title by the channel's // `recordedDate` rule, absent for a channel without one, re-derived when the // rule changes and dropped when it goes. Synthetic titles. // // Run with: node_modules/.bin/tsx --test common/controller/buildIndexRecordedDate.test.ts import { after, test } from "node:test"; import assert from "node:assert/strict"; import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; const ROOT = mkdtempSync(path.join(tmpdir(), "build-index-recorded-")); const PINNED: Record = { TRANSCRIPTS_DIR: path.join(ROOT, "transcripts"), SAVED_VIDEOS_DIR: path.join(ROOT, "saved-videos"), SITES_DIR: path.join(ROOT, "transcripts", "sites"), SETTINGS_FILE: path.join(ROOT, "settings.json"), EXPORT_PUBLIC_DIR: path.join(ROOT, "public"), EXPORT_INDEX_DIR: path.join(ROOT, ".export-index"), EXPORT_BUILDS_DIR: path.join(ROOT, ".export-builds"), EDITOR_CHANGELOG_FILE: path.join(ROOT, "editor-CHANGELOG.md"), EXPORT_CHANGELOG_FILE: path.join(ROOT, "export-CHANGELOG.md"), CHARTS_CONFIG_FILE: path.join(ROOT, "chart-templates.json"), SEARCH_ALIASES_FILE: path.join(ROOT, "transcripts", "search-aliases.json"), CURATED_TAGS_FILE: path.join(ROOT, "transcripts", "tags.json"), ARCHILYZER_CONFIG_DIR: path.join(ROOT, "config"), ARCHILYZER_SOURCE_SCRATCH: path.join(ROOT, "source-scratch"), }; Object.assign(process.env, PINNED); delete process.env.ARCHILYZER_INDEX_ALLOW_HELD; after(() => rmSync(ROOT, { recursive: true, force: true })); const { getPaths } = await import("../lib/paths"); const { buildIndex } = await import("./buildIndex"); const { open } = await import("lmdb"); const paths = getPaths(); const MIRROR = "example-vods"; const PLAIN = "example-plain"; const SITE = "testsite"; const ISO = String.raw`(?\d{4})-(?\d{2})-(?\d{2})`; const US = String.raw`(?\d{1,2})/(?\d{1,2})/(?\d{2})`; const writeJson = (file: string, value: unknown) => { mkdirSync(path.dirname(file), { recursive: true }); writeFileSync(file, JSON.stringify(value, null, 2)); }; const ROLLING = readFileSync( path.join(import.meta.dirname, "..", "lib", "__fixtures__", "vtt-rolling.vtt"), "utf8", ); function config(slug: string, recordedDate?: { titlePattern: string }) { writeJson(path.join(paths.channelsDir, slug, "config.json"), { handling: "youtube", name: slug, ...(recordedDate ? { recordedDate } : {}), }); } function video(slug: string, id: string, title: string, uploadDate: string) { const dir = path.join(paths.channelsDir, slug, "data", id); writeJson(path.join(dir, "metadata.info.json"), { id, title, upload_date: uploadDate, duration: 30, webpage_url: `https://www.youtube.com/watch?v=${id}`, extractor_key: "Youtube", }); writeFileSync(path.join(dir, "transcript.en.vtt"), ROLLING); } function seed(): void { writeFileSync(paths.settingsFile, "{}"); config(MIRROR, { titlePattern: ISO }); config(PLAIN); writeJson(path.join(paths.sitesDir, SITE, "site.json"), { siteId: SITE, siteTitle: "Test Site", siteDescription: "fixture", headerTitle: "Test Site", homeTagline: "", socialLinks: [], groups: [{ id: "default", name: "All channels", selectedByDefault: true }], defaultGroupId: "default", channels: [ { slug: MIRROR, groupId: "default" }, { slug: PLAIN, groupId: "default" }, ], }); video(MIRROR, "MirrorVod001", "Stream VOD 2024-03-05 [3/5/24]", "20240310"); video(MIRROR, "MirrorVod002", "Stream VOD with no date", "20240311"); video(MIRROR, "MirrorVod003", "Stream VOD — next one 2024-04-01", "20240312"); video(PLAIN, "PlainVideo01", "An upload about 2024-01-02", "20240115"); } type Summary = { id: string; uploadDate: string; recordedDate?: string }; function recorded(): Record { const root = open({ path: paths.lmdbPath, maxDbs: 18, compression: true }); try { const sums = root.openDB({ name: "sums", encoding: "msgpack" }); const out: Record = {}; for (const { value } of sums.getRange()) { const s = value as Summary; out[s.id] = s.recordedDate; } return out; } finally { root.close(); } } // Every published transcript page of a channel, as text. function pagesOf(slug: string): string { const dir = path.join(paths.exportSharedTranscriptsDir, slug); return readdirSync(dir) .filter((f) => f.endsWith(".json")) .map((f) => readFileSync(path.join(dir, f), "utf8")) .join("\n"); } async function runIndex(): Promise { const log: string[] = []; await buildIndex({ paths, onLog: (s) => log.push(s) }); return log; } test("a mirror's records carry the title's recorded date; another channel's never do", async () => { seed(); await runIndex(); assert.deepEqual(recorded(), { MirrorVod001: "20240305", MirrorVod002: undefined, // no date in the title MirrorVod003: undefined, // the title's date is after the upload PlainVideo01: undefined, // no rule }); assert.match(pagesOf(MIRROR), /"recordedDate":\s*"20240305"/); assert.doesNotMatch(pagesOf(PLAIN), /recordedDate/); }); test("a changed rule re-derives the channel's records once; a removed rule drops them", async () => { config(MIRROR, { titlePattern: US }); const log = await runIndex(); assert.ok( log.includes(`Recorded dates: ${MIRROR}: its rule changed; 3 record(s) re-derived.`), log.join("\n"), ); assert.equal(log.some((l) => l.includes(`Recorded dates: ${PLAIN}`)), false); assert.equal(recorded().MirrorVod001, "20240305"); // Unchanged: no second pass. const again = await runIndex(); assert.equal(again.some((l) => l.startsWith("Recorded dates:")), false, again.join("\n")); config(MIRROR); const dropped = await runIndex(); assert.ok(dropped.includes(`Recorded dates: ${MIRROR}: its rule changed; 3 record(s) re-derived.`)); assert.equal(recorded().MirrorVod001, undefined); assert.doesNotMatch(pagesOf(MIRROR), /recordedDate/); });