// Integration: a record labelled before its platform was known is relabelled // by the index — through the REAL buildIndex, over a temp corpus. // // A BitChute record downloaded before the bitchute platform existed was // summarized "youtube" (and with no file to play), and a transcribed one // carries that summary frozen in its transcript.cues.json. Its files never // change again, so the mtime diff alone would never look at it: the build's // one-off platform-labels pass (PLATFORM_LABELS_VERSION) queues it, and the // per-video step re-derives a stale normalized summary from the metadata. // // Run with: node_modules/.bin/tsx --test common/controller/buildIndexPlatformLabels.test.ts import { after, test } from "node:test"; import assert from "node:assert/strict"; import { mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; const ROOT = mkdtempSync(path.join(tmpdir(), "build-index-labels-")); const PINNED: Record = { TRANSCRIPTS_DIR: path.join(ROOT, "transcripts"), SAVED_VIDEOS_DIR: path.join(ROOT, "saved-videos"), SITES_DIR: path.join(ROOT, "transcripts", "sites"), SETTINGS_FILE: path.join(ROOT, "settings.json"), EXPORT_PUBLIC_DIR: path.join(ROOT, "public"), EXPORT_INDEX_DIR: path.join(ROOT, ".export-index"), EXPORT_BUILDS_DIR: path.join(ROOT, ".export-builds"), EDITOR_CHANGELOG_FILE: path.join(ROOT, "editor-CHANGELOG.md"), EXPORT_CHANGELOG_FILE: path.join(ROOT, "export-CHANGELOG.md"), CHARTS_CONFIG_FILE: path.join(ROOT, "chart-templates.json"), SEARCH_ALIASES_FILE: path.join(ROOT, "transcripts", "search-aliases.json"), CURATED_TAGS_FILE: path.join(ROOT, "transcripts", "tags.json"), ARCHILYZER_CONFIG_DIR: path.join(ROOT, "config"), ARCHILYZER_SOURCE_SCRATCH: path.join(ROOT, "source-scratch"), }; Object.assign(process.env, PINNED); delete process.env.ARCHILYZER_INDEX_ALLOW_HELD; after(() => rmSync(ROOT, { recursive: true, force: true })); const { getPaths } = await import("../lib/paths"); const { buildIndex } = await import("./buildIndex"); const { open } = await import("lmdb"); const paths = getPaths(); const CHANNEL = "example-channel"; const SITE = "testsite"; const BC = "Zq3xVb7Kp2Lm"; const YT = "AbC123xyz_9"; const BC_PAGE = `https://www.bitchute.com/video/${BC}/`; const BC_FILE = `https://seed901.bitchute.com/AbCdEfGhIjKl/${BC}.mp4`; const writeJson = (file: string, value: unknown) => { mkdirSync(path.dirname(file), { recursive: true }); writeFileSync(file, JSON.stringify(value, null, 2)); }; const dirOf = (id: string) => path.join(paths.channelsDir, CHANNEL, "data", id); const VTT = "WEBVTT\nKind: captions\nLanguage: en\n\n" + "00:00:00.000 --> 00:00:05.000 align:start position:0%\n" + "First<00:00:01.000> caption<00:00:02.000> line.\n"; const CUES = [{ start: 0, end: 5, text: "Normalized line." }]; function seed(): void { rmSync(paths.transcriptsDir, { recursive: true, force: true }); rmSync(PINNED.EXPORT_INDEX_DIR, { recursive: true, force: true }); writeFileSync(paths.settingsFile, "{}"); writeJson(path.join(paths.channelsDir, CHANNEL, "config.json"), { handling: "transcribe", name: CHANNEL, }); writeJson(path.join(paths.sitesDir, SITE, "site.json"), { siteId: SITE, siteTitle: "Test Site", siteDescription: "fixture", headerTitle: "Test Site", homeTagline: "", socialLinks: [], groups: [{ id: "default", name: "All channels", selectedByDefault: true }], defaultGroupId: "default", channels: [{ slug: CHANNEL, groupId: "default" }], }); writeJson(path.join(dirOf(YT), "metadata.info.json"), { id: YT, title: "A YouTube video", upload_date: "20260601", duration: 120, webpage_url: `https://www.youtube.com/watch?v=${YT}`, extractor_key: "Youtube", }); writeFileSync(path.join(dirOf(YT), "transcript.en.vtt"), VTT); writeJson(path.join(dirOf(BC), "metadata.info.json"), { id: BC, title: "A BitChute video", upload_date: "20260602", duration: 61, webpage_url: BC_PAGE, extractor: "BitChute", extractor_key: "BitChute", formats: [{ format_id: "0", ext: "mp4", url: BC_FILE }], url: BC_FILE, }); writeFileSync(path.join(dirOf(BC), "transcript.en.vtt"), VTT); // The normalized transcript an earlier build of the app wrote: its summary // says "youtube" and carries no file. Newer than everything, so fresh. const cuesPath = path.join(dirOf(BC), "transcript.cues.json"); writeJson(cuesPath, { version: 2, source: "vtt", slug: `${CHANNEL}/${BC}`, id: BC, channelSlug: CHANNEL, title: "A BitChute video", uploadDate: "20260602", duration: 61, channel: CHANNEL, description: "", tags: [], isLivestream: false, ageRestricted: false, platform: "youtube", webpageUrl: BC_PAGE, cues: CUES, }); const later = new Date(Date.now() + 10_000); utimesSync(cuesPath, later, later); } type Summary = { id: string; platform: string; mediaUrl?: string }; function withIndex(fn: (db: (name: string) => ReturnType["openDB"]>) => T): T { const root = open({ path: paths.lmdbPath, maxDbs: 18, compression: true }); try { return fn((name) => root.openDB({ name, encoding: "msgpack" })); } finally { root.close(); } } const summaryOf = (id: string): Summary => withIndex((db) => { for (const { value } of db("sums").getRange()) { if ((value as Summary).id === id) return value as Summary; } throw new Error(`${id} is not indexed`); }); const cuesOf = (id: string) => withIndex((db) => { for (const { key, value } of db("sums").getRange()) { if ((value as Summary).id === id) return db("cues").get(key) as typeof CUES; } return undefined; }); async function runIndex(): Promise { const log: string[] = []; await buildIndex({ paths, onLog: (s) => log.push(s) }); return log; } test("a normalized summary frozen as youtube is re-derived as bitchute, with its file; its cues are kept", async () => { seed(); await runIndex(); const bc = summaryOf(BC); assert.equal(bc.platform, "bitchute"); assert.equal(bc.mediaUrl, BC_FILE); assert.deepEqual(cuesOf(BC), CUES); assert.equal(summaryOf(YT).platform, "youtube"); }); test("a summary already indexed under the old label is relabelled once, by the platform-labels pass", async () => { // The index as an older build left it: the record stored as "youtube", and // no platform-labels version recorded. withIndex((db) => { const sums = db("sums"); for (const { key, value } of sums.getRange()) { const v = value as Summary; if (v.id === BC) { const stale = { ...v, platform: "youtube" } as Summary; delete stale.mediaUrl; sums.putSync(key, stale); } } db("meta").removeSync("platformLabels"); }); assert.equal(summaryOf(BC).platform, "youtube"); const log = await runIndex(); assert.ok( log.some((l) => /Platform labels v\d+: 1 record\(s\)/.test(l)), log.join("\n"), ); const bc = summaryOf(BC); assert.equal(bc.platform, "bitchute"); assert.equal(bc.mediaUrl, BC_FILE); assert.equal(summaryOf(YT).platform, "youtube"); // Recorded: the next build does not look again. const again = await runIndex(); assert.equal(again.some((l) => /Platform labels/.test(l)), false, again.join("\n")); });