import { test } from "node:test"; import assert from "node:assert/strict"; import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import type { Paths } from "../lib/paths"; import type { ChannelConfig } from "../lib/channelConfig"; import { loadMetadataScan, metadataScanPath, settledByTitleFilterIds, upsertMetadataScan, type MetadataScanEntry, } from "./metadataScanStore"; // Run with: // pnpm --filter yt-dlp-transcript-common exec tsx --test common/controller/metadataScanStore.test.ts const SLUG = "chan"; const T1 = "2026-09-01T00:00:00.000Z"; const T2 = "2026-09-02T00:00:00.000Z"; async function withPaths(fn: (paths: Paths) => Promise): Promise { const dir = await mkdtemp(path.join(tmpdir(), "ttb-mdscan-")); const paths = { channelsDir: path.join(dir, "channels") } as Paths; await mkdir(path.join(paths.channelsDir, SLUG), { recursive: true }); try { await fn(paths); } finally { await rm(dir, { recursive: true, force: true }); } } function entry(over: Partial = {}): MetadataScanEntry { return { title: "Synthetic vid", description: "Synthetic video vid", uploadDate: "20240101", scannedAt: T1, ...over, }; } function config(include?: string, exclude?: string): ChannelConfig { return { handling: "youtube", name: "Chan", ...(include || exclude ? { downloadFilter: { ...(include ? { include } : {}), ...(exclude ? { exclude } : {}), }, } : {}), }; } test("a missing file loads as an empty store, not an error", async () => { await withPaths(async (paths) => { const scan = await loadMetadataScan(paths, SLUG); assert.deepEqual(scan.entries, {}); assert.deepEqual(scan.errors, {}); assert.equal(scan.lastRun, null); }); }); test("upsert is additive and persists across loads", async () => { await withPaths(async (paths) => { await upsertMetadataScan( paths, SLUG, { entries: { a: entry({ title: "A" }) } }, T1, ); await upsertMetadataScan( paths, SLUG, { entries: { b: entry({ title: "B" }) } }, T2, ); const scan = await loadMetadataScan(paths, SLUG); assert.deepEqual(Object.keys(scan.entries).sort(), ["a", "b"]); assert.equal(scan.entries.a.title, "A"); assert.equal(scan.updatedAt, T2); }); }); test("an entry clears that id's error; a later error never un-scans it", async () => { await withPaths(async (paths) => { await upsertMetadataScan( paths, SLUG, { errors: { a: { class: "needs_auth", message: "age", at: T1 } } }, T1, ); assert.ok((await loadMetadataScan(paths, SLUG)).errors.a); await upsertMetadataScan(paths, SLUG, { entries: { a: entry() } }, T2); const after = await loadMetadataScan(paths, SLUG); assert.equal(after.errors.a, undefined); assert.ok(after.entries.a); // A transient failure on a later pass must not throw away what we know. await upsertMetadataScan( paths, SLUG, { errors: { a: { class: "error", message: "flaky", at: T2 } } }, T2, ); const later = await loadMetadataScan(paths, SLUG); assert.equal(later.errors.a, undefined); assert.ok(later.entries.a); }); }); test("an unchanged upsert skips the write entirely", async () => { await withPaths(async (paths) => { await upsertMetadataScan(paths, SLUG, { entries: { a: entry() } }, T1); const file = metadataScanPath(paths, SLUG); const before = (await stat(file)).mtimeMs; // Same content, different scannedAt — scannedAt alone is not a change, // otherwise every re-run rewrites a multi-megabyte file for nothing. await upsertMetadataScan( paths, SLUG, { entries: { a: entry({ scannedAt: T2 }) } }, T2, ); assert.equal((await stat(file)).mtimeMs, before); assert.equal((await loadMetadataScan(paths, SLUG)).updatedAt, T1); }); }); test("an identical error a cooldown old refreshes its `at` (25 h later rewrites)", async () => { await withPaths(async (paths) => { const err = { class: "members_only", message: "join", at: T1 }; await upsertMetadataScan(paths, SLUG, { errors: { a: err } }, T1); const later = new Date(Date.parse(T1) + 25 * 3_600_000).toISOString(); await upsertMetadataScan( paths, SLUG, { errors: { a: { ...err, at: later } } }, later, ); const scan = await loadMetadataScan(paths, SLUG); assert.equal(scan.errors.a.at, later); assert.equal(scan.updatedAt, later); }); }); test("an identical error inside the cooldown leaves the file alone (1 h later)", async () => { await withPaths(async (paths) => { const err = { class: "members_only", message: "join", at: T1 }; await upsertMetadataScan(paths, SLUG, { errors: { a: err } }, T1); const file = metadataScanPath(paths, SLUG); const before = (await stat(file)).mtimeMs; const later = new Date(Date.parse(T1) + 3_600_000).toISOString(); await upsertMetadataScan( paths, SLUG, { errors: { a: { ...err, at: later } } }, later, ); assert.equal((await stat(file)).mtimeMs, before); const scan = await loadMetadataScan(paths, SLUG); assert.equal(scan.errors.a.at, T1); assert.equal(scan.updatedAt, T1); }); }); test("normalize-on-read drops unknown fields and malformed records", async () => { await withPaths(async (paths) => { await writeFile( metadataScanPath(paths, SLUG), JSON.stringify({ version: 1, updatedAt: T1, lastRun: { startedAt: T1, finishedAt: T1, scanned: 2, errors: 0, junk: 1 }, entries: { good: { title: "T", description: "D", uploadDate: "20240101", scannedAt: T1, junk: true }, noStamp: { title: "T" }, notAnObject: 7, }, errors: { bad: { class: "deleted", message: "gone", at: T1, junk: 2 } }, }), ); const scan = await loadMetadataScan(paths, SLUG); assert.deepEqual(Object.keys(scan.entries), ["good"]); assert.deepEqual(Object.keys(scan.entries.good).sort(), [ "description", "scannedAt", "title", "uploadDate", ]); assert.deepEqual(Object.keys(scan.errors), ["bad"]); assert.deepEqual(Object.keys(scan.errors.bad).sort(), ["at", "class", "message"]); assert.equal(scan.lastRun?.scanned, 2); assert.equal("junk" in (scan.lastRun ?? {}), false); }); }); test("a corrupt file loads as empty rather than throwing", async () => { await withPaths(async (paths) => { await writeFile(metadataScanPath(paths, SLUG), "{not json"); assert.deepEqual((await loadMetadataScan(paths, SLUG)).entries, {}); }); }); // --- settled, derived -------------------------------------------------------- async function seed(paths: Paths) { await upsertMetadataScan( paths, SLUG, { entries: { guest1: entry({ title: "Synthetic guestvid0001", description: "" }), guest2: entry({ title: "Episode 9", description: "with a guest" }), plain1: entry({ title: "Synthetic plainvid0001", description: "" }), plain2: entry({ title: "Synthetic plainvid0002", description: "" }), }, errors: { unreadable: { class: "needs_auth", message: "age", at: T1 } }, }, T1, ); } test("settled = the ids the CURRENT filter rejects", async () => { await withPaths(async (paths) => { await seed(paths); assert.deepEqual( [...(await settledByTitleFilterIds(paths, SLUG, config("guest")))].sort(), ["plain1", "plain2"], ); // Edit the pattern: the whole channel is re-decided, with no rescan and // nothing stored per video to migrate. assert.deepEqual( [...(await settledByTitleFilterIds(paths, SLUG, config("plain")))].sort(), ["guest1", "guest2"], ); // Exclude wins. assert.deepEqual( [ ...(await settledByTitleFilterIds(paths, SLUG, config(undefined, "guest"))), ].sort(), ["guest1", "guest2"], ); }); }); test("no filter, and an inert filter, settle nothing", async () => { await withPaths(async (paths) => { await seed(paths); assert.equal((await settledByTitleFilterIds(paths, SLUG, config())).size, 0); assert.equal((await settledByTitleFilterIds(paths, SLUG, null)).size, 0); // Unparseable = inert at download time, so it must settle nothing either. assert.equal( (await settledByTitleFilterIds(paths, SLUG, config("elf("))).size, 0, ); }); }); test("an id we only have an ERROR for is never settled", async () => { await withPaths(async (paths) => { await seed(paths); const settled = await settledByTitleFilterIds(paths, SLUG, config("guest")); // We do not know what `unreadable` is called, so we must not decide it — // it stays ordinary undownloaded work. assert.equal(settled.has("unreadable"), false); }); }); test("an unscanned channel settles nothing, whatever the filter says", async () => { await withPaths(async (paths) => { assert.equal( (await settledByTitleFilterIds(paths, SLUG, config("guest"))).size, 0, ); assert.equal( await readFile(metadataScanPath(paths, SLUG), "utf8").catch(() => null), null, ); }); }); // --- write safety ------------------------------------------------------------ test("overlapping upserts do not collide on the tmp file", async () => { await withPaths(async (paths) => { // Two writers in the same process used to share one pid-named tmp path, so // the second rename found it already moved and threw ENOENT — which failed // the whole scan job. The download path and the scan both write here. await Promise.all( Array.from({ length: 12 }, (_, i) => upsertMetadataScan( paths, SLUG, { entries: { [`v${i}`]: entry({ title: `T${i}` }) } }, T1, ), ), ); const scan = await loadMetadataScan(paths, SLUG); // Every write landed, and nothing threw. (Last-write-wins on the merged // document is expected; losing a write is not the failure this pins — the // ENOENT crash is.) assert.ok(Object.keys(scan.entries).length >= 1); }); }); test("a delta upsert does not drop what is already stored", async () => { await withPaths(async (paths) => { await upsertMetadataScan(paths, SLUG, { entries: { a: entry() } }, T1); // The scan flushes only what it has not written yet; the store must merge. await upsertMetadataScan(paths, SLUG, { entries: { b: entry() } }, T2); assert.deepEqual( Object.keys((await loadMetadataScan(paths, SLUG)).entries).sort(), ["a", "b"], ); }); });