import { test } from "node:test"; import assert from "node:assert/strict"; import type { Cue } from "../lib/vtt"; import type { TranscriptSummary } from "../lib/transcripts"; import type { CuratedTagDef, CuratedTagsConfig } from "../lib/curatedTags"; import { applyCuratedTagsToSummary, clearCuratedPagesPending, CURATED_REAPPLY_YIELD_EVERY, createTagCounter, curatedTagsRuntime, deriveCuratedTags, hashCuratedAssignments, hashCuratedRules, publishedTagsFrom, reapplyCuratedTags, META_ASSIGN_HASH, META_ASSIGN_SIGS, META_PAGES_PENDING, META_RULES_HASH, } from "./curatedTagsIndex"; type IndexKey = [string, string, string]; // Minimal stand-ins for the LMDB sub-DBs buildIndex hands the re-apply pass. // Keys are the same tuples; range scans only need to be correct, not fast. function makeDb() { const map = new Map(); const k = (key: unknown) => JSON.stringify(key); return { map, get(key: unknown): V | undefined { return map.get(k(key))?.value; }, put(key: unknown, value: V): void { map.set(k(key), { key, value }); }, // Mirrors lmdb-js: `values: false` yields BARE keys, not {key, value} — // a caller that destructures {key} under that option must fail here too. getRange(options?: { start?: unknown[]; values?: boolean }): Iterable<{ key: unknown; value: V }> { if (options?.values === false) { return this.getRangeBareKeys(options) as unknown as Iterable<{ key: unknown; value: V }>; } return this.getRangeEntries(options); }, *getRangeBareKeys(options?: { start?: unknown[] }): Iterable { for (const e of this.getRangeEntries(options)) yield e.key; }, getRangeEntries(options?: { start?: unknown[] }): Iterable<{ key: unknown; value: V }> { const start = options?.start as string[] | undefined; const all = Array.from(map.values()).sort((a, b) => k(a.key) < k(b.key) ? -1 : 1, ); if (!start) return all; return all.filter((e) => (e.key as string[])[0] === start[0]); }, }; } function makeMeta() { const map = new Map(); return { map, get: (key: string) => map.get(key), put: (key: string, value: unknown) => void map.set(key, value), }; } function summary( channelSlug: string, id: string, uploadDate: string, over: Partial = {}, ): TranscriptSummary { return { slug: `${channelSlug}/${id}`, id, channelSlug, title: `Video ${id}`, uploadDate, duration: 100, channel: channelSlug, description: "", tags: [], isLivestream: false, ageRestricted: false, platform: "youtube", webpageUrl: `https://example.com/${id}`, ...over, }; } const COLLAB_RULE: CuratedTagDef = { id: "eva-collab", label: "Collab", order: 1, rules: [{ id: "meta", kind: "metadata", pattern: "elfpire", enabled: true }], }; function fixture(config: CuratedTagsConfig) { const sums = makeDb(); const cues = makeDb(); const subs = makeDb<{ track: string; cues: Cue[] }[]>(); const byChannel = makeDb(); const meta = makeMeta(); const put = (s: TranscriptSummary) => { const ik: IndexKey = [s.uploadDate, s.channelSlug, s.id]; sums.put(ik, s); byChannel.put([s.channelSlug, s.uploadDate, s.id], 1); return ik; }; return { sums, cues, subs, byChannel, meta, put, runtime: curatedTagsRuntime(config) }; } function cfg( tags: CuratedTagDef[], assignments: CuratedTagsConfig["assignments"] = {}, ): CuratedTagsConfig { return { version: 1, tags, assignments }; } // ─── hashes ─── test("hashCuratedRules ignores presentation, reacts to derivation", () => { const base = hashCuratedRules([COLLAB_RULE]); assert.equal( hashCuratedRules([{ ...COLLAB_RULE, label: "On mic", color: "#fff", hidden: true }]), base, "relabelling must not re-page the corpus", ); assert.notEqual( hashCuratedRules([ { ...COLLAB_RULE, rules: [{ id: "meta", kind: "metadata", pattern: "eva", enabled: true }] }, ]), base, ); assert.notEqual(hashCuratedRules([{ ...COLLAB_RULE, order: 9 }]), base); assert.notEqual( hashCuratedRules([ { ...COLLAB_RULE, rules: [{ ...COLLAB_RULE.rules![0], channels: ["legal-mindset"] }], }, ]), base, ); // An exclusion changes which records the rule derives, so it is in the hash // — and the empty list it replaces must not be, or the field's arrival would // re-derive the corpus a second time. assert.notEqual( hashCuratedRules([ { ...COLLAB_RULE, rules: [{ ...COLLAB_RULE.rules![0], channelsExclude: ["mommaocco"] }], }, ]), base, ); assert.equal( hashCuratedRules([ { ...COLLAB_RULE, rules: [{ ...COLLAB_RULE.rules![0], channelsExclude: [] }], }, ]), base, ); // A disabled rule is not part of the derivation at all. assert.equal( hashCuratedRules([ { ...COLLAB_RULE, rules: [COLLAB_RULE.rules![0], { id: "off", kind: "caption", pattern: "x", enabled: false }], }, ]), base, ); }); test("hashCuratedAssignments is order-insensitive", () => { const a = hashCuratedAssignments({ "c/v1": { manual: ["b", "a"] }, "c/v2": { suppressed: ["z"] }, }); const b = hashCuratedAssignments({ "c/v2": { suppressed: ["z"] }, "c/v1": { manual: ["a", "b"] }, }); assert.equal(a, b); assert.notEqual(a, hashCuratedAssignments({ "c/v1": { manual: ["a"] } })); }); // ─── the summary field ─── test("applyCuratedTagsToSummary omits the key when empty", () => { const s = summary("c", "v", "20260101"); assert.equal(applyCuratedTagsToSummary(s, []), false); assert.equal("curatedTags" in s, false); assert.equal(applyCuratedTagsToSummary(s, ["a"]), true); assert.deepEqual(s.curatedTags, ["a"]); assert.equal(applyCuratedTagsToSummary(s, ["a"]), false, "idempotent"); assert.equal(applyCuratedTagsToSummary(s, []), true); assert.equal("curatedTags" in s, false, "the key is deleted, not emptied"); }); // ─── derivation ─── test("deriveCuratedTags folds rule hits with pins and suppressions", () => { const runtime = curatedTagsRuntime( cfg([COLLAB_RULE, { id: "manual-only", label: "Manual", order: 2 }], { "lm/v2": { manual: ["manual-only"] }, "lm/v3": { suppressed: ["eva-collab"] }, }), ); assert.deepEqual( deriveCuratedTags(runtime, { channelSlug: "lm", id: "v1", title: "with Elfpire" }), ["eva-collab"], ); assert.deepEqual( deriveCuratedTags(runtime, { channelSlug: "lm", id: "v2", title: "nothing" }), ["manual-only"], ); assert.deepEqual( deriveCuratedTags(runtime, { channelSlug: "lm", id: "v3", title: "with Elfpire" }), [], "a suppression kills a rule hit", ); }); // ─── reapplyCuratedTags ─── test("unchanged hashes do no work at all", async () => { const f = fixture(cfg([COLLAB_RULE])); f.put(summary("lm", "v1", "20260101", { title: "Elfpire collab" })); f.meta.put(META_RULES_HASH, f.runtime.rulesHash); f.meta.put(META_ASSIGN_HASH, f.runtime.assignHash); const res = await reapplyCuratedTags({ ...f, collectChanged: true }); assert.deepEqual(res, { rulesChanged: false, assignmentsChanged: false, changedCount: 0, changed: [], examined: 0, yields: 0, pagesPending: false, }); }); // THE COLD START, and the reason curated tags need no SCHEMA_VERSION bump: an // index built before they existed carries no hash at all, which can never equal // the current one, so the first build re-derives every record by itself. test("an index with no stored hashes re-derives every record", async () => { const f = fixture(cfg([COLLAB_RULE])); f.put(summary("lm", "v1", "20260101", { title: "Elfpire collab" })); f.put(summary("lm", "v2", "20260102", { title: "ordinary" })); assert.equal(f.meta.get(META_RULES_HASH), undefined); const res = await reapplyCuratedTags({ ...f, collectChanged: true }); assert.equal(res.rulesChanged, true); assert.equal(res.examined, 2, "every record is a candidate on a cold index"); assert.equal(res.changedCount, 1); assert.deepEqual(res.changed?.map((k) => k[2]), ["v1"]); }); test("an UNTAGGED corpus cold-starts to zero changes — no page rewrites", async () => { // The state of every existing install the day this ships: no vocabulary, no // assignments. The pass still runs (the hashes have to be recorded), but // every record derives [] and comes out byte-identical, so nothing is written // and the caller never flips sharedNeedsBuild. This is what makes the absent // SCHEMA_VERSION bump free rather than merely cheap. const f = fixture(cfg([])); for (let i = 0; i < 10; i++) f.put(summary("lm", `v${i}`, `2026010${i}`)); const before = JSON.stringify(Array.from(f.sums.map.values())); const res = await reapplyCuratedTags({ ...f }); assert.equal(res.changedCount, 0); assert.equal(res.pagesPending, false, "nothing moved, so nothing to re-page"); assert.equal(res.examined, 10); // The keys are not collected unless asked for — a 30k-record first build must // not hold 30k tuples to answer a boolean. assert.equal(res.changed, undefined); assert.equal(JSON.stringify(Array.from(f.sums.map.values())), before); for (const { value } of f.sums.map.values()) { assert.equal("curatedTags" in value, false); } // And the hashes are now recorded, so every later build is a true no-op. assert.equal((await reapplyCuratedTags({ ...f })).examined, 0); }); test("a rule change re-derives the whole corpus from LMDB and reports the movers", async () => { const f = fixture(cfg([COLLAB_RULE])); f.put(summary("lm", "v1", "20260101", { title: "Elfpire collab" })); f.put(summary("lm", "v2", "20260102", { title: "ordinary stream" })); f.put(summary("other", "v3", "20260103", { description: "ELFPIRE guested" })); const res = await reapplyCuratedTags({ ...f, collectChanged: true }); assert.equal(res.rulesChanged, true); assert.equal(res.examined, 3); assert.equal(res.changedCount, 2); assert.deepEqual( res.changed?.map((k) => k[2]).sort(), ["v1", "v3"], ); assert.deepEqual(f.sums.get(["20260101", "lm", "v1"])!.curatedTags, ["eva-collab"]); assert.equal(f.sums.get(["20260102", "lm", "v2"])!.curatedTags, undefined); // The hashes are recorded, so a second pass is a no-op. assert.equal(f.meta.get(META_RULES_HASH), f.runtime.rulesHash); assert.equal((await reapplyCuratedTags({ ...f })).examined, 0); }); test("removing the last rule strips the tag back off every record", async () => { const f = fixture(cfg([COLLAB_RULE])); const ik = f.put(summary("lm", "v1", "20260101", { title: "Elfpire collab" })); await reapplyCuratedTags({ ...f }); assert.deepEqual(f.sums.get(ik)!.curatedTags, ["eva-collab"]); const gone = { ...f, runtime: curatedTagsRuntime(cfg([])) }; const res = await reapplyCuratedTags(gone); assert.equal(res.changedCount, 1); assert.equal(f.sums.get(ik)!.curatedTags, undefined); }); test("an assignment-only change touches ONLY the videos whose assignment moved", async () => { const f = fixture(cfg([COLLAB_RULE])); for (let i = 0; i < 20; i++) { f.put(summary("lm", `v${i}`, `202601${String(i + 10).padStart(2, "0")}`)); } f.put(summary("other", "w1", "20260201")); await reapplyCuratedTags({ ...f }); // establish the baseline hashes const pinned = { ...f, runtime: curatedTagsRuntime( cfg([COLLAB_RULE], { "lm/v7": { manual: ["eva-collab"] } }), ), }; const res = await reapplyCuratedTags({ ...pinned, collectChanged: true }); assert.equal(res.rulesChanged, false); assert.equal(res.assignmentsChanged, true); // 21 videos in the corpus; exactly one was even looked at. assert.equal(res.examined, 1); assert.deepEqual(res.changed?.map((k) => k[2]), ["v7"]); assert.deepEqual(f.sums.get(["20260117", "lm", "v7"])!.curatedTags, ["eva-collab"]); // Clearing it again is also a one-video pass, and the key comes back off. const cleared = { ...f, runtime: curatedTagsRuntime(cfg([COLLAB_RULE])) }; const res2 = await reapplyCuratedTags(cleared); assert.equal(res2.examined, 1); assert.equal(f.sums.get(["20260117", "lm", "v7"])!.curatedTags, undefined); }); test("videos the worker already derived this build are not done twice", async () => { const f = fixture(cfg([COLLAB_RULE])); const ik = f.put(summary("lm", "v1", "20260101", { title: "Elfpire collab" })); const res = await reapplyCuratedTags({ ...f, alreadyFresh: new Set([`${ik[0]}\x00${ik[1]}\x00${ik[2]}`]), collectChanged: true, }); assert.equal(res.examined, 0); assert.deepEqual(res.changed, []); }); test("a schema bump records the hashes and re-applies nothing", async () => { const f = fixture(cfg([COLLAB_RULE])); f.put(summary("lm", "v1", "20260101", { title: "Elfpire collab" })); const res = await reapplyCuratedTags({ ...f, allFresh: true }); assert.equal(res.changedCount, 0); assert.equal(res.examined, 0); // The pages ARE rebuilt after a wipe, so the debt is recorded all the same — // an interrupt mid-rebuild must not be forgotten. assert.equal(res.pagesPending, true); assert.equal(f.meta.get(META_RULES_HASH), f.runtime.rulesHash); assert.deepEqual(f.meta.get(META_ASSIGN_SIGS), {}); }); // ─── the interrupted-build flag ─── test("a change sets curatedPagesPending, and only clearing it settles the debt", async () => { const f = fixture(cfg([COLLAB_RULE])); f.put(summary("lm", "v1", "20260101", { title: "Elfpire collab" })); const first = await reapplyCuratedTags({ ...f }); assert.equal(first.changedCount, 1); assert.equal(first.pagesPending, true); assert.equal(f.meta.get(META_PAGES_PENDING), true); // THE INTERRUPT: the hashes are stored but the page build never ran, so the // flag was never cleared. The next build finds the hashes equal — it // re-derives nothing — and must STILL report that the pages owe a rewrite. const afterCrash = await reapplyCuratedTags({ ...f }); assert.equal(afterCrash.rulesChanged, false); assert.equal(afterCrash.changedCount, 0); assert.equal( afterCrash.pagesPending, true, "hashes match but the shards are still stale", ); // The page build completed this time. clearCuratedPagesPending(f.meta); assert.equal(f.meta.get(META_PAGES_PENDING), false); assert.equal((await reapplyCuratedTags({ ...f })).pagesPending, false); }); test("no change means no debt — an untagged corpus never sets the flag", async () => { const f = fixture(cfg([])); f.put(summary("lm", "v1", "20260101")); assert.equal((await reapplyCuratedTags({ ...f })).pagesPending, false); assert.notEqual(f.meta.get(META_PAGES_PENDING), true); }); test("cue-backed rules read cues out of LMDB, and only where they are scoped", async () => { const capRule: CuratedTagDef = { id: "eva-topic", label: "Discussed", rules: [ { id: "cap", kind: "caption", pattern: "elfpire", channels: ["lm"], enabled: true, }, ], }; const chatRule: CuratedTagDef = { id: "eva-in-chat", label: "In chat", rules: [{ id: "chat", kind: "chat-author", pattern: "elfpire", enabled: true }], }; const f = fixture(cfg([capRule, chatRule])); const a = f.put(summary("lm", "v1", "20260101")); const b = f.put(summary("other", "v2", "20260102")); f.cues.put(a, [{ start: 0, end: 1, text: "we talked about Elfpire" }]); f.cues.put(b, [{ start: 0, end: 1, text: "we talked about Elfpire" }]); f.subs.put(a, [ { track: "en", cues: [{ start: 0, end: 1, text: "ElfpireEva: not chat" }] }, { track: "live_chat", cues: [{ start: 0, end: 1, text: "ElfpireEva: hi" }] }, ]); await reapplyCuratedTags({ ...f }); // Order follows the vocabulary (neither def sets `order`, so it is the // config's own order), not the alphabet. assert.deepEqual(f.sums.get(a)!.curatedTags, ["eva-topic", "eva-in-chat"]); // The caption rule is scoped to `lm`, so the other channel gets nothing even // though its cues would match. assert.equal(f.sums.get(b)!.curatedTags, undefined); }); // ─── yielding ─── // The bug this section exists for: on the live editor a single unscoped caption // rule turned this pass into two to three minutes of unbroken synchronous work // — 77k records, each one decoding its cues out of LMDB — and every page render // in the same process timed out while it ran. The fix is not to make it faster; // it is to make it interruptible. // // `ticks` is the instrument: a self-rescheduling setImmediate chain can only // advance if the event loop actually reaches its check phase, which it cannot // do while a synchronous loop holds the thread. function immediateTicker() { let ticks = 0; let running = true; const tick = () => { if (!running) return; ticks++; setImmediate(tick); }; setImmediate(tick); return { get ticks() { return ticks; }, stop() { running = false; }, }; } test("the whole-corpus walk hands the event loop back as it goes", async () => { const capRule: CuratedTagDef = { id: "eva-topic", label: "Discussed", rules: [{ id: "cap", kind: "caption", pattern: "elfpire", enabled: true }], }; const f = fixture(cfg([capRule])); const n = 1000; for (let i = 0; i < n; i++) { const ik = f.put(summary("lm", `v${String(i).padStart(4, "0")}`, "20260101")); // A caption rule is the expensive shape: every record pays a cue read. f.cues.put(ik, [{ start: 0, end: 1, text: `nothing here ${i}` }]); } const ticker = immediateTicker(); const res = await reapplyCuratedTags({ ...f }); ticker.stop(); assert.equal(res.examined, n); assert.equal( res.yields, Math.floor(n / CURATED_REAPPLY_YIELD_EVERY), "one yield per CURATED_REAPPLY_YIELD_EVERY records walked", ); assert.ok( ticker.ticks > 0, `the event loop never ran during the pass (${ticker.ticks} ticks)`, ); }); // The negative control, and the state of every untagged install: the pass still // runs — the hashes have to be recorded — but it must not buy a round trip // through the event loop to walk ten records. test("a corpus below the yield interval returns without yielding once", async () => { const f = fixture(cfg([])); for (let i = 0; i < 10; i++) f.put(summary("lm", `v${i}`, `2026010${i}`)); const ticker = immediateTicker(); const res = await reapplyCuratedTags({ ...f }); ticker.stop(); assert.equal(res.examined, 10); assert.equal(res.yields, 0); assert.equal(ticker.ticks, 0, "not even one trip through the check phase"); }); // ─── counts + publication ─── test("createTagCounter accumulates per tag and per channel", () => { const c = createTagCounter(); c.add("lm", ["eva-collab", "eva-topic"]); c.add("lm", ["eva-collab"]); c.add("nux", ["eva-collab"]); c.add("nux", undefined); c.add("nux", []); const file = c.file("2026-09-21T00:00:00Z"); assert.equal(file.version, 1); assert.deepEqual(file.tags["eva-collab"], { count: 3, channels: { lm: 2, nux: 1 }, }); assert.deepEqual(file.tags["eva-topic"], { count: 1, channels: { lm: 1 } }); }); test("publishedTagsFrom drops hidden and zero-count tags and sorts by order", () => { const defs: CuratedTagDef[] = [ { id: "b-tag", label: "B", order: 2, group: "eva", groupLabel: "Eva" }, { id: "a-tag", label: "A", order: 1, color: "#fff" }, { id: "hidden-tag", label: "Hidden", order: 0, hidden: true }, { id: "unused", label: "Unused", order: 3 }, ]; const counts = { version: 1, generatedAt: "", tags: { "a-tag": { count: 2, channels: { lm: 2 } }, "b-tag": { count: 1, channels: { nux: 1 } }, "hidden-tag": { count: 5, channels: { lm: 5 } }, }, }; const published = publishedTagsFrom(defs, counts); assert.deepEqual( published.tags.map((t) => t.id), ["a-tag", "b-tag"], ); assert.deepEqual(published.tags[0], { id: "a-tag", label: "A", color: "#fff", order: 1, count: 2, channels: { lm: 2 }, }); assert.equal(published.tags[1].groupLabel, "Eva"); // No counts at all → nothing to publish (compose then writes no file). assert.deepEqual(publishedTagsFrom(defs, null).tags, []); }); // ─── site-scoped tags (`sites`) ─── test("a site-scoped tag: rule hits and pins only on its sites' channels; no scope, no hash change", async () => { const { sanitizeTagsConfig, tagsForSite } = await import("../lib/curatedTags"); // The sanitizer keeps `sites` (valid ids, deduplicated, sorted) at the corpus // layer and drops it at the site layer. const [def] = sanitizeTagsConfig({ tags: [{ ...COLLAB_RULE, sites: ["anilyzer", "Anilyzer", "bad id", "zz"] }] }).tags; assert.deepEqual(def.sites, ["anilyzer", "zz"]); assert.equal("sites" in sanitizeTagsConfig({ tags: [{ id: "x", sites: ["anilyzer"] }] }, { layer: "site" }).tags[0], false); const scoped: CuratedTagDef = { ...COLLAB_RULE, sites: ["anilyzer"] }; const members = { anilyzer: ["vtuber"], hasanalyzer: ["hasanabi"] }; const runtime = curatedTagsRuntime( cfg([scoped], { "hasanabi/pinned": { manual: ["eva-collab"] }, "vtuber/pinned": { manual: ["eva-collab"] } }), members, ); const video = (channelSlug: string, id = "v1") => ({ channelSlug, id, title: "with elfpire" }); assert.deepEqual(deriveCuratedTags(runtime, video("vtuber")), ["eva-collab"]); assert.deepEqual(deriveCuratedTags(runtime, video("hasanabi")), [], "a rule hit outside the scope"); assert.deepEqual(deriveCuratedTags(runtime, video("hasanabi", "pinned")), [], "a pin outside the scope"); assert.deepEqual(deriveCuratedTags(runtime, video("vtuber", "pinned")), ["eva-collab"]); // The hash: unchanged by the field's existence when nothing is scoped; moved // by a scope, and by a change in the scoped sites' membership. assert.equal(curatedTagsRuntime(cfg([COLLAB_RULE]), members).rulesHash, hashCuratedRules([COLLAB_RULE])); const moved = curatedTagsRuntime(cfg([scoped]), { ...members, anilyzer: ["vtuber", "another"] }); assert.notEqual(runtime.rulesHash, hashCuratedRules([COLLAB_RULE])); assert.notEqual(moved.rulesHash, runtime.rulesHash); // A site's view of a record: a tag scoped elsewhere is dropped; the same // array comes back when nothing is. const tags = ["eva-collab", "other"]; assert.deepEqual(tagsForSite(tags, "hasanalyzer", [scoped]), ["other"]); assert.equal(tagsForSite(tags, "anilyzer", [scoped]), tags); });