import { test } from "node:test"; import assert from "node:assert/strict"; import type { ChannelTranscriptsManifest } from "../manifest"; import type { TranscriptDetail } from "../transcripts"; import type { ChannelDigestsManifest, VideoDigest } from "../digests"; import { manifestUrl, pageUrl } from "./contract"; import { PageCache, PromiseMap, RemoteSource, type ArchiveReader, type ChannelRef, } from "./reader"; // ─── an in-memory archive ─── // // A Map from URL to JSON body, handed to RemoteSource as `fetch`. That makes // the thing under test the REAL reader (its caches, its walk, its tolerance) // rather than a stub that agrees with it — and it makes every assertion below // about fetches countable, which is the only way to state "one read, not N". const ORIGIN = "https://fixture.example"; type Archive = { fetches: string[]; body: Map; // Resolvers for URLs deliberately held open, so two concurrent callers can be // observed sharing one in-flight read. gate?: { url: string; release: () => void }; }; function makeArchive(entries: Record): Archive { const a: Archive = { fetches: [], body: new Map(Object.entries(entries)) }; return a; } function installFetch(a: Archive, hold?: string): () => void { const real = globalThis.fetch; let release: (() => void) | undefined; globalThis.fetch = (async (input: RequestInfo | URL) => { const url = String(input); a.fetches.push(url); if (!a.body.has(url)) { return new Response("not found", { status: 404, statusText: "Not Found" }); } const text = JSON.stringify(a.body.get(url)); if (hold !== undefined && url === hold) { await new Promise((r) => { release = r; }); } return new Response(text, { status: 200 }); }) as typeof fetch; a.gate = { url: hold ?? "", release: () => release?.() }; return () => { globalThis.fetch = real; }; } function pageRecords(slug: string, ids: string[]): TranscriptDetail[] { return ids.map( (id) => ({ id, slug: `${slug}/${id}`, title: id, uploadDate: "20240101", duration: 60, channel: slug, channelSlug: slug, cues: [{ start: 0, end: 1, text: "hello" }], }) as unknown as TranscriptDetail, ); } const CORPUS = { spec: 3, kind: "site", site: { id: "fixture", title: "Fixture", url: ORIGIN }, channels: [{ slug: "alpha", name: "Alpha", videoCount: 3 }], }; const ALPHA_MANIFEST: ChannelTranscriptsManifest = { version: 1, channelSlug: "alpha", pageCount: 2, maxPageBytes: 1000, generatedAt: "2026-09-12T00:00:00.000Z", slugToPage: { "alpha/a1": 0, "alpha/a2": 0, "alpha/a3": 1 }, }; function fullArchive(): Archive { return makeArchive({ [`${ORIGIN}/corpus.json`]: CORPUS, [manifestUrl("transcripts", "alpha", ORIGIN)]: ALPHA_MANIFEST, [pageUrl("transcripts", "alpha", 0, ORIGIN)]: pageRecords("alpha", ["a1", "a2"]), [pageUrl("transcripts", "alpha", 1, ORIGIN)]: pageRecords("alpha", ["a3"]), }); } const ALPHA: ChannelRef = { key: "alpha", slug: "alpha", name: "Alpha" }; // ─── the walk ─── test("the documented walk: corpus.json -> manifest -> slugToPage -> page", async () => { const a = fullArchive(); const restore = installFetch(a); try { const reader: ArchiveReader = new RemoteSource(ORIGIN); const channels = await reader.listChannels(); assert.deepEqual( channels.map((c) => c.slug), ["alpha"], ); // The channel ref carries its owning origin, which is what lets a hub // attribute a hit to the member site it came from. assert.equal(channels[0].siteUrl, ORIGIN); const manifest = await reader.transcriptsManifest(channels[0]); const page = manifest.slugToPage["alpha/a3"]; assert.equal(page, 1); const records = await reader.transcriptPage(channels[0], page); assert.deepEqual( records.map((r) => r.id), ["a3"], ); assert.deepEqual(a.fetches, [ `${ORIGIN}/corpus.json`, `${ORIGIN}/transcripts/alpha/manifest.json`, `${ORIGIN}/transcripts/alpha/page-0001.json`, ]); } finally { restore(); } }); test("the reader reproduces the contract's URL shapes exactly", async () => { const a = fullArchive(); const restore = installFetch(a); try { const reader = new RemoteSource(ORIGIN); await reader.transcriptsManifest(ALPHA); await reader.transcriptPage(ALPHA, 0); assert.deepEqual(a.fetches, [ manifestUrl("transcripts", "alpha", ORIGIN), pageUrl("transcripts", "alpha", 0, ORIGIN), ]); // …and the shapes really are the published ones, not just self-consistent. assert.deepEqual(a.fetches, [ `${ORIGIN}/transcripts/alpha/manifest.json`, `${ORIGIN}/transcripts/alpha/page-0000.json`, ]); } finally { restore(); } }); test("a trailing slash on the origin does not double up", async () => { const a = fullArchive(); const restore = installFetch(a); try { const reader = new RemoteSource(`${ORIGIN}/`); await reader.transcriptsManifest(ALPHA); assert.deepEqual(a.fetches, [`${ORIGIN}/transcripts/alpha/manifest.json`]); } finally { restore(); } }); // ─── caching ─── test("the manifest memo is the promise: two concurrent callers, one fetch", async () => { const a = fullArchive(); const held = manifestUrl("transcripts", "alpha", ORIGIN); const restore = installFetch(a, held); try { const reader = new RemoteSource(ORIGIN); const first = reader.transcriptsManifest(ALPHA); const second = reader.transcriptsManifest(ALPHA); // Both callers are waiting on the SAME in-flight read — not two reads that // happen to land on the same answer. assert.equal(a.fetches.length, 1); a.gate?.release(); const [m1, m2] = await Promise.all([first, second]); assert.equal(m1, m2); assert.equal(a.fetches.length, 1); // A third, after it resolved, is still free. await reader.transcriptsManifest(ALPHA); assert.equal(a.fetches.length, 1); } finally { restore(); } }); test("listChannels({refresh}) is the only way to re-read, and it drops the page caches too", async () => { const a = fullArchive(); const restore = installFetch(a); try { const reader = new RemoteSource(ORIGIN); await reader.listChannels(); await reader.transcriptsManifest(ALPHA); await reader.transcriptPage(ALPHA, 0); const before = a.fetches.length; await reader.listChannels(); await reader.transcriptsManifest(ALPHA); await reader.transcriptPage(ALPHA, 0); assert.equal(a.fetches.length, before, "everything served from cache"); await reader.listChannels({ refresh: true }); await reader.transcriptsManifest(ALPHA); await reader.transcriptPage(ALPHA, 0); assert.equal(a.fetches.length, before * 2); } finally { restore(); } }); test("a failed read is not memoised", async () => { const a = makeArchive({}); const restore = installFetch(a); try { const reader = new RemoteSource(ORIGIN); await assert.rejects(() => reader.listChannels(), /404/); // The corpus appears on the second try; a memoised failure would hide it. a.body.set(`${ORIGIN}/corpus.json`, CORPUS); const channels = await reader.listChannels(); assert.equal(channels.length, 1); } finally { restore(); } }); // ─── the sparse layers: a 404 is data, and the negative is cached ─── test("a channel with no digests caches the absence — one probe, not one per query", async () => { const a = fullArchive(); const restore = installFetch(a); try { const reader = new RemoteSource(ORIGIN); assert.equal(await reader.digestsManifest?.(ALPHA), null); assert.equal(await reader.digestsManifest?.(ALPHA), null); assert.equal(await reader.digestsManifest?.(ALPHA), null); assert.deepEqual(a.fetches, [manifestUrl("digests", "alpha", ORIGIN)]); } finally { restore(); } }); test("a channel WITH digests reads them through the same walk", async () => { const digests: ChannelDigestsManifest = { version: 1, channelSlug: "alpha", pageCount: 1, maxPageBytes: 1000, generatedAt: "2026-09-12T00:00:00.000Z", slugToPage: { "alpha/a1": 0 }, } as unknown as ChannelDigestsManifest; const a = fullArchive(); a.body.set(manifestUrl("digests", "alpha", ORIGIN), digests); a.body.set(pageUrl("digests", "alpha", 0, ORIGIN), [ { slug: "alpha/a1", chapters: [] } as unknown as VideoDigest, ]); const restore = installFetch(a); try { const reader = new RemoteSource(ORIGIN); const m = await reader.digestsManifest?.(ALPHA); assert.equal(m?.slugToPage["alpha/a1"], 0); const page = await reader.digestPage?.(ALPHA, 0); assert.equal(page?.length, 1); } finally { restore(); } }); test("subs and posts are absent-tolerant in the same way", async () => { const a = fullArchive(); const restore = installFetch(a); try { const reader = new RemoteSource(ORIGIN); assert.equal(await reader.subsManifest(ALPHA), null); assert.equal(await reader.postsManifest(ALPHA), null); assert.equal(await reader.subsManifest(ALPHA), null); assert.equal(await reader.postsManifest(ALPHA), null); assert.deepEqual(a.fetches, [ manifestUrl("subs", "alpha", ORIGIN), manifestUrl("posts", "alpha", ORIGIN), ]); } finally { restore(); } }); test("an absent summaries index is an empty map, not an error", async () => { const a = fullArchive(); const restore = installFetch(a); try { const reader = new RemoteSource(ORIGIN); const index = await reader.videoIndex?.(); assert.equal(index?.size, 0); // And the availability map IS that index — one read, not two. const avail = await reader.availabilityMap(); assert.equal(avail.size, 0); } finally { restore(); } }); test("an absent duplicates.json is no clusters, not a failure", async () => { const a = fullArchive(); const restore = installFetch(a); try { const reader = new RemoteSource(ORIGIN); const dupes = await reader.duplicateIndex?.(); assert.equal(dupes?.size, 0); assert.deepEqual(a.fetches, [`${ORIGIN}/duplicates.json`]); } finally { restore(); } }); test("an absent search-aliases.json leaves search plain", async () => { const a = fullArchive(); const restore = installFetch(a); try { const reader = new RemoteSource(ORIGIN); assert.deepEqual(await reader.loadAliases(), []); assert.deepEqual(await reader.loadAliases(), []); assert.deepEqual(a.fetches, [`${ORIGIN}/search-aliases.json`]); } finally { restore(); } }); // ─── PromiseMap ─── test("PromiseMap: one load per key, and a rejection evicts itself", async () => { const map = new PromiseMap(); let loads = 0; const load = () => { loads++; return Promise.resolve(7); }; const [a, b] = await Promise.all([map.take("k", load), map.take("k", load)]); assert.equal(a, 7); assert.equal(b, 7); assert.equal(loads, 1); let fails = 0; const boom = () => { fails++; return Promise.reject(new Error("nope")); }; await assert.rejects(() => map.take("bad", boom)); await assert.rejects(() => map.take("bad", boom)); assert.equal(fails, 2, "a transient failure is never cached as a permanent one"); map.clear(); await map.take("k", load); assert.equal(loads, 2); }); // ─── PageCache ─── test("PageCache evicts by RAW BYTES, least-recently-used first", async () => { // Budget for exactly three 10-byte pages. const cache = new PageCache(30); const loaded: string[] = []; const put = (k: string) => cache.take(k, async () => { loaded.push(k); return { value: k, bytes: 10 }; }); await put("p1"); await put("p2"); await put("p3"); assert.deepEqual(loaded, ["p1", "p2", "p3"], "three pages fit the budget exactly"); // A hit moves its entry to the most-recently-used end. Touch p1 so the // least-recently-used is p2, not the insertion order's p1 — that is the whole // difference between an LRU and a queue. await put("p1"); assert.deepEqual(loaded, ["p1", "p2", "p3"], "all three still resident"); // A fourth page takes the total to 40 > 30, so one goes: p2. await put("p4"); assert.deepEqual(loaded, ["p1", "p2", "p3", "p4"]); await put("p3"); await put("p1"); assert.equal(loaded.length, 4, "p3 and p1 survived"); await put("p2"); assert.deepEqual(loaded, ["p1", "p2", "p3", "p4", "p2"], "p2 was the victim"); }); test("PageCache keeps MIN_CACHED_PAGES even when one page blows the budget", async () => { // One page is four times the whole budget: thrashing on it would mean never // caching anything, which is worse than going over. const cache = new PageCache(10); const loaded: string[] = []; const put = (k: string) => cache.take(k, async () => { loaded.push(k); return { value: k, bytes: 40 }; }); await put("big1"); await put("big2"); await put("big1"); await put("big2"); assert.deepEqual(loaded, ["big1", "big2"]); }); test("PageCache with a zero budget caches nothing but still serves", async () => { const cache = new PageCache(0); const loaded: string[] = []; const put = (k: string) => cache.take(k, async () => { loaded.push(k); return { value: k, bytes: 10 }; }); assert.equal(await put("p"), "p"); assert.equal(await put("p"), "p"); assert.deepEqual(loaded, ["p", "p"]); }); test("PageCache coalesces concurrent callers and drops a rejection", async () => { const cache = new PageCache(1000); let loads = 0; let release!: () => void; const gate = new Promise((r) => { release = r; }); const load = async () => { loads++; await gate; return { value: "v", bytes: 10 }; }; const a = cache.take("k", load); const b = cache.take("k", load); assert.equal(loads, 1); release(); assert.deepEqual(await Promise.all([a, b]), ["v", "v"]); let fails = 0; const boom = async () => { fails++; throw new Error("nope"); }; await assert.rejects(() => cache.take("bad", boom)); await assert.rejects(() => cache.take("bad", boom)); assert.equal(fails, 2); });