import { test } from "node:test"; import assert from "node:assert/strict"; import { mkdtemp, readFile, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { ArchiveOrgClient } from "./archiveOrgClient"; import { ensureArchiveOrgProvenance, loadArchiveOrgProvenance, } from "./archiveOrg-server"; import { loadMetadataHistory } from "./metadataHistory-server"; import { archiveOrgDetailsUrl } from "./archiveOrgId"; // Run with: // pnpm --filter yt-dlp-transcript-common exec tsx --test lib/archiveOrg-server.test.ts // // The provenance step on a temp video dir, against a scripted archive.org. // Every name here is invented. const ITEM = "example-item"; const FILE = "First Upload-AbC123xyz_9.mp4"; const URL_FILE = archiveOrgDetailsUrl({ identifier: ITEM, file: FILE }); const ITEM_META = { metadata: { identifier: ITEM, title: "Example Channel Archive", creator: "Example Creator", collection: "community" }, files: [ { name: FILE, source: "original" }, { name: "First Upload-AbC123xyz_9.info.json", source: "original" }, { name: "Second-Def456uvw_8.mp4", source: "original" }, { name: `${ITEM}_archive.torrent`, source: "metadata" }, ], }; const INFO = { id: "AbC123xyz_9", extractor_key: "Youtube", title: "First Upload (original)", upload_date: "20230405", uploader: "Example Creator", }; function fakeClient(answers: Record, seen: string[]) { return new ArchiveOrgClient( { sleep: async () => {}, fetch: async (url) => { seen.push(url); if (!(url in answers)) return new Response("{}", { status: 404 }); return new Response(JSON.stringify(answers[url]), { status: 200 }); }, }, { minGapMs: 0 }, ); } async function videoDir(info: Record): Promise { const dir = await mkdtemp(path.join(os.tmpdir(), "archiveorg-prov-")); await writeFile(path.join(dir, "metadata.info.json"), JSON.stringify(info)); return dir; } test("writes the sidecar once and corrects the record, through the history", async () => { const seen: string[] = []; const client = fakeClient( { [`https://archive.org/metadata/${ITEM}`]: ITEM_META, [`https://archive.org/download/${ITEM}/First%20Upload-AbC123xyz_9.info.json`]: INFO, }, seen, ); const dir = await videoDir({ id: `${ITEM}/${FILE}`, extractor_key: "ArchiveOrg", title: "Example Channel Archive", webpage_url: `https://archive.org/details/${ITEM}`, upload_date: "20240304", }); const lines: string[] = []; const prov = await ensureArchiveOrgProvenance({ videoDir: dir, videoUrl: URL_FILE, client, onLog: (l) => lines.push(l) }); assert.equal(prov?.mirror?.id, "AbC123xyz_9"); assert.deepEqual(await loadArchiveOrgProvenance(dir), prov); const info = JSON.parse(await readFile(path.join(dir, "metadata.info.json"), "utf8")); assert.equal(info.webpage_url, URL_FILE); assert.equal(info.title, "First Upload (original)"); assert.equal(info.upload_date, "20230405"); assert.equal(info.id, `${ITEM}/${FILE}`); const history = await loadMetadataHistory(dir); assert.equal(history?.entries.at(-1)?.by, "archiveorg-provenance"); assert.equal(seen.length, 2); // A second run (the next download of the record) asks nothing. await ensureArchiveOrgProvenance({ videoDir: dir, videoUrl: URL_FILE, client }); assert.equal(seen.length, 2); }); test("archive.org unreachable: no sidecar, but the page is still the file's", async () => { const seen: string[] = []; const client = fakeClient({}, seen); const dir = await videoDir({ id: `${ITEM}/${FILE}`, webpage_url: `https://archive.org/details/${ITEM}`, title: "Item" }); const lines: string[] = []; const prov = await ensureArchiveOrgProvenance({ videoDir: dir, videoUrl: URL_FILE, client, onLog: (l) => lines.push(l) }); assert.equal(prov, null); assert.equal(await loadArchiveOrgProvenance(dir), null); const info = JSON.parse(await readFile(path.join(dir, "metadata.info.json"), "utf8")); assert.equal(info.webpage_url, URL_FILE); assert.equal(info.title, "Item"); assert.ok(lines.some((l) => /provenance not fetched/.test(l))); }); test("a URL that is not archive.org is left alone", async () => { const dir = await videoDir({ id: "x", webpage_url: "https://example.com/x" }); assert.equal(await ensureArchiveOrgProvenance({ videoDir: dir, videoUrl: "https://example.com/x" }), null); });