import { test } from "node:test"; import assert from "node:assert/strict"; import { mkdtemp, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { buildWaybackProvenance, coerceWaybackProvenance, jwPlayerMediaId, parseWaybackUrl, waybackCaptureDate, waybackCitationLinks, } from "./wayback"; import { ensureWaybackProvenance, loadWaybackProvenance } from "./wayback-server"; import { extractVideoId } from "./videoId"; import { platformMomentUrl } from "./momentUrl"; import { platformFromMetadata, summarize } from "./transcripts-server"; // Run with: // pnpm --filter yt-dlp-transcript-common exec tsx --test lib/wayback.test.ts // // Every id, account and token here is invented. const YT = "Abc123def45"; const JW = "Qw3rTy12"; const ARCHIVED_PAGE = `https://web.archive.org/web/20210102030405/https://www.youtube.com/watch?v=${YT}`; const JW_FILE = `https://videos-fms.jwpsrv.com/content/conversions/AcCt1234/videos/${JW}-12345678.mp4?token=0_abc_0xdef`; const ARCHIVED_FILE = `https://web.archive.org/web/20200102030405id_/${JW_FILE}`; test("parseWaybackUrl: timestamp, modifier and the original with its own query", () => { assert.deepEqual(parseWaybackUrl(ARCHIVED_PAGE), { captureTs: "20210102030405", modifier: "", originalUrl: `https://www.youtube.com/watch?v=${YT}`, }); assert.deepEqual(parseWaybackUrl(ARCHIVED_FILE), { captureTs: "20200102030405", modifier: "id_", originalUrl: JW_FILE, }); // A short timestamp, another modifier, no scheme, a collapsed scheme, an // encoded original, the older path without /web/. assert.equal(parseWaybackUrl("https://web.archive.org/web/2019im_/example.com/a.png")?.originalUrl, "http://example.com/a.png"); assert.equal(parseWaybackUrl("https://web.archive.org/web/2019/https:/example.com/x")?.originalUrl, "https://example.com/x"); assert.equal( parseWaybackUrl("https://web.archive.org/web/2019/https%3A%2F%2Fexample.com%2Fx")?.originalUrl, "https://example.com/x", ); assert.equal(parseWaybackUrl("http://wayback.archive.org/20190101000000/http://example.com/")?.captureTs, "20190101000000"); // Not captures. assert.equal(parseWaybackUrl("https://web.archive.org/web/*/example.com"), null); assert.equal(parseWaybackUrl("https://archive.org/details/some-item"), null); assert.equal(parseWaybackUrl(`https://www.youtube.com/watch?v=${YT}`), null); assert.equal(parseWaybackUrl("not a url"), null); }); test("jwPlayerMediaId: the media id across hosts and renditions", () => { assert.equal(jwPlayerMediaId(JW_FILE), JW); assert.equal(jwPlayerMediaId(`https://cdn.jwplayer.com/videos/${JW}-AbCdEf12.mp4`), JW); assert.equal(jwPlayerMediaId(`https://content.jwplatform.com/videos/${JW}.mp4`), JW); assert.equal(jwPlayerMediaId(`https://cdn.jwplayer.com/manifests/${JW}.m3u8`), JW); assert.equal(jwPlayerMediaId(`https://cdn.jwplayer.com/v2/media/${JW}`), JW); assert.equal(jwPlayerMediaId("https://cdn.jwplayer.com/libraries/AbCd1234.js"), null); assert.equal(jwPlayerMediaId(`https://example.com/videos/${JW}-1.mp4`), null); }); test("extractVideoId unwraps a capture into what it is a capture of", () => { assert.equal(extractVideoId(ARCHIVED_PAGE), YT); assert.equal(extractVideoId(`https://web.archive.org/web/2021if_/https://youtu.be/${YT}`), YT); assert.equal(extractVideoId(ARCHIVED_FILE), JW); assert.equal(extractVideoId(JW_FILE), JW); // Unwrapped once: a capture of a capture names no id. assert.equal(extractVideoId(`https://web.archive.org/web/2021/${ARCHIVED_PAGE}`), null); // A Wayback page that is not a capture names no record. assert.equal(extractVideoId("https://web.archive.org/web/*/example.com"), null); // A capture of a page the app does not know: the original's last segment. assert.equal(extractVideoId("https://web.archive.org/web/2021/https://example.com/media/clip-7.mp4"), "clip-7.mp4"); // Nothing else moved. assert.equal(extractVideoId(`https://www.youtube.com/watch?v=${YT}`), YT); assert.equal(extractVideoId("https://example.com/feed/episode-1.mp3"), "episode-1.mp3"); }); test("buildWaybackProvenance: an archived YouTube page's original is its watch URL", () => { assert.deepEqual(buildWaybackProvenance(`https://web.archive.org/web/20210102id_/https://m.youtube.com/watch?v=${YT}&feature=share`), { originalUrl: `https://www.youtube.com/watch?v=${YT}`, captureTs: "20210102", waybackUrl: `https://web.archive.org/web/20210102/https://m.youtube.com/watch?v=${YT}&feature=share`, rawUrl: `https://web.archive.org/web/20210102id_/https://m.youtube.com/watch?v=${YT}&feature=share`, }); const file = buildWaybackProvenance(ARCHIVED_FILE)!; assert.equal(file.originalUrl, JW_FILE); assert.equal(file.waybackUrl, `https://web.archive.org/web/20200102030405/${JW_FILE}`); assert.equal(file.rawUrl, ARCHIVED_FILE); assert.equal(buildWaybackProvenance(JW_FILE), null); assert.deepEqual(coerceWaybackProvenance(JSON.parse(JSON.stringify(file))), file); assert.equal(coerceWaybackProvenance({ ...file, captureTs: "yesterday" }), null); assert.equal(coerceWaybackProvenance([]), null); }); test("waybackCaptureDate and the citation links", () => { assert.equal(waybackCaptureDate("20210102030405"), "2021-01-02"); assert.equal(waybackCaptureDate("2021"), "2021"); assert.equal(waybackCaptureDate("202101"), "2021-01"); const prov = buildWaybackProvenance(ARCHIVED_PAGE)!; const links = waybackCitationLinks(prov, { originalMomentUrl: platformMomentUrl(prov.originalUrl, null, 90), }); assert.deepEqual(links.original, { label: "Original (may be gone)", url: `https://www.youtube.com/watch?v=${YT}&t=90s`, }); assert.deepEqual(links.copy, { label: "Wayback Machine copy, 2021-01-02", url: `https://web.archive.org/web/20210102030405/https://www.youtube.com/watch?v=${YT}`, }); }); test("platformMomentUrl: a capture is the page that plays, never given a time param", () => { assert.equal(platformMomentUrl(ARCHIVED_PAGE, "youtube", 125), ARCHIVED_PAGE); assert.equal(platformMomentUrl(`https://www.youtube.com/watch?v=${YT}`, "youtube", 125), `https://www.youtube.com/watch?v=${YT}&t=125s`); }); test("platformFromMetadata: YoutubeWebArchive is the original's platform, YouTube", () => { assert.equal(platformFromMetadata({ extractor_key: "YoutubeWebArchive", extractor: "web.archive:youtube", webpage_url: ARCHIVED_PAGE }), "youtube"); assert.equal(platformFromMetadata({ extractor: "web.archive:youtube", webpage_url: ARCHIVED_PAGE }), "youtube"); }); test("summarize: a capture's id is its dir's name, not yt-dlp's file-name id", () => { const s = summarize("demo", `${JW}`, { id: `${JW}-12345678`, title: `${JW}-12345678`, extractor_key: "Generic", webpage_url: ARCHIVED_FILE, }); assert.equal(s.id, JW); assert.equal(s.slug, `demo/${JW}`); const page = summarize("demo", YT, { id: YT, title: "A title", extractor_key: "YoutubeWebArchive", webpage_url: ARCHIVED_PAGE }); assert.equal(page.id, YT); assert.equal(page.platform, "youtube"); assert.equal(page.webpageUrl, ARCHIVED_PAGE); }); test("ensureWaybackProvenance writes the sidecar once, and nothing for a non-capture", async () => { const dir = await mkdtemp(path.join(tmpdir(), "wayback-sidecar-")); assert.deepEqual(await ensureWaybackProvenance(dir, JW_FILE), { provenance: null, written: false }); assert.equal(await loadWaybackProvenance(dir), null); const dry = await ensureWaybackProvenance(dir, ARCHIVED_PAGE, { dryRun: true }); assert.equal(dry.written, true); assert.equal(await loadWaybackProvenance(dir), null); const first = await ensureWaybackProvenance(dir, ARCHIVED_PAGE); assert.equal(first.written, true); const onDisk = JSON.parse(await readFile(path.join(dir, "wayback.json"), "utf8")); assert.deepEqual(onDisk, buildWaybackProvenance(ARCHIVED_PAGE)); assert.equal((await ensureWaybackProvenance(dir, ARCHIVED_PAGE)).written, false); // A sidecar for another capture is rewritten. await writeFile(path.join(dir, "wayback.json"), JSON.stringify(buildWaybackProvenance(ARCHIVED_FILE))); assert.equal((await ensureWaybackProvenance(dir, ARCHIVED_PAGE)).written, true); assert.deepEqual(await loadWaybackProvenance(dir), buildWaybackProvenance(ARCHIVED_PAGE)); });