import { test } from "node:test"; import assert from "node:assert/strict"; import { mkdtemp, readFile, rm, writeFile, stat } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { ARCHIVE_ORG_USER_AGENT, ArchiveOrgClient, ArchiveOrgRequestError, parseRetryAfterMs, } from "./archiveOrgClient"; // Run with: // pnpm --filter yt-dlp-transcript-common exec tsx --test lib/archiveOrgClient.test.ts // // THE POLITENESS RULES, ON A FAKE CLOCK. No request leaves the process: the // fetch is a script of canned responses, the clock only moves when the client // sleeps (or when a "request" takes time), and every sleep is recorded. type Scripted = { status: number; body?: unknown; headers?: Record; takesMs?: number }; function harness(script: Scripted[], opts: ConstructorParameters[1] = {}) { let now = 1_000_000; const sleeps: number[] = []; const calls: { url: string; at: number; ua: string | null }[] = []; const client = new ArchiveOrgClient( { now: () => now, sleep: async (ms) => { sleeps.push(ms); now += ms; }, random: () => 0.5, // no jitter fetch: async (url, init) => { const headers = new Headers(init.headers); calls.push({ url, at: now, ua: headers.get("user-agent") }); const next = script.shift(); if (!next) throw new Error("script exhausted"); now += next.takesMs ?? 100; return new Response(next.body === undefined ? "{}" : JSON.stringify(next.body), { status: next.status, headers: next.headers, }); }, }, opts, ); return { client, sleeps, calls, advance: (ms: number) => (now += ms) }; } const META = { metadata: { identifier: "example-item", title: "Example" }, files: [{ name: "a.mp4", source: "original" }] }; test("an item's metadata is asked for once, identified, and cached", async () => { const h = harness([{ status: 200, body: META }]); const a = await h.client.itemMetadata("example-item"); const b = await h.client.itemMetadata("example-item"); assert.equal(a, b); assert.equal(h.calls.length, 1); assert.equal(h.calls[0].url, "https://archive.org/metadata/example-item"); assert.equal(h.calls[0].ua, ARCHIVE_ORG_USER_AGENT); assert.match(ARCHIVE_ORG_USER_AGENT, /\(\+https?:\/\//); }); test("the cache expires", async () => { const h = harness([{ status: 200, body: META }, { status: 200, body: META }], { cacheTtlMs: 1000 }); await h.client.itemMetadata("example-item"); h.advance(500); await h.client.itemMetadata("example-item"); assert.equal(h.calls.length, 1); h.advance(1000); await h.client.itemMetadata("example-item"); assert.equal(h.calls.length, 2); }); test("requests are one at a time with a gap between them", async () => { const h = harness( [ { status: 200, body: { n: 1 }, takesMs: 500 }, { status: 200, body: { n: 2 }, takesMs: 500 }, { status: 200, body: { n: 3 }, takesMs: 500 }, ], { minGapMs: 2000 }, ); // Started together; served in order, each after the previous ended + the gap. const results = await Promise.all([ h.client.getJson("https://archive.org/download/i/1.json"), h.client.getJson("https://archive.org/download/i/2.json"), h.client.getJson("https://archive.org/download/i/3.json"), ]); assert.deepEqual(results, [{ n: 1 }, { n: 2 }, { n: 3 }]); assert.equal(h.calls[1].at - h.calls[0].at, 2500); assert.equal(h.calls[2].at - h.calls[1].at, 2500); }); test("a 429 waits for Retry-After, then succeeds", async () => { const h = harness([ { status: 429, headers: { "retry-after": "30" } }, { status: 200, body: META }, ]); await h.client.itemMetadata("example-item"); assert.equal(h.calls.length, 2); assert.ok(h.sleeps.includes(30_000), `slept ${h.sleeps}`); }); test("a 503 without Retry-After backs off exponentially, and stops after maxAttempts", async () => { const h = harness( [{ status: 503 }, { status: 503 }, { status: 503 }, { status: 503 }, { status: 200, body: META }], { maxAttempts: 4, baseBackoffMs: 5000, minGapMs: 0 }, ); await assert.rejects(h.client.itemMetadata("example-item"), (err: unknown) => { assert.ok(err instanceof ArchiveOrgRequestError); assert.equal(err.rateLimited, true); assert.match(err.message, /after 4 attempts/); return true; }); assert.equal(h.calls.length, 4); // 5 s, 10 s, 20 s between the four attempts; none after the last. assert.deepEqual(h.sleeps, [5000, 10000, 20000]); }); test("a 404 is not retried; an empty answer is no item", async () => { const h = harness([{ status: 404 }]); await assert.rejects(h.client.getJson("https://archive.org/download/i/x.json"), /HTTP 404/); assert.equal(h.calls.length, 1); const h2 = harness([{ status: 200, body: {} }]); await assert.rejects(h2.client.itemMetadata("no-such-item"), /has no item "no-such-item"/); }); test("Retry-After as seconds or as an HTTP date", () => { assert.equal(parseRetryAfterMs("12", 0), 12_000); assert.equal(parseRetryAfterMs(new Date(60_000).toUTCString(), 0), 60_000); assert.equal(parseRetryAfterMs(null, 0), null); assert.equal(parseRetryAfterMs("soon", 0), null); }); // ─── A file's bytes (the fallback when a torrent cannot be used) ─── type FileStep = { status: number; body?: string; headers?: Record; expectRange?: string | null }; function fileHarness(steps: FileStep[]) { const sleeps: number[] = []; const ranges: (string | null)[] = []; const client = new ArchiveOrgClient( { sleep: async (ms) => { sleeps.push(ms); }, random: () => 0.5, fetch: async (url, init) => { const h = new Headers(init.headers); ranges.push(h.get("range")); assert.equal(h.get("user-agent"), ARCHIVE_ORG_USER_AGENT); assert.equal(url, "https://archive.org/download/example-item/Clip%20One.mp4"); const s = steps.shift(); if (!s) throw new Error("script exhausted"); return new Response(s.body ?? "", { status: s.status, headers: s.headers }); }, }, { minGapMs: 0, baseBackoffMs: 1000 }, ); return { client, sleeps, ranges }; } test("downloadFile: a partial on disk is resumed with a Range request and appended", async () => { const dir = await mkdtemp(path.join(os.tmpdir(), "aoc-dl-")); try { const dest = path.join(dir, "Clip One.mp4"); await writeFile(`${dest}.part`, "hello "); const { client, ranges } = fileHarness([ { status: 206, body: "world", headers: { "content-length": "5" } }, ]); const r = await client.downloadFile("example-item", "Clip One.mp4", dest, { expectedSize: 11 }); assert.equal(r.bytes, 11); assert.deepEqual(ranges, ["bytes=6-"]); assert.equal(await readFile(dest, "utf8"), "hello world"); await assert.rejects(stat(`${dest}.part`)); } finally { await rm(dir, { recursive: true, force: true }); } }); test("downloadFile: a 503 waits its Retry-After; a cut-off stream resumes where it stopped", async () => { const dir = await mkdtemp(path.join(os.tmpdir(), "aoc-dl-")); try { const dest = path.join(dir, "Clip One.mp4"); const { client, sleeps, ranges } = fileHarness([ { status: 503, headers: { "retry-after": "7" } }, // Says 11 bytes, sends 4: the stream was cut. { status: 200, body: "hell", headers: { "content-length": "11" } }, { status: 206, body: "o world", headers: { "content-length": "7" } }, ]); const r = await client.downloadFile("example-item", "Clip One.mp4", dest); assert.equal(r.bytes, 11); assert.deepEqual(ranges, [null, null, "bytes=4-"]); assert.equal(sleeps[0], 7000); assert.equal(await readFile(dest, "utf8"), "hello world"); } finally { await rm(dir, { recursive: true, force: true }); } }); test("downloadFile: a 404 is final; archive.org refusing to the end is a rate limit", async () => { const dir = await mkdtemp(path.join(os.tmpdir(), "aoc-dl-")); try { const dest = path.join(dir, "Clip One.mp4"); const gone = fileHarness([{ status: 404 }]); await assert.rejects( gone.client.downloadFile("example-item", "Clip One.mp4", dest), (e: unknown) => e instanceof ArchiveOrgRequestError && e.status === 404 && !e.rateLimited, ); const busy = fileHarness([{ status: 429 }, { status: 429 }, { status: 429 }, { status: 429 }]); await assert.rejects( busy.client.downloadFile("example-item", "Clip One.mp4", dest), (e: unknown) => e instanceof ArchiveOrgRequestError && e.rateLimited, ); } finally { await rm(dir, { recursive: true, force: true }); } }); test("itemTorrent: fetched once, then cached", async () => { let calls = 0; const client = new ArchiveOrgClient( { sleep: async () => {}, fetch: async (url) => { calls++; assert.equal(url, "https://archive.org/download/example-item/example-item_archive.torrent"); return new Response(new Uint8Array([100, 101]), { status: 200 }); }, }, { minGapMs: 0 }, ); assert.deepEqual([...(await client.itemTorrent("example-item"))], [100, 101]); await client.itemTorrent("example-item"); assert.equal(calls, 1); });