import { test } from "node:test"; import assert from "node:assert/strict"; import { bdecode, BencodeError } from "./bencode"; import { DEFAULT_ARCHIVE_ORG_FETCH_SETTINGS, aria2cArgs, findTorrentFile, parseAria2cSize, parseAria2cStatusLine, parseTorrent, torrentFilePieces, torrentFileSegments, torrentProgressLine, } from "./archiveOrgTorrent"; // Run with: // pnpm --filter yt-dlp-transcript-common exec tsx --test lib/archiveOrgTorrent.test.ts // // A synthetic torrent, built here by a ten-line encoder, shaped like // archive.org's `_archive.torrent`: name = identifier, a files // list, archive.org web seeds. Every name is invented. type Enc = number | string | Buffer | Enc[] | { [k: string]: Enc }; function benc(v: Enc): Buffer { if (typeof v === "number") return Buffer.from(`i${v}e`); if (typeof v === "string") return benc(Buffer.from(v, "utf8")); if (Buffer.isBuffer(v)) return Buffer.concat([Buffer.from(`${v.length}:`), v]); if (Array.isArray(v)) return Buffer.concat([Buffer.from("l"), ...v.map(benc), Buffer.from("e")]); const keys = Object.keys(v).sort(); return Buffer.concat([Buffer.from("d"), ...keys.flatMap((k) => [benc(k), benc(v[k])]), Buffer.from("e")]); } const PIECE = 16384; const FILES = [ { path: ["example-item_meta.xml"], length: 1000 }, { path: ["Clip One.mp4"], length: 40000 }, { path: ["sub dir", "Clip Two [AbC123xyz_9].mp4"], length: 50000 }, { path: ["Clip One.info.json"], length: 2000 }, ]; const total = FILES.reduce((n, f) => n + f.length, 0); const TORRENT = benc({ announce: "http://bt1.archive.org:6969/announce", "announce-list": [["http://bt1.archive.org:6969/announce"], ["http://bt2.archive.org:6969/announce"]], "url-list": ["https://archive.org/download/", "http://ia000000.us.archive.org/0/items/"], info: { name: "example-item", "piece length": PIECE, pieces: Buffer.alloc(20 * Math.ceil(total / PIECE)), files: FILES, }, }); test("bencode: integers, strings, lists, dictionaries", () => { assert.equal(bdecode(Buffer.from("i-42e")), -42); assert.deepEqual(bdecode(Buffer.from("4:spam")), Buffer.from("spam")); const v = bdecode(Buffer.from("d3:bari1e3:fool1:a1:bee")) as Record; assert.equal(v.bar, 1); assert.deepEqual((v.foo as Buffer[]).map((b) => b.toString()), ["a", "b"]); }); test("bencode: malformed input throws, naming the offset", () => { for (const bad of ["i12", "5:abc", "l1:a", "d1:ai1e", "x", "i1ei2e", "i1x2e"]) { assert.throws(() => bdecode(Buffer.from(bad)), BencodeError, bad); } }); test("a torrent: name, pieces, files with 1-based indexes and offsets, web seeds, trackers", () => { const t = parseTorrent(TORRENT)!; assert.equal(t.name, "example-item"); assert.equal(t.pieceLength, PIECE); assert.equal(t.pieceCount, Math.ceil(total / PIECE)); assert.equal(t.multiFile, true); assert.deepEqual( t.files.map((f) => [f.index, f.path, f.offset]), [ [1, "example-item_meta.xml", 0], [2, "Clip One.mp4", 1000], [3, "sub dir/Clip Two [AbC123xyz_9].mp4", 41000], [4, "Clip One.info.json", 91000], ], ); assert.equal(t.webSeeds.length, 2); assert.deepEqual(t.trackers, ["http://bt1.archive.org:6969/announce", "http://bt2.archive.org:6969/announce"]); }); test("the file's index is found by its exact path in the item; absent is null", () => { const t = parseTorrent(TORRENT)!; assert.equal(findTorrentFile(t, "sub dir/Clip Two [AbC123xyz_9].mp4")?.index, 3); assert.equal(findTorrentFile(t, "Clip One.mp4")?.index, 2); assert.equal(findTorrentFile(t, "clip one.mp4"), null); assert.equal(findTorrentFile(t, "Added Later.mp4"), null); const two = findTorrentFile(t, "sub dir/Clip Two [AbC123xyz_9].mp4")!; assert.deepEqual(torrentFileSegments(t, two), ["example-item", "sub dir", "Clip Two [AbC123xyz_9].mp4"]); // 41000..90999 spans pieces 2..5 of 16 KiB. assert.deepEqual(torrentFilePieces(t, two), { first: 2, last: 5, count: 4 }); }); test("path.utf-8 wins over path; not a torrent is null", () => { const t = parseTorrent( benc({ info: { name: "x", "piece length": PIECE, pieces: Buffer.alloc(20), files: [{ path: ["mangled"], "path.utf-8": ["Café.mp3"], length: 10 }], }, }), )!; assert.equal(t.files[0].path, "Café.mp3"); assert.equal(parseTorrent(Buffer.from("not bencode")), null); assert.equal(parseTorrent(benc({ announce: "x" })), null); }); test("a single-file torrent writes /", () => { const t = parseTorrent(benc({ info: { name: "solo.mp3", "piece length": PIECE, pieces: Buffer.alloc(20), length: 99 } }))!; assert.equal(t.multiFile, false); assert.deepEqual(torrentFileSegments(t, t.files[0]), ["solo.mp3"]); }); test("aria2c arguments: one file, seeding bounds, politeness, the hook, no word-splitting", () => { const args = aria2cArgs({ torrentPath: "/s/item.torrent", dir: "/s/dir with space", fileIndex: 3, settings: { ...DEFAULT_ARCHIVE_ORG_FETCH_SETTINGS, maxUploadKiBps: 200 }, onCompleteHook: "/s/on-complete.sh", userAgent: "Example UA (+https://example.org)", parentPid: 1234, }); assert.ok(args.includes("--dir=/s/dir with space")); assert.ok(args.includes("--select-file=3")); assert.ok(args.includes("--seed-time=10")); assert.ok(args.includes("--seed-ratio=1.0")); assert.ok(args.includes("--bt-max-peers=30")); assert.ok(args.includes("--max-overall-download-limit=0")); assert.ok(args.includes("--max-overall-upload-limit=200K")); assert.ok(args.includes("--bt-remove-unselected-file=true")); assert.ok(args.includes("--follow-torrent=mem")); assert.ok(args.includes("--file-allocation=none")); assert.ok(args.includes("--max-connection-per-server=1")); assert.ok(args.includes("--on-bt-download-complete=/s/on-complete.sh")); assert.ok(args.includes("--user-agent=Example UA (+https://example.org)")); assert.ok(args.includes("--stop-with-process=1234")); assert.equal(args.at(-1), "--torrent-file=/s/item.torrent"); assert.ok(args.every((a) => a.startsWith("--"))); // No seeding when the operator says so. const none = aria2cArgs({ torrentPath: "t", dir: "d", fileIndex: 1, settings: { ...DEFAULT_ARCHIVE_ORG_FETCH_SETTINGS, seedMinutes: 0, seedRatio: 0 }, onCompleteHook: "h", userAgent: "u", }); assert.ok(none.includes("--seed-time=0")); // A fresh run does not hash-check pieces that are not there; a resume does. assert.ok(!args.includes("--check-integrity=true")); assert.ok( aria2cArgs({ torrentPath: "t", dir: "d", fileIndex: 1, settings: DEFAULT_ARCHIVE_ORG_FETCH_SETTINGS, onCompleteHook: "h", userAgent: "u", resume: true }) .includes("--check-integrity=true"), ); assert.ok(!none.some((a) => a.startsWith("--stop-with-process"))); }); test("aria2c progress lines, as 1.37 prints them", () => { assert.equal(parseAria2cSize("16KiB"), 16384); assert.equal(parseAria2cSize("2.8MiB"), Math.round(2.8 * 1024 * 1024)); assert.equal(parseAria2cSize("0B"), 0); assert.equal(parseAria2cSize("lots"), null); assert.deepEqual(parseAria2cStatusLine("[#09cba8 608KiB/2.8MiB(20%) CN:1 DL:302KiB ETA:7s]"), { seeding: false, completedBytes: 608 * 1024, totalBytes: Math.round(2.8 * 1024 * 1024), percent: 20, connections: 1, }); const bt = parseAria2cStatusLine("[#2089b0 1.1MiB/33MiB(3%) CN:5 SD:2 DL:115KiB UL:4KiB(16KiB) ETA:4m48s]"); assert.equal(bt?.seeding, false); assert.equal(bt && !bt.seeding ? bt.seeders : -1, 2); assert.equal(bt?.connections, 5); const seed = parseAria2cStatusLine("[#2089b0 SEED(0.4) CN:2 SD:0 UL:12KiB(4.9MiB)]"); assert.equal(seed?.seeding, true); assert.equal(seed && seed.seeding ? seed.ratio : -1, 0.4); assert.equal(parseAria2cStatusLine("FILE: out/blob.bin"), null); assert.equal(parseAria2cStatusLine("10/05 15:46:39 [NOTICE] Download complete: out/blob.bin"), null); }); test("the human line: pieces of the file's own span, peers, web seed", () => { const line = torrentProgressLine({ file: "Clip One.mp4", status: { seeding: false, completedBytes: 50, totalBytes: 100, percent: 50, connections: 4, seeders: 1 }, pieces: 9, webSeed: true, }); assert.equal(line, "torrent: Clip One.mp4 (4 of 9 pieces, peers 4 (1 seeding), web seed yes)"); });