// archive.org OVER BITTORRENT — the pure half: what an item's torrent holds, // which of its files is the one wanted, the aria2c command line that fetches // only that file, and what aria2c's progress lines say. // // WHY A TORRENT (the operator: "use torrents when possible to be extra polite // to archive.org"). archive.org derives `_archive.torrent` for every // item, and lists archive.org itself as a WEB SEED in it (`url-list`): a // torrent client takes pieces from any peer that has them and from archive.org // for the rest, so what other peers can give never touches archive.org's disks, // and the file is then SEEDED back for a while. One file of a 160-file item is // fetched alone (`--select-file`). // // The process side — spawning aria2c, watching it stall, seeding, killing it // by its process group — is lib/archiveOrgTorrent-server.ts; the ladder that // falls back to a plain download is controller/archiveOrgDownload.ts. import { bdecode, bInt, bList, bString, isDict, type BDict, type BValue } from "./bencode"; // ─── The torrent ─── export type TorrentFileEntry = { // 1-based, in the torrent's own order: what aria2c's --select-file takes. index: number; // The path inside the torrent, `/`-joined (an archive.org torrent's name is // the identifier and its paths are the item's file names). path: string; length: number; // Byte offset of the file in the torrent's concatenated payload. offset: number; }; export type ParsedTorrent = { name: string; pieceLength: number; pieceCount: number; // A single-file torrent has one entry whose path is `name`. multiFile: boolean; files: TorrentFileEntry[]; // BEP 19 web seeds (`url-list`). archive.org's name archive.org itself. webSeeds: string[]; trackers: string[]; }; function pathOf(file: BDict): string | null { // `path.utf-8` (a BitComet extension archive.org also writes) wins over the // raw `path` when present. const parts = bList(file["path.utf-8"] ?? file.path) .map((p) => bString(p)) .filter((p): p is string => typeof p === "string"); return parts.length > 0 ? parts.join("/") : null; } function urlList(v: BValue | undefined): string[] { const one = bString(v); if (one) return [one]; return bList(v) .map((x) => bString(x)) .filter((x): x is string => !!x); } // Null for anything that is not a torrent with an info dictionary. export function parseTorrent(buf: Buffer): ParsedTorrent | null { let root: BValue; try { root = bdecode(buf); } catch { return null; } if (!isDict(root) || !isDict(root.info)) return null; const info = root.info; const name = bString(info["name.utf-8"] ?? info.name) ?? ""; const pieceLength = bInt(info["piece length"]) ?? 0; const pieces = info.pieces; const pieceCount = Buffer.isBuffer(pieces) ? Math.floor(pieces.length / 20) : 0; if (!name || pieceLength <= 0) return null; const files: TorrentFileEntry[] = []; let multiFile = false; if (Array.isArray(info.files)) { multiFile = true; let offset = 0; let index = 0; for (const f of info.files) { index++; if (!isDict(f)) continue; const length = bInt(f.length) ?? 0; const p = pathOf(f); if (p !== null) files.push({ index, path: p, length, offset }); offset += length; } } else { files.push({ index: 1, path: name, length: bInt(info.length) ?? 0, offset: 0 }); } const trackers = [ ...urlList(root.announce), ...bList(root["announce-list"]).flatMap((tier) => urlList(tier)), ]; return { name, pieceLength, pieceCount, multiFile, files, webSeeds: urlList(root["url-list"]), trackers: [...new Set(trackers)], }; } // The torrent's entry for one file of the item, or null when the torrent does // not carry it (archive.org regenerates an item's torrent when the item // changes, but a file added since, or one archive.org leaves out, is not in it). export function findTorrentFile(t: ParsedTorrent, file: string): TorrentFileEntry | null { return t.files.find((f) => f.path === file) ?? null; } // Where aria2c writes the file under its --dir: `//` for a // multi-file torrent, `/` for a single-file one. As path segments, // for the caller to join. export function torrentFileSegments(t: ParsedTorrent, f: TorrentFileEntry): string[] { return t.multiFile ? [t.name, ...f.path.split("/")] : [t.name]; } // The pieces the file spans: a file shares its first and last piece with its // neighbours, which is why aria2c writes a sliver of each (and // --bt-remove-unselected-file deletes them when the file is complete). export function torrentFilePieces( t: ParsedTorrent, f: TorrentFileEntry, ): { first: number; last: number; count: number } { const first = Math.floor(f.offset / t.pieceLength); const last = Math.max(first, Math.floor((f.offset + Math.max(0, f.length - 1)) / t.pieceLength)); return { first, last, count: last - first + 1 }; } // ─── Politeness ─── // settings.json `archiveOrg` (lib/settingsSchema.ts documents each key). export type ArchiveOrgFetchSettings = { // Fetch over BitTorrent when the item's torrent carries the file and aria2c // is installed. False = always the plain download. torrent: boolean; // Seed the file for this long after it is complete (0 = do not seed). seedMinutes: number; // ...or until this much has been uploaded relative to the file's size, // whichever comes first (0 = no ratio limit; the time alone ends it). seedRatio: number; // No progress for this long while downloading: aria2c is stopped and the // file is downloaded directly. stallMinutes: number; maxPeers: number; // 0 = unlimited. maxDownloadKiBps: number; maxUploadKiBps: number; }; export const DEFAULT_ARCHIVE_ORG_FETCH_SETTINGS: ArchiveOrgFetchSettings = { torrent: true, seedMinutes: 10, seedRatio: 1, stallMinutes: 5, maxPeers: 30, maxDownloadKiBps: 0, maxUploadKiBps: 0, }; // ─── The aria2c command line ─── export type Aria2cSpec = { torrentPath: string; // Where aria2c writes (the record's staging dir, on the corpus disk). dir: string; fileIndex: number; settings: ArchiveOrgFetchSettings; // Called by aria2c when the selected file is complete, BEFORE seeding (its // documented `--on-bt-download-complete` contract): how the runner knows the // download is over while aria2c keeps running to seed. onCompleteHook: string; userAgent: string; // aria2c stops on its own if this process dies, so a crashed editor never // leaves a seeder behind. parentPid?: number; summaryIntervalSec?: number; // Part of the file is already in --dir (a cancelled run's partial, or a run // killed while seeding): hash-check it and carry on, rather than refusing it // or fetching it again. Off on a fresh run, where aria2c's check of pieces // that are not there yet prints a "Checksum error" that means nothing. resume?: boolean; }; // Every flag is spelled `--name=value`, one argv entry each, so nothing here // is ever word-split. export function aria2cArgs(spec: Aria2cSpec): string[] { const s = spec.settings; const args = [ `--dir=${spec.dir}`, `--select-file=${spec.fileIndex}`, // The slivers of neighbouring files a shared piece leaves are removed when // the selected file is complete. "--bt-remove-unselected-file=true", `--seed-time=${Math.max(0, s.seedMinutes)}`, `--seed-ratio=${Math.max(0, s.seedRatio).toFixed(1)}`, `--bt-max-peers=${Math.max(1, Math.floor(s.maxPeers))}`, `--max-overall-download-limit=${s.maxDownloadKiBps > 0 ? `${Math.floor(s.maxDownloadKiBps)}K` : "0"}`, `--max-overall-upload-limit=${s.maxUploadKiBps > 0 ? `${Math.floor(s.maxUploadKiBps)}K` : "0"}`, // One connection to each web seed: archive.org is the web seed. "--max-connection-per-server=1", `--user-agent=${spec.userAgent}`, "--follow-torrent=mem", "--file-allocation=none", // A re-run resumes: the partial and its .aria2 control file are kept on a // cancel (see `resume`). "--continue=true", ...(spec.resume ? ["--check-integrity=true"] : []), "--auto-file-renaming=false", "--bt-save-metadata=false", `--on-bt-download-complete=${spec.onCompleteHook}`, `--summary-interval=${spec.summaryIntervalSec ?? 10}`, "--show-console-readout=false", "--console-log-level=notice", "--enable-color=false", ...(spec.parentPid ? [`--stop-with-process=${spec.parentPid}`] : []), `--torrent-file=${spec.torrentPath}`, ]; return args; } // ─── aria2c's progress lines ─── // The bracket line of a progress summary: // [#09cba8 608KiB/2.8MiB(20%) CN:1 SD:3 DL:302KiB ETA:7s] // [#09cba8 SEED(0.4) CN:2 SD:0 UL:12KiB(4.9MiB)] export type Aria2cStatus = | { seeding: false; completedBytes: number; totalBytes: number; percent: number; connections: number; seeders?: number; } | { seeding: true; ratio: number; connections: number; uploadedBytes?: number }; const UNITS: Record = { B: 1, KiB: 1024, MiB: 1024 ** 2, GiB: 1024 ** 3, TiB: 1024 ** 4 }; export function parseAria2cSize(s: string): number | null { const m = /^([\d.]+)(B|KiB|MiB|GiB|TiB)$/.exec(s.trim()); if (!m) return null; const n = Number(m[1]); return Number.isFinite(n) ? Math.round(n * UNITS[m[2]]) : null; } export function parseAria2cStatusLine(line: string): Aria2cStatus | null { const m = /^\s*\[#[0-9a-f]+ (.*)\]\s*$/.exec(line); if (!m) return null; const body = m[1]; const field = (k: string) => { const f = new RegExp(`(?:^| )${k}:([^ \\]]+)`).exec(body); return f ? f[1] : undefined; }; const cn = Number(field("CN") ?? 0) || 0; const seed = /^SEED\(([\d.]+|inf)\)/.exec(body); if (seed) { const ul = /UL:[^(]*\(([^)]+)\)/.exec(body); const uploaded = ul ? parseAria2cSize(ul[1]) : null; return { seeding: true, ratio: seed[1] === "inf" ? Infinity : Number(seed[1]), connections: cn, ...(uploaded !== null ? { uploadedBytes: uploaded } : {}), }; } const prog = /^([\d.]+(?:B|KiB|MiB|GiB|TiB))\/([\d.]+(?:B|KiB|MiB|GiB|TiB))\((\d+)%\)/.exec(body); if (!prog) return null; const done = parseAria2cSize(prog[1]); const total = parseAria2cSize(prog[2]); if (done === null || total === null) return null; const sd = field("SD"); return { seeding: false, completedBytes: done, totalBytes: total, percent: Number(prog[3]), connections: cn, ...(sd !== undefined && Number.isFinite(Number(sd)) ? { seeders: Number(sd) } : {}), }; } // The line a human follows: "torrent: (n of m pieces, peers p, web // seed yes)". The piece count is the file's own span; n is read off the bytes // aria2c reports, so it is the pieces' worth done, not a bitfield. export function torrentProgressLine(opts: { file: string; status: Extract; pieces: number; webSeed: boolean; }): string { const { status, pieces } = opts; const frac = status.totalBytes > 0 ? status.completedBytes / status.totalBytes : 0; const done = Math.min(pieces, Math.floor(frac * pieces)); return ( `torrent: ${opts.file} (${done} of ${pieces} pieces, ` + `peers ${status.connections}${status.seeders !== undefined ? ` (${status.seeders} seeding)` : ""}, ` + `web seed ${opts.webSeed ? "yes" : "no"})` ); }