commit 57ae8c2a8abe38f01747471d9ff8c5892a526f5b
parent 6d9cb23d1cee35c03e999dc33224207f4570ae86
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 9 Oct 2026 13:24:58 -0400
common: prepare-playable — each saved container remuxed losslessly into a browser-playable mp4 (+faststart) or webm, one single-file torrent per copy, sha-keyed so a re-run is a no-op
D3 of release 21. lib/playableMedia.ts is the decision table (h264/vp9/av1 +
aac/opus/mp3 → mp4; vp9/vp8/av1 + opus/vorbis in webm → webm; anything else
listed, never re-encoded), first video + first audio only, metadata and
chapters dropped, +bitexact. create-torrent 6.1.3 + parse-torrent 11.0.24:
name <id>.<ext>, 256 KiB–1 MiB pieces, no urlList, no comment, no created
by, no private flag, fixed creation date, announce from a parameter.
Copies, torrents and playable.json under playable/<slug>/ beside the
saved-video store. Job kind prepare-playable (drainable, needsMedia, ingest,
one queue machine-wide), ops route + action, pnpm ops row, archilyzer media
playable.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
16 files changed, 1500 insertions(+), 1 deletion(-)
diff --git a/common/bin/archilyzer.ts b/common/bin/archilyzer.ts
@@ -498,6 +498,28 @@ export const COMMANDS: Command[] = [
});
},
},
+ {
+ path: ["media", "playable"],
+ usage:
+ "<slug> [--ids <a,b>] [--trackers <url,url>] [--root <dir>] [--dry-run] remux a channel's saved containers losslessly (-c copy) into browser-playable mp4 (+faststart) or webm, one single-file torrent each (no web seed; --trackers is the announce list, outside the infohash), under playable/ beside the saved-video store; keyed by the source's sha256, so a re-run is a no-op; offline",
+ flags: { ids: "string", trackers: "string", root: "string", "dry-run": "boolean" },
+ maxPositionals: 1,
+ run: async ({ positionals, flags }) => {
+ const [slug] = positionals;
+ if (!slug) {
+ console.error("media playable: which channel? Pass its slug.");
+ return 2;
+ }
+ return (await import("./media-playable")).main({
+ slug,
+ ...(typeof flags.ids === "string" ? { ids: flags.ids } : {}),
+ ...(typeof flags.trackers === "string" ? { trackers: flags.trackers } : {}),
+ ...(typeof flags.root === "string" ? { root: flags.root } : {}),
+ dryRun: flags["dry-run"] === true,
+ signal: interrupted(),
+ });
+ },
+ },
// The bins that parse their own flags, run as children with their argv
// verbatim (_spawnBin.ts says why). Each row is the whole integration.
script(["duplicates"], "duplicate-shorts.ts",
diff --git a/common/bin/media-playable.ts b/common/bin/media-playable.ts
@@ -0,0 +1,61 @@
+// `archilyzer media playable <slug> [--ids a,b] [--trackers u1,u2] [--root
+// <dir>] [--dry-run]` — a channel's saved containers remuxed losslessly into
+// browser-playable copies, with one torrent each, offline. The work and its
+// rules are controller/preparePlayable.ts; the editor runs the same function
+// as the `prepare-playable` job (`POST /api/ops/prepare-playable`).
+//
+// Disk-bound and long (a remux reads and writes every byte): run it where the
+// editor's own prepare-playable is not running.
+
+import path from "node:path";
+import { getPaths } from "../lib/paths";
+import { assertChannelMediaReachable } from "../lib/channelMedia";
+import { isValidChannelSlug, readChannelConfig } from "../controller/channels";
+import { preparePlayable } from "../controller/preparePlayable";
+
+const list = (s: string | undefined) =>
+ s === undefined ? undefined : s.split(",").map((x) => x.trim()).filter(Boolean);
+
+export async function main(opts: {
+ slug: string;
+ ids?: string;
+ trackers?: string;
+ root?: string;
+ dryRun: boolean;
+ signal?: AbortSignal;
+}): Promise<number> {
+ if (!isValidChannelSlug(opts.slug)) {
+ console.error(`media playable: "${opts.slug}" is not a channel slug`);
+ return 2;
+ }
+ if (opts.root !== undefined && !path.isAbsolute(opts.root)) {
+ console.error("media playable: --root must be an absolute path");
+ return 2;
+ }
+ try {
+ const paths = getPaths();
+ const channelConfig = await readChannelConfig(paths, opts.slug);
+ if (!channelConfig) {
+ console.error(`media playable: no channel "${opts.slug}"`);
+ return 2;
+ }
+ if (!opts.dryRun) await assertChannelMediaReachable(paths, opts.slug, channelConfig);
+ const ids = list(opts.ids);
+ const trackers = list(opts.trackers);
+ const result = await preparePlayable({
+ paths,
+ slug: opts.slug,
+ channelConfig,
+ ...(opts.root ? { root: opts.root } : {}),
+ ...(ids ? { ids } : {}),
+ ...(trackers ? { trackers } : {}),
+ ...(opts.dryRun ? { dryRun: true } : {}),
+ onLog: (line) => console.log(line.replace(/\n$/, "")),
+ ...(opts.signal ? { signal: opts.signal } : {}),
+ });
+ return result.failed.length > 0 || (result.stopped && result.stopped !== "drained") ? 1 : 0;
+ } catch (err) {
+ console.error(`media playable: ${(err as Error).message}`);
+ return 1;
+ }
+}
diff --git a/common/controller/preparePlayable.test.ts b/common/controller/preparePlayable.test.ts
@@ -0,0 +1,254 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { copyFile, mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { execa } from "execa";
+import parseTorrent from "parse-torrent";
+import type { Paths } from "../lib/paths";
+import type { ChannelConfig } from "../lib/channelConfig";
+import { loadSavedVideo } from "../lib/savedVideo-server";
+import {
+ loadPlayableManifest,
+ makePlayableTorrent,
+ preparePlayable,
+ type PreparePlayableOptions,
+} from "./preparePlayable";
+
+// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test controller/preparePlayable.test.ts
+//
+// Fixture media made here by ffmpeg (a few seconds of test pattern and tone),
+// in a temp dir. Skipped where ffmpeg/ffprobe are not installed.
+
+const SLUG = "demo";
+const have = async (bin: string) => (await execa(bin, ["-version"], { reject: false }).catch(() => null))?.exitCode === 0;
+const tools = (await have("ffmpeg")) && (await have("ffprobe"));
+
+async function make(file: string, vArgs: string[], aArgs: string[]): Promise<void> {
+ await execa("ffmpeg", [
+ "-nostdin", "-hide_banner", "-loglevel", "error", "-y",
+ "-f", "lavfi", "-i", "testsrc=d=2:s=96x64:r=10",
+ "-f", "lavfi", "-i", "sine=d=2:f=440",
+ ...vArgs, ...aArgs, "-shortest", file,
+ ]);
+}
+
+const sha = async (f: string) => createHash("sha256").update(await readFile(f)).digest("hex");
+
+// Top-level MP4 atoms in order — "moov" before "mdat" is a faststart file.
+async function topAtoms(file: string): Promise<string[]> {
+ const b = await readFile(file);
+ const out: string[] = [];
+ let at = 0;
+ while (at + 8 <= b.length) {
+ let size = b.readUInt32BE(at);
+ const type = b.toString("latin1", at + 4, at + 8);
+ if (size === 1) size = Number(b.readBigUInt64BE(at + 8));
+ if (size < 8) break;
+ out.push(type);
+ at += size;
+ }
+ return out;
+}
+
+async function videoFrameMd5(file: string): Promise<string> {
+ const r = await execa("ffmpeg", ["-nostdin", "-loglevel", "error", "-i", file, "-map", "0:V:0", "-c", "copy", "-f", "framemd5", "-"]);
+ // Drop the header comments (they name the container's timebase).
+ return String(r.stdout).split("\n").filter((l) => l && !l.startsWith("#")).map((l) => l.split(",").pop()).join("\n");
+}
+
+type Fixture = { dir: string; paths: Paths; root: string; opts: (o?: Partial<PreparePlayableOptions>) => PreparePlayableOptions; lines: string[] };
+
+async function withFixture(fn: (f: Fixture) => Promise<void>): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-playable-"));
+ const paths = {
+ transcriptsDir: path.join(dir, "corpus"),
+ channelsDir: path.join(dir, "corpus", "channels"),
+ savedVideosDir: path.join(dir, "media", "saved-videos"),
+ ffmpegBin: "ffmpeg",
+ ffprobeBin: "ffprobe",
+ } as Paths;
+ const root = path.join(dir, "media", "playable");
+ const lines: string[] = [];
+ const opts = (o: Partial<PreparePlayableOptions> = {}): PreparePlayableOptions => ({
+ paths,
+ slug: SLUG,
+ channelConfig: { name: "Demo", handling: "youtube", platform: "youtube" } as unknown as ChannelConfig,
+ onLog: (l) => lines.push(l),
+ deps: { now: () => new Date("2026-10-09T12:00:00Z") },
+ ...o,
+ });
+ try {
+ await fn({ dir, paths, root, opts, lines });
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+// One saved container per id, with a pointer (sha256 recorded or not).
+async function saved(paths: Paths, id: string, src: string, ext: string, withSha = true): Promise<string> {
+ const storeDir = path.join(paths.savedVideosDir, SLUG, id);
+ await mkdir(storeDir, { recursive: true });
+ const file = path.join(storeDir, `source-media.${ext}`);
+ await copyFile(src, file);
+ const vd = path.join(paths.channelsDir, SLUG, "data", id);
+ await mkdir(vd, { recursive: true });
+ await writeFile(
+ path.join(vd, "saved-video.json"),
+ JSON.stringify({
+ storedAt: "x",
+ dir: storeDir,
+ file: `source-media.${ext}`,
+ bytes: (await stat(file)).size,
+ keepReason: "pin",
+ ...(withSha ? { sha256: await sha(file) } : {}),
+ }),
+ );
+ return file;
+}
+
+test("remuxes each container losslessly, makes its torrent, and a re-run is a no-op", { skip: !tools && "ffmpeg/ffprobe not installed" }, async () => {
+ await withFixture(async ({ dir, paths, root, opts }) => {
+ const src = path.join(dir, "src");
+ await mkdir(src);
+ await make(path.join(src, "a.mp4"), ["-c:v", "libx264", "-preset", "ultrafast"], ["-c:a", "aac"]);
+ await make(path.join(src, "b.mkv"), ["-c:v", "libvpx-vp9", "-deadline", "realtime", "-cpu-used", "8"], ["-c:a", "libopus"]);
+ await make(path.join(src, "c.webm"), ["-c:v", "libvpx-vp9", "-deadline", "realtime", "-cpu-used", "8"], ["-c:a", "libopus"]);
+ await make(path.join(src, "d.mkv"), ["-c:v", "mpeg4"], ["-c:a", "aac"]);
+ // The source mp4 has its moov at the back (ffmpeg's default).
+ assert.ok((await topAtoms(path.join(src, "a.mp4"))).indexOf("moov") > (await topAtoms(path.join(src, "a.mp4"))).indexOf("mdat"));
+
+ const aSrc = await saved(paths, "aaaaaaaaaaa", path.join(src, "a.mp4"), "mp4");
+ await saved(paths, "bbbbbbbbbbb", path.join(src, "b.mkv"), "mkv", false);
+ await saved(paths, "ccccccccccc", path.join(src, "c.webm"), "webm");
+ await saved(paths, "ddddddddddd", path.join(src, "d.mkv"), "mkv");
+ await mkdir(path.join(paths.channelsDir, SLUG, "data", "eeeeeeeeeee"), { recursive: true }); // held, no container
+
+ let remuxes = 0;
+ const counting = (o: Partial<PreparePlayableOptions> = {}) =>
+ opts({
+ root,
+ ...o,
+ deps: {
+ now: () => new Date("2026-10-09T12:00:00Z"),
+ remux: async (s, d, args) => {
+ remuxes += 1;
+ await execa("ffmpeg", ["-nostdin", "-hide_banner", "-loglevel", "error", "-y", "-i", s, ...args, d]);
+ },
+ },
+ });
+
+ const r = await preparePlayable(counting());
+ assert.deepEqual(r.prepared, ["aaaaaaaaaaa", "bbbbbbbbbbb", "ccccccccccc"]);
+ assert.deepEqual(r.unplayable.map((u) => u.id), ["ddddddddddd"]);
+ assert.match(r.unplayable[0].reason, /mpeg4/);
+ assert.deepEqual(r.failed, []);
+ assert.equal(remuxes, 3);
+
+ const m = await loadPlayableManifest(root, SLUG);
+ assert.deepEqual(Object.keys(m.entries).sort(), ["aaaaaaaaaaa", "bbbbbbbbbbb", "ccccccccccc"]);
+ const a = m.entries.aaaaaaaaaaa;
+ assert.equal(a.file, "aaaaaaaaaaa/aaaaaaaaaaa.mp4");
+ assert.equal(a.mime, "video/mp4");
+ assert.equal(a.sourceSha256, await sha(aSrc));
+ assert.equal(m.entries.bbbbbbbbbbb.ext, "mp4", "vp9/opus in mkv → mp4");
+ assert.equal(m.entries.ccccccccccc.ext, "webm", "vp9/opus in webm stays webm");
+ assert.equal(m.entries.bbbbbbbbbbb.acodec, "opus");
+
+ // Faststart: the index is in front now.
+ const atoms = await topAtoms(path.join(root, SLUG, a.file));
+ assert.ok(atoms.indexOf("moov") < atoms.indexOf("mdat"), atoms.join(","));
+ // Lossless: every video packet is bit-identical.
+ assert.equal(await videoFrameMd5(path.join(root, SLUG, a.file)), await videoFrameMd5(aSrc));
+ // Bitexact: the same source remuxes to the same bytes (so to the same infohash).
+ const again1 = path.join(dir, "again.mp4");
+ const decision = (await import("../lib/playableMedia")).decidePlayable({ formatName: "", ext: "mp4", vcodec: "h264", acodec: "aac" });
+ assert.ok(decision.playable);
+ if (decision.playable) {
+ await execa("ffmpeg", ["-nostdin", "-loglevel", "error", "-y", "-i", aSrc, ...decision.args, again1]);
+ assert.equal(await sha(again1), await sha(path.join(root, SLUG, a.file)));
+ }
+
+ // The torrent: one file, named <id>.<ext>, the manifest's infohash, and
+ // nothing that names a person, a tool or a URL.
+ const tBuf = await readFile(path.join(root, SLUG, a.torrent));
+ const t = await parseTorrent(tBuf);
+ assert.equal(t.infoHash, a.infoHash);
+ assert.equal(t.name, "aaaaaaaaaaa.mp4");
+ assert.equal(t.length, a.bytes);
+ assert.deepEqual(t.urlList, []);
+ assert.deepEqual(t.announce, []);
+ assert.equal(t.createdBy, undefined);
+ assert.equal(t.comment, undefined);
+ assert.notEqual(t.private, true);
+ for (const k of ["created by", "comment", "url-list", "announce", "private"]) {
+ assert.equal(tBuf.includes(Buffer.from(`${k.length}:${k}`)), false, `no ${k}`);
+ }
+
+ // A pointer that had no hash got one.
+ assert.match((await loadSavedVideo(path.join(paths.channelsDir, SLUG, "data", "bbbbbbbbbbb")))?.sha256 ?? "", /^[0-9a-f]{64}$/);
+
+ // Again: nothing to do, nothing remuxed, the manifest unchanged.
+ const before = await readFile(path.join(root, SLUG, "playable.json"), "utf8");
+ const again = await preparePlayable(counting());
+ assert.deepEqual(again.unchanged.sort(), ["aaaaaaaaaaa", "bbbbbbbbbbb", "ccccccccccc"]);
+ assert.deepEqual(again.prepared, []);
+ assert.equal(remuxes, 3);
+ assert.equal(await readFile(path.join(root, SLUG, "playable.json"), "utf8"), before);
+ assert.deepEqual(again.noContainer, []);
+ });
+});
+
+test("dryRun decides and writes nothing", { skip: !tools && "ffmpeg/ffprobe not installed" }, async () => {
+ await withFixture(async ({ dir, paths, root, opts }) => {
+ await make(path.join(dir, "a.mp4"), ["-c:v", "libx264", "-preset", "ultrafast"], ["-c:a", "aac"]);
+ await saved(paths, "aaaaaaaaaaa", path.join(dir, "a.mp4"), "mp4", false);
+ const r = await preparePlayable(opts({ root, dryRun: true }));
+ assert.deepEqual(r.prepared, ["aaaaaaaaaaa"]);
+ await assert.rejects(stat(root));
+ assert.equal((await loadSavedVideo(path.join(paths.channelsDir, SLUG, "data", "aaaaaaaaaaa")))?.sha256, undefined);
+ });
+});
+
+test("the default root is playable/ beside the store's real directory; an absent drive refuses", async () => {
+ await withFixture(async ({ dir, paths, opts }) => {
+ await mkdir(paths.savedVideosDir, { recursive: true });
+ const r = await preparePlayable(opts({ ids: [] }));
+ assert.equal(r.root, path.join(dir, "media", "playable"));
+ await assert.rejects(
+ preparePlayable(opts({ root: path.join(dir, "not-mounted", "playable"), ids: [] })),
+ /is not there/,
+ );
+ });
+});
+
+test("torrent determinism: same bytes + piece length → same infohash, whatever the trackers or the path", async () => {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-torrent-"));
+ try {
+ const bytes = Buffer.from(Array.from({ length: 700_000 }, (_, i) => (i * 31) & 0xff));
+ const f1 = path.join(dir, "one.bin");
+ const f2 = path.join(dir, "elsewhere.bin");
+ await writeFile(f1, bytes);
+ await writeFile(f2, bytes);
+ const base = { name: "x.mp4", bytes: bytes.length };
+ const a = await makePlayableTorrent({ ...base, file: f1 });
+ const b = await makePlayableTorrent({ ...base, file: f2 });
+ const c = await makePlayableTorrent({ ...base, file: f1, trackers: ["wss://tracker.example/announce", "udp://t.example:6969"] });
+ const d = await makePlayableTorrent({ ...base, file: f1, pieceLength: 512 * 1024 });
+ assert.match(a.infoHash, /^[0-9a-f]{40}$/);
+ assert.equal(a.infoHash, b.infoHash, "the path does not matter");
+ assert.equal(a.infoHash, c.infoHash, "the trackers are outside the info dict");
+ assert.notEqual(a.infoHash, d.infoHash, "the piece length is inside it");
+ assert.equal(a.pieceLength, 256 * 1024);
+ assert.deepEqual(a.torrent, b.torrent, "fixed creation date: the same .torrent byte for byte");
+ const parsed = await parseTorrent(c.torrent);
+ assert.deepEqual(parsed.announce, ["wss://tracker.example/announce", "udp://t.example:6969"]);
+ // Different bytes, different hash.
+ bytes[0] ^= 1;
+ await writeFile(f2, bytes);
+ assert.notEqual((await makePlayableTorrent({ ...base, file: f2 })).infoHash, a.infoHash);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+});
diff --git a/common/controller/preparePlayable.ts b/common/controller/preparePlayable.ts
@@ -0,0 +1,352 @@
+// PLAYABLE COPIES AND THEIR TORRENTS (release 21 D3, the `prepare-playable`
+// job).
+//
+// For each of a channel's videos with a saved source container: a browser-
+// playable copy made by a LOSSLESS remux (lib/playableMedia.ts decides the
+// container and the ffmpeg arguments), and one single-file torrent of that
+// copy. Written under the playable root —
+//
+// <root>/<slug>/<id>/<id>.<ext> the copy
+// <root>/<slug>/<id>/<id>.torrent its torrent
+// <root>/<slug>/playable.json the manifest (lib/playableMedia.ts)
+//
+// — where <root> is, unless named, `playable/` BESIDE the saved-video store's
+// real directory (the store is on the archive drive; so are the copies).
+//
+// KEYED BY THE SOURCE'S SHA-256, so a re-run is a no-op: an entry whose
+// `sourceSha256` matches the pointer's and whose files are on disk at their
+// recorded size is left alone. A pointer with no hash (a download's, before
+// the backup step reached it) is hashed once and the hash recorded on the
+// pointer, as the backup step would.
+//
+// THE TORRENT. `create-torrent`, from the WebTorrent ecosystem: one file,
+// named `<id>.<ext>`; piece length scaled to the size (256 KiB–1 MiB); no web
+// seed (`urlList`), no comment, no `created by`, a fixed creation date — the
+// .torrent names nobody and no private URL. The announce list is the
+// `trackers` parameter (none by default): it is outside the info dict, so the
+// infohash depends only on the bytes and the piece length, and a site can
+// announce the same infohash to other trackers later without a re-hash.
+//
+// Each copy is written to a `.part` and renamed; the manifest is rewritten
+// after each video, so a cancelled or drained run resumes where it stopped.
+
+/// <reference path="../types/webtorrent-modules.d.ts" />
+import path from "node:path";
+import { createReadStream } from "node:fs";
+import { mkdir, readdir, readFile, realpath, rename, rm, stat, writeFile } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import { execa } from "execa";
+import createTorrent from "create-torrent";
+import parseTorrent from "parse-torrent";
+import type { Paths } from "../lib/paths";
+import type { ChannelConfig } from "../lib/channelConfig";
+import { savedVideoPath, savedVideoRoot } from "../lib/savedVideo";
+import { loadSavedVideo, updateSavedVideoChecksum } from "../lib/savedVideo-server";
+import { writeJsonAtomic } from "../lib/jsonFile-server";
+import { getFreeBytes } from "../lib/diskSpace";
+import { formatBytes } from "../lib/format";
+import {
+ PLAYABLE_MANIFEST_FILENAME,
+ decidePlayable,
+ parsePlayableManifest,
+ parseProbe,
+ pieceLengthFor,
+ type MediaProbe,
+ type PlayableEntry,
+ type PlayableManifest,
+} from "../lib/playableMedia";
+
+// The creation date every torrent carries: fixed, so the same copy makes the
+// same .torrent file byte for byte (the infohash never depended on it).
+export const PLAYABLE_TORRENT_CREATION_DATE = new Date("2026-01-01T00:00:00Z");
+
+export const PLAYABLE_DISK_MARGIN_BYTES = 2 * 1024 ** 3;
+
+export async function defaultPlayableRoot(
+ paths: Paths,
+ config: Pick<ChannelConfig, "savedVideosDir"> | null,
+): Promise<string> {
+ const store = savedVideoRoot(paths, config);
+ const real = await realpath(store).catch(() => store);
+ return path.join(path.dirname(real), "playable");
+}
+
+export function playableManifestPath(root: string, slug: string): string {
+ return path.join(root, slug, PLAYABLE_MANIFEST_FILENAME);
+}
+
+export async function loadPlayableManifest(root: string, slug: string): Promise<PlayableManifest> {
+ try {
+ return parsePlayableManifest(JSON.parse(await readFile(playableManifestPath(root, slug), "utf8")), slug);
+ } catch {
+ return parsePlayableManifest(null, slug);
+ }
+}
+
+// --- the torrent ------------------------------------------------------------
+
+export type TorrentMade = { torrent: Buffer; infoHash: string; pieceLength: number };
+
+// One single-file torrent of `file`, named `name`. Deterministic: the same
+// bytes and piece length give the same infohash, whatever `trackers` is.
+export async function makePlayableTorrent(opts: {
+ file: string;
+ name: string;
+ bytes: number;
+ trackers?: readonly string[];
+ pieceLength?: number;
+}): Promise<TorrentMade> {
+ const pieceLength = opts.pieceLength ?? pieceLengthFor(opts.bytes);
+ // What is NOT passed matters as much as what is. `createdBy`, `comment`
+ // and `urlList` absent: create-torrent then writes no such key. `private`
+ // absent: even `false` would put `private: 0` INSIDE the info dict and so
+ // change the infohash — the one thing that must not move when the torrent
+ // goes public later.
+ const torrent = await new Promise<Buffer>((resolve, reject) => {
+ createTorrent(
+ opts.file,
+ {
+ name: opts.name,
+ pieceLength,
+ creationDate: PLAYABLE_TORRENT_CREATION_DATE,
+ // A list, even an empty one, not create-torrent's default public
+ // trackers: the pilot announces only where it is told to. A fresh
+ // array each call (create-torrent appends to the one it is given).
+ announceList: opts.trackers && opts.trackers.length ? [[...opts.trackers]] : [],
+ },
+ (err, buf) => (err ? reject(err) : resolve(Buffer.from(buf))),
+ );
+ });
+ const parsed = await parseTorrent(torrent);
+ if (!parsed.infoHash) throw new Error(`no infohash in the torrent made for ${opts.name}`);
+ return { torrent, infoHash: parsed.infoHash, pieceLength };
+}
+
+// --- the run ----------------------------------------------------------------
+
+export type PreparePlayableOptions = {
+ paths: Paths;
+ slug: string;
+ channelConfig: ChannelConfig;
+ root?: string;
+ ids?: string[];
+ trackers?: string[];
+ dryRun?: boolean;
+ onLog: (line: string) => void;
+ signal?: AbortSignal;
+ drainSignal?: AbortSignal;
+ deps?: {
+ now?: () => Date;
+ probe?: (file: string) => Promise<MediaProbe>;
+ remux?: (src: string, dest: string, args: string[], signal?: AbortSignal) => Promise<void>;
+ };
+};
+
+export type PreparePlayableResult = {
+ root: string;
+ prepared: string[];
+ unchanged: string[];
+ unplayable: { id: string; reason: string }[];
+ noContainer: string[];
+ failed: { id: string; error: string }[];
+ stopped?: string;
+ dryRun: boolean;
+};
+
+async function sha256File(file: string, signal?: AbortSignal): Promise<string> {
+ const h = createHash("sha256");
+ for await (const chunk of createReadStream(file, { highWaterMark: 1 << 20, ...(signal ? { signal } : {}) })) {
+ h.update(chunk as Buffer);
+ }
+ return h.digest("hex");
+}
+
+async function ffprobe(paths: Paths, file: string, signal?: AbortSignal): Promise<MediaProbe> {
+ const r = await execa(
+ paths.ffprobeBin,
+ ["-v", "error", "-show_streams", "-show_format", "-of", "json", file],
+ { ...(signal ? { cancelSignal: signal } : {}) },
+ );
+ return parseProbe(JSON.parse(String(r.stdout)), path.extname(file));
+}
+
+async function ffmpegRemux(paths: Paths, src: string, dest: string, args: string[], signal?: AbortSignal): Promise<void> {
+ await execa(
+ paths.ffmpegBin,
+ ["-nostdin", "-hide_banner", "-loglevel", "error", "-y", "-i", src, ...args, dest],
+ { ...(signal ? { cancelSignal: signal } : {}) },
+ );
+}
+
+async function sizeOf(file: string): Promise<number | null> {
+ const st = await stat(file).catch(() => null);
+ return st?.isFile() ? st.size : null;
+}
+
+async function heldWithPointers(dataDir: string): Promise<string[]> {
+ const entries = await readdir(dataDir, { withFileTypes: true }).catch((err: NodeJS.ErrnoException) => {
+ if (err.code === "ENOENT") return [];
+ throw err;
+ });
+ const out: string[] = [];
+ for (const d of entries) {
+ if (!d.isDirectory() || d.name.startsWith(".")) continue;
+ if (await loadSavedVideo(path.join(dataDir, d.name))) out.push(d.name);
+ }
+ return out.sort();
+}
+
+export async function preparePlayable(opts: PreparePlayableOptions): Promise<PreparePlayableResult> {
+ const onLog = (s: string) => opts.onLog(s.endsWith("\n") ? s : `${s}\n`);
+ const now = opts.deps?.now ?? (() => new Date());
+ const probe = opts.deps?.probe ?? ((f: string) => ffprobe(opts.paths, f, opts.signal));
+ const remux =
+ opts.deps?.remux ??
+ ((src: string, dest: string, args: string[], signal?: AbortSignal) => ffmpegRemux(opts.paths, src, dest, args, signal));
+ const root = opts.root ?? (await defaultPlayableRoot(opts.paths, opts.channelConfig));
+ const result: PreparePlayableResult = {
+ root,
+ prepared: [],
+ unchanged: [],
+ unplayable: [],
+ noContainer: [],
+ failed: [],
+ dryRun: !!opts.dryRun,
+ };
+ // The root's parent must exist: a copy aimed at an unmounted drive is never
+ // materialised on the root filesystem by a mkdir.
+ const parent = await stat(path.dirname(root)).catch(() => null);
+ if (!parent?.isDirectory()) {
+ throw new Error(`${path.dirname(root)} is not there (is its drive mounted?) — the playable root ${root} cannot be made`);
+ }
+ const dataDir = path.join(opts.paths.channelsDir, opts.slug, "data");
+ const ids = opts.ids ?? (await heldWithPointers(dataDir));
+ const slugDir = path.join(root, opts.slug);
+ const manifest = await loadPlayableManifest(root, opts.slug);
+ onLog(`${ids.length} video(s) with a saved container; playable root ${root}`);
+
+ let n = 0;
+ for (const id of ids) {
+ n += 1;
+ if (opts.signal?.aborted) {
+ result.stopped = "cancelled";
+ break;
+ }
+ if (opts.drainSignal?.aborted) {
+ result.stopped = "drained";
+ onLog(`Drained: ${ids.length - n + 1} left for the next run.`);
+ break;
+ }
+ const videoDir = path.join(dataDir, id);
+ const pointer = await loadSavedVideo(videoDir);
+ const src = pointer ? savedVideoPath(pointer) : null;
+ const srcBytes = src ? await sizeOf(src) : null;
+ if (!pointer || !src || srcBytes === null) {
+ result.noContainer.push(id);
+ onLog(`${id}: no saved container${pointer ? ` (${src} is missing)` : ""}`);
+ continue;
+ }
+ try {
+ let sourceSha256 = pointer.sha256;
+ if (!sourceSha256) {
+ onLog(`${id}: hashing ${pointer.file} (${formatBytes(srcBytes)})`);
+ sourceSha256 = await sha256File(src, opts.signal);
+ if (!opts.dryRun) await updateSavedVideoChecksum(videoDir, sourceSha256);
+ }
+ const prior = manifest.entries[id];
+ if (prior && prior.sourceSha256 === sourceSha256) {
+ const [fb, tb] = await Promise.all([
+ sizeOf(path.join(slugDir, prior.file)),
+ sizeOf(path.join(slugDir, prior.torrent)),
+ ]);
+ if (fb === prior.bytes && tb !== null) {
+ result.unchanged.push(id);
+ continue;
+ }
+ }
+ const decision = decidePlayable(await probe(src));
+ if (!decision.playable) {
+ result.unplayable.push({ id, reason: decision.reason });
+ onLog(`${id}: not playable — ${decision.reason}`);
+ continue;
+ }
+ if (opts.dryRun) {
+ onLog(`[${n}/${ids.length}] ${id}: would remux — ${decision.reason}`);
+ result.prepared.push(id);
+ continue;
+ }
+ const free = await getFreeBytes(path.dirname(root));
+ if (free < srcBytes + PLAYABLE_DISK_MARGIN_BYTES) {
+ result.stopped = `the playable root's disk has ${formatBytes(free)} free; ${id} needs about ${formatBytes(srcBytes)} plus ${formatBytes(PLAYABLE_DISK_MARGIN_BYTES)}`;
+ onLog(`Stopped: ${result.stopped}. A re-run resumes.`);
+ break;
+ }
+ const outDir = path.join(slugDir, id);
+ await mkdir(outDir, { recursive: true });
+ const name = `${id}.${decision.ext}`;
+ const out = path.join(outDir, name);
+ const part = path.join(outDir, `.${name}.part`);
+ onLog(`[${n}/${ids.length}] ${id}: ${decision.reason}`);
+ await rm(part, { force: true });
+ try {
+ await remux(src, part, decision.args, opts.signal);
+ await rename(part, out);
+ } catch (err) {
+ await rm(part, { force: true }).catch(() => {});
+ throw err;
+ }
+ // A copy of another extension from an earlier run is not this video's any more.
+ for (const f of await readdir(outDir)) {
+ if (f !== name && f !== `${id}.torrent` && f.startsWith(`${id}.`)) await rm(path.join(outDir, f), { force: true });
+ }
+ const bytes = (await sizeOf(out)) ?? 0;
+ const made = await makePlayableTorrent({
+ file: out,
+ name,
+ bytes,
+ ...(opts.trackers ? { trackers: opts.trackers } : {}),
+ });
+ const torrentPath = path.join(outDir, `${id}.torrent`);
+ await writeFile(`${torrentPath}.part`, made.torrent);
+ await rename(`${torrentPath}.part`, torrentPath);
+ const probed = await probe(out).catch(() => null);
+ const entry: PlayableEntry = {
+ sourceSha256,
+ file: `${id}/${name}`,
+ torrent: `${id}/${id}.torrent`,
+ ext: decision.ext,
+ mime: decision.mime,
+ bytes,
+ infoHash: made.infoHash,
+ pieceLength: made.pieceLength,
+ vcodec: probed?.vcodec ?? "",
+ acodec: probed?.acodec ?? null,
+ preparedAt: now().toISOString(),
+ };
+ manifest.entries[id] = entry;
+ await writeJsonAtomic(playableManifestPath(root, opts.slug), manifest);
+ result.prepared.push(id);
+ onLog(`${id}: ${name} ${formatBytes(bytes)}, infohash ${made.infoHash}, ${made.pieceLength / 1024} KiB pieces`);
+ } catch (err) {
+ if (opts.signal?.aborted) {
+ result.stopped = "cancelled";
+ break;
+ }
+ result.failed.push({ id, error: (err as Error).message });
+ onLog(`${id}: FAILED — ${(err as Error).message}`);
+ }
+ }
+ onLog(
+ `summary: ${JSON.stringify({
+ dryRun: result.dryRun,
+ root,
+ prepared: result.prepared.length,
+ unchanged: result.unchanged.length,
+ unplayable: result.unplayable,
+ noContainer: result.noContainer,
+ failed: result.failed,
+ ...(result.stopped ? { stopped: result.stopped } : {}),
+ })}`,
+ );
+ return result;
+}
diff --git a/common/jobs/jobKinds.test.ts b/common/jobs/jobKinds.test.ts
@@ -307,3 +307,15 @@ test("release 21: attach-media is a drainable media ingest kind, not replayable"
assert.equal(kindNeedsText("attach-media"), false);
assert.equal(isIngestKind("attach-media"), true);
});
+
+// RELEASE 21 D3: playable copies and torrents. Reads and writes big files,
+// stops between videos, and writes the manifest a site's build reads.
+test("release 21: prepare-playable is a drainable media ingest kind, not replayable", () => {
+ const meta = getJobKind("prepare-playable");
+ assert.ok(meta, "registered");
+ assert.equal(jobKindLabel("prepare-playable"), "Prepare playable copies");
+ assert.equal(isDrainableKind("prepare-playable"), true);
+ assert.equal(meta.replayable, false);
+ assert.equal(kindNeedsMedia("prepare-playable"), true);
+ assert.equal(isIngestKind("prepare-playable"), true);
+});
diff --git a/common/jobs/jobKinds.ts b/common/jobs/jobKinds.ts
@@ -977,6 +977,22 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
queueKeyStrategy: "custom",
needsMedia: true,
},
+ // PLAYABLE COPIES AND THEIR TORRENTS (release 21 D3,
+ // controller/preparePlayable.ts): each saved container remuxed losslessly
+ // into a browser's container, and one torrent per copy. Media: it reads the
+ // saved container and writes copies as big. Drainable: it stops between
+ // videos. An INGEST kind: the playable manifest it writes is what a site's
+ // build reads for its torrents (D4). One queue machine-wide (`playable`):
+ // a remux is disk-bound on the archive drive, and two at once only thrash
+ // it. Heavy — it belongs in release 19's heavy slot once that lands.
+ "prepare-playable": {
+ kind: "prepare-playable",
+ label: "Prepare playable copies",
+ drainable: true,
+ replayable: false,
+ queueKeyStrategy: "custom",
+ needsMedia: true,
+ },
};
export function getJobKind(kind: string): JobKindMeta | undefined {
@@ -1042,6 +1058,7 @@ const INGEST_KINDS: ReadonlySet<string> = new Set([
"quick-availability-check",
"check-maybe-missing",
"attach-media",
+ "prepare-playable",
]);
export function isIngestKind(kind: string | undefined): boolean {
diff --git a/common/lib/playableMedia.test.ts b/common/lib/playableMedia.test.ts
@@ -0,0 +1,140 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ MAX_PIECE_LENGTH,
+ MIN_PIECE_LENGTH,
+ decidePlayable,
+ magnetFor,
+ parsePlayableManifest,
+ parseProbe,
+ pieceLengthFor,
+ type MediaProbe,
+} from "./playableMedia";
+
+// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test lib/playableMedia.test.ts
+
+const p = (ext: string, vcodec: string | null, acodec: string | null): MediaProbe => ({
+ formatName: ext === "mp4" ? "mov,mp4,m4a,3gp,3g2,mj2" : "matroska,webm",
+ ext,
+ vcodec,
+ acodec,
+});
+
+// The whole table: video × audio × source container → the playable container.
+// The pilot archive's measured mix is every row marked ★.
+const TABLE: [string, string, string | null, "mp4" | "webm" | null][] = [
+ ["mp4", "h264", "aac", "mp4"], // ★ 145
+ ["mp4", "h264", "opus", "mp4"],
+ ["mp4", "vp9", "aac", "mp4"],
+ ["mp4", "vp9", "opus", "mp4"],
+ ["mp4", "av1", "aac", "mp4"], // ★ 1
+ ["mp4", "av1", "opus", "mp4"],
+ ["mkv", "h264", "aac", "mp4"],
+ ["mkv", "h264", "opus", "mp4"], // ★ 2
+ ["mkv", "vp9", "aac", "mp4"], // ★ 11
+ ["mkv", "vp9", "opus", "mp4"],
+ ["mkv", "av1", "aac", "mp4"],
+ ["mkv", "av1", "opus", "mp4"], // ★ 1
+ ["webm", "vp9", "opus", "webm"], // ★ 9
+ ["webm", "av1", "opus", "webm"],
+ ["webm", "vp8", "vorbis", "webm"],
+ ["webm", "vp9", null, "webm"],
+ ["webm", "h264", "aac", "mp4"], // not a webm pair: mp4
+ ["webm", "vp9", "aac", "mp4"],
+ ["mp4", "h264", null, "mp4"],
+ ["mp4", "h264", "mp3", "mp4"],
+ ["mkv", "hevc", "aac", null], // no lossless remux a browser plays
+ ["mkv", "mpeg4", "mp3", null],
+ ["mkv", "h264", "ac3", null],
+ ["webm", "vp8", "aac", null],
+];
+
+test("the remux decision table", () => {
+ for (const [ext, v, a, want] of TABLE) {
+ const d = decidePlayable(p(ext, v, a));
+ const label = `${v}/${a ?? "none"} in ${ext}`;
+ if (want === null) {
+ assert.equal(d.playable, false, label);
+ continue;
+ }
+ assert.equal(d.playable, true, label);
+ if (!d.playable) continue;
+ assert.equal(d.ext, want, label);
+ assert.equal(d.mime, `video/${want}`, label);
+ // Lossless always; the index in front always.
+ assert.ok(d.args.includes("copy"), label);
+ assert.ok(!d.args.some((x) => /^lib|^-crf$|^-b:/.test(x)), `${label}: no encoder`);
+ if (want === "mp4") assert.ok(d.args.join(" ").includes("-movflags +faststart"), label);
+ if (want === "webm") assert.ok(d.args.join(" ").includes("-cues_to_front 1"), label);
+ }
+});
+
+test("no video is not playable; metadata and extra streams are dropped", () => {
+ assert.equal(decidePlayable(p("mp4", null, "aac")).playable, false);
+ const d = decidePlayable(p("mp4", "h264", "aac"));
+ assert.ok(d.playable);
+ if (!d.playable) return;
+ const a = d.args.join(" ");
+ assert.match(a, /-map 0:V:0 -map 0:a:0\?/, "first real video (not cover art), first audio if any");
+ assert.match(a, /-map_metadata -1/);
+ assert.match(a, /-map_chapters -1/);
+ assert.match(a, /-sn -dn/);
+ assert.match(a, /-fflags \+bitexact/);
+});
+
+test("parseProbe skips cover art and reads the extension", () => {
+ const probe = parseProbe(
+ {
+ format: { format_name: "matroska,webm" },
+ streams: [
+ { codec_type: "video", codec_name: "mjpeg", disposition: { attached_pic: 1 } },
+ { codec_type: "video", codec_name: "vp9", disposition: { attached_pic: 0 } },
+ { codec_type: "audio", codec_name: "opus" },
+ { codec_type: "subtitle", codec_name: "ass" },
+ ],
+ },
+ ".WEBM",
+ );
+ assert.deepEqual(probe, { formatName: "matroska,webm", ext: "webm", vcodec: "vp9", acodec: "opus" });
+ assert.deepEqual(parseProbe(null, "mp4"), { formatName: "", ext: "mp4", vcodec: null, acodec: null });
+});
+
+test("piece length: a power of two, 256 KiB–1 MiB, scaled to the size", () => {
+ assert.equal(pieceLengthFor(1), MIN_PIECE_LENGTH);
+ assert.equal(pieceLengthFor(20 * 1024 ** 2), MIN_PIECE_LENGTH);
+ assert.equal(pieceLengthFor(800 * 1024 ** 2), 512 * 1024);
+ assert.equal(pieceLengthFor(2 * 1024 ** 3), MAX_PIECE_LENGTH);
+ assert.equal(pieceLengthFor(50 * 1024 ** 3), MAX_PIECE_LENGTH);
+ for (const n of [1e6, 3e8, 7e8, 1.5e9]) {
+ const len = pieceLengthFor(n);
+ assert.equal(len & (len - 1), 0, `${len} is a power of two`);
+ }
+});
+
+test("a magnet names the infohash, the file and only the trackers given", () => {
+ const hash = "a".repeat(40);
+ assert.equal(magnetFor({ infoHash: hash, name: "abc.mp4" }), `magnet:?xt=urn:btih:${hash}&dn=abc.mp4`);
+ assert.equal(
+ magnetFor({ infoHash: hash, name: "a b.mp4", trackers: ["wss://t.example/announce"] }),
+ `magnet:?xt=urn:btih:${hash}&dn=a%20b.mp4&tr=wss%3A%2F%2Ft.example%2Fannounce`,
+ );
+});
+
+test("the manifest parse is tolerant: a bad entry is dropped, not fatal", () => {
+ const good = {
+ sourceSha256: "b".repeat(64),
+ file: "x/x.mp4",
+ torrent: "x/x.torrent",
+ ext: "mp4",
+ mime: "video/mp4",
+ bytes: 10,
+ infoHash: "c".repeat(40),
+ pieceLength: 262144,
+ vcodec: "h264",
+ acodec: "aac",
+ preparedAt: "2026-10-09T00:00:00Z",
+ };
+ const m = parsePlayableManifest({ version: 1, slug: "s", entries: { x: good, y: { ...good, infoHash: "nope" } } }, "s");
+ assert.deepEqual(Object.keys(m.entries), ["x"]);
+ assert.deepEqual(parsePlayableManifest("garbage", "s").entries, {});
+});
diff --git a/common/lib/playableMedia.ts b/common/lib/playableMedia.ts
@@ -0,0 +1,218 @@
+// A SAVED CONTAINER, MADE PLAYABLE IN A BROWSER — WITHOUT RE-ENCODING
+// (release 21 D3).
+//
+// The sites play an archived video in a native <video> fed by WebTorrent
+// (D5). A browser plays mp4 (h264/vp9/av1 with aac/opus/mp3) and webm
+// (vp9/vp8/av1 with opus/vorbis); it does not play Matroska, and an mp4 whose
+// `moov` atom is at the back cannot start until the file's last pieces
+// arrive. So each container is REMUXED — `ffmpeg -c copy`, every frame and
+// sample bit-for-bit — into the browser's container with its index in front:
+//
+// video audio source → playable
+// h264/vp9/av1 aac/opus/mp3/none mp4/mkv/… → mp4 (+faststart)
+// vp9/vp8/av1 opus/vorbis/none webm → webm (cues in front)
+// anything else (hevc, mpeg4, …, or no video) → not playable: a
+// re-encode is a ruling,
+// not a default
+//
+// The pilot archive's measured mix (release 21): 145 mp4 h264/aac (124 of
+// them moov-at-back), 9 webm vp9/opus, 11 mkv vp9/aac, 2 mkv h264/opus, 1 mp4
+// av1/aac, 1 mkv av1/opus — every one of them lossless under this table.
+//
+// Only the first real video stream (not cover art) and the first audio stream
+// are kept; subtitles, data, attachments, global metadata and chapters are
+// dropped — a published file carries the picture and the sound and nothing
+// that names anyone. `+bitexact` keeps ffmpeg's version string out, so the
+// same source remuxes to the same bytes, and so to the same infohash.
+//
+// PURE: the probe is parsed here from ffprobe's JSON; running ffprobe and
+// ffmpeg is the controller's (controller/preparePlayable.ts).
+
+export type ProbedStream = {
+ codec_type?: string;
+ codec_name?: string;
+ disposition?: { attached_pic?: number };
+};
+
+export type MediaProbe = {
+ // ffprobe's format_name ("mov,mp4,m4a,3gp,3g2,mj2", "matroska,webm").
+ formatName: string;
+ // The source file's extension, lower-case, no dot: ffprobe says
+ // "matroska,webm" for both an .mkv and a .webm, and only the name tells
+ // them apart.
+ ext: string;
+ vcodec: string | null;
+ acodec: string | null;
+};
+
+export type PlayableDecision =
+ | {
+ playable: true;
+ ext: "mp4" | "webm";
+ mime: "video/mp4" | "video/webm";
+ // The ffmpeg arguments between `-i <src>` and the output path.
+ args: string[];
+ reason: string;
+ }
+ | { playable: false; reason: string };
+
+const MP4_VIDEO = new Set(["h264", "vp9", "av1"]);
+const MP4_AUDIO = new Set(["aac", "opus", "mp3"]);
+const WEBM_VIDEO = new Set(["vp9", "vp8", "av1"]);
+const WEBM_AUDIO = new Set(["opus", "vorbis"]);
+
+// ffprobe -show_streams -show_format -of json → the probe.
+export function parseProbe(json: unknown, ext: string): MediaProbe {
+ const j = (json ?? {}) as { streams?: ProbedStream[]; format?: { format_name?: string } };
+ const streams = Array.isArray(j.streams) ? j.streams : [];
+ const video = streams.find((s) => s.codec_type === "video" && !s.disposition?.attached_pic);
+ const audio = streams.find((s) => s.codec_type === "audio");
+ return {
+ formatName: j.format?.format_name ?? "",
+ ext: ext.replace(/^\./, "").toLowerCase(),
+ vcodec: video?.codec_name ?? null,
+ acodec: audio?.codec_name ?? null,
+ };
+}
+
+const COMMON_ARGS = [
+ "-map", "0:V:0",
+ "-map", "0:a:0?",
+ "-c", "copy",
+ "-sn", "-dn",
+ "-map_metadata", "-1",
+ "-map_chapters", "-1",
+ "-fflags", "+bitexact",
+ "-flags:v", "+bitexact",
+ "-flags:a", "+bitexact",
+];
+
+export function decidePlayable(p: MediaProbe): PlayableDecision {
+ const v = p.vcodec;
+ const a = p.acodec;
+ const what = `${v ?? "no video"}/${a ?? "no audio"} in ${p.ext || p.formatName}`;
+ if (!v) return { playable: false, reason: `${what}: no video stream` };
+ const isWebm = p.ext === "webm";
+ if (isWebm && WEBM_VIDEO.has(v) && (a === null || WEBM_AUDIO.has(a))) {
+ return {
+ playable: true,
+ ext: "webm",
+ mime: "video/webm",
+ args: [...COMMON_ARGS, "-f", "webm", "-cues_to_front", "1"],
+ reason: `${what} → webm, remuxed with its cues in front`,
+ };
+ }
+ if (MP4_VIDEO.has(v) && (a === null || MP4_AUDIO.has(a))) {
+ return {
+ playable: true,
+ ext: "mp4",
+ mime: "video/mp4",
+ args: [...COMMON_ARGS, "-f", "mp4", "-movflags", "+faststart"],
+ reason: `${what} → mp4, remuxed with its index in front`,
+ };
+ }
+ return {
+ playable: false,
+ reason: `${what}: no lossless remux a browser plays (a re-encode is an operator ruling)`,
+ };
+}
+
+// --- torrents ---------------------------------------------------------------
+
+export const MIN_PIECE_LENGTH = 256 * 1024;
+export const MAX_PIECE_LENGTH = 1024 * 1024;
+// Aim for about this many pieces: a few thousand is what a browser peer
+// handles comfortably, and a seek fetches at most one piece of slack.
+const TARGET_PIECES = 2048;
+
+// A power of two between 256 KiB and 1 MiB, scaled to the file: a 20 MB clip
+// gets 256 KiB pieces, a 2 GB episode 1 MiB.
+export function pieceLengthFor(bytes: number): number {
+ let len = MIN_PIECE_LENGTH;
+ while (len < MAX_PIECE_LENGTH && bytes / len > TARGET_PIECES) len *= 2;
+ return len;
+}
+
+// A magnet for an infohash, its display name and the announce URLs a site
+// names. The trackers are not in the info dict, so the same infohash can be
+// announced anywhere later.
+export function magnetFor(opts: { infoHash: string; name: string; trackers?: readonly string[] }): string {
+ const parts = [`xt=urn:btih:${opts.infoHash}`, `dn=${encodeURIComponent(opts.name)}`];
+ for (const tr of opts.trackers ?? []) parts.push(`tr=${encodeURIComponent(tr)}`);
+ return `magnet:?${parts.join("&")}`;
+}
+
+// --- the manifest -----------------------------------------------------------
+
+// `<root>/<slug>/playable.json`: what has been prepared, keyed by video id.
+// A re-run compares `sourceSha256` and skips an entry whose files are there.
+export type PlayableEntry = {
+ // The saved container this was made from, by hash.
+ sourceSha256: string;
+ // Relative to `<root>/<slug>/`.
+ file: string;
+ torrent: string;
+ ext: "mp4" | "webm";
+ mime: "video/mp4" | "video/webm";
+ bytes: number;
+ infoHash: string;
+ pieceLength: number;
+ vcodec: string;
+ acodec: string | null;
+ preparedAt: string;
+};
+
+export type PlayableManifest = {
+ version: 1;
+ slug: string;
+ entries: Record<string, PlayableEntry>;
+};
+
+export const PLAYABLE_MANIFEST_FILENAME = "playable.json";
+
+export function emptyPlayableManifest(slug: string): PlayableManifest {
+ return { version: 1, slug, entries: {} };
+}
+
+const HEX40 = /^[0-9a-f]{40}$/;
+const HEX64 = /^[0-9a-f]{64}$/;
+
+// Tolerant: a malformed entry is dropped (and so re-prepared), never fatal.
+export function parsePlayableManifest(raw: unknown, slug: string): PlayableManifest {
+ const out = emptyPlayableManifest(slug);
+ if (!raw || typeof raw !== "object" || Array.isArray(raw)) return out;
+ const entries = (raw as { entries?: unknown }).entries;
+ if (!entries || typeof entries !== "object" || Array.isArray(entries)) return out;
+ for (const [id, e] of Object.entries(entries as Record<string, unknown>)) {
+ if (!e || typeof e !== "object") continue;
+ const r = e as Record<string, unknown>;
+ if (
+ typeof r.sourceSha256 !== "string" || !HEX64.test(r.sourceSha256) ||
+ typeof r.file !== "string" || typeof r.torrent !== "string" ||
+ (r.ext !== "mp4" && r.ext !== "webm") ||
+ (r.mime !== "video/mp4" && r.mime !== "video/webm") ||
+ typeof r.bytes !== "number" ||
+ typeof r.infoHash !== "string" || !HEX40.test(r.infoHash) ||
+ typeof r.pieceLength !== "number" ||
+ typeof r.vcodec !== "string" ||
+ (r.acodec !== null && typeof r.acodec !== "string") ||
+ typeof r.preparedAt !== "string"
+ ) {
+ continue;
+ }
+ out.entries[id] = {
+ sourceSha256: r.sourceSha256,
+ file: r.file,
+ torrent: r.torrent,
+ ext: r.ext,
+ mime: r.mime,
+ bytes: r.bytes,
+ infoHash: r.infoHash,
+ pieceLength: r.pieceLength,
+ vcodec: r.vcodec,
+ acodec: r.acodec as string | null,
+ preparedAt: r.preparedAt,
+ };
+ }
+ return out;
+}
diff --git a/common/package.json b/common/package.json
@@ -61,6 +61,7 @@
"class-variance-authority": "^0.7.1",
"clsx": "^2.1.1",
"cmdk": "^1.1.1",
+ "create-torrent": "^6.1.3",
"d3-scale": "^4.0.2",
"execa": "^9.6.1",
"flexsearch": "^0.8.0",
@@ -70,6 +71,7 @@
"lucide-react": "^1.16.0",
"markdown-to-jsx": "^7.7.4",
"p-limit": "^7.3.0",
+ "parse-torrent": "^11.0.24",
"radix-ui": "^1.6.0",
"react-player": "^2.16.1",
"recharts": "2.15.4",
diff --git a/common/types/webtorrent-modules.d.ts b/common/types/webtorrent-modules.d.ts
@@ -0,0 +1,41 @@
+// Types for the WebTorrent-ecosystem packages release 21 uses, which ship
+// none: only the surface the code calls.
+
+declare module "create-torrent" {
+ export type CreateTorrentOptions = {
+ name?: string;
+ comment?: string;
+ createdBy?: string;
+ creationDate?: Date | number;
+ private?: boolean;
+ pieceLength?: number;
+ announceList?: string[][];
+ urlList?: string[];
+ info?: Record<string, unknown>;
+ };
+ export default function createTorrent(
+ input: string | string[] | Uint8Array,
+ opts: CreateTorrentOptions,
+ cb: (err: Error | null, torrent: Uint8Array) => void,
+ ): void;
+}
+
+declare module "parse-torrent" {
+ export type ParsedTorrent = {
+ infoHash: string;
+ name?: string;
+ length?: number;
+ pieceLength?: number;
+ announce?: string[];
+ urlList?: string[];
+ created?: Date;
+ createdBy?: string;
+ comment?: string;
+ private?: boolean;
+ info?: Record<string, unknown>;
+ files?: { path: string; name: string; length: number; offset: number }[];
+ };
+ export default function parseTorrent(torrent: Uint8Array | string): Promise<ParsedTorrent>;
+ export function toTorrentFile(parsed: ParsedTorrent): Uint8Array;
+ export function toMagnetURI(parsed: Partial<ParsedTorrent>): string;
+}
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,7 @@
# Changelog
## [Unreleased]
+- **Saved videos can be made browser-playable, with a torrent each.** `pnpm ops prepare-playable` (`POST /api/ops/prepare-playable`) and `archilyzer media playable <slug>` remux each of a channel's saved containers — without re-encoding (`-c copy`) — into an mp4 with its index in front, or a webm when it already is one (VP9/AV1 with Opus), drop subtitles, metadata and chapters, and make one single-file torrent of the copy: named `<id>.<ext>`, no web seed, no comment, no "created by", 256 KiB–1 MiB pieces. They go to `playable/<slug>/<id>/` beside the saved-video store, listed in `playable/<slug>/playable.json` with each infohash. `"trackers"` is the announce list written into each torrent (none by default); it is not part of the infohash, so the same torrent can be announced elsewhere later. A video already prepared from the same source (by sha256) is skipped, so a re-run is a no-op; a codec a browser cannot play without re-encoding (HEVC, MPEG-4 Part 2) is listed and left alone.
- **A channel's videos can get their media from a local archive.** `pnpm ops attach-media` (`POST /api/ops/attach-media`) and `archilyzer media attach <slug> <source>` take `{"slug", "source"}` — an absolute path to a directory, a `.zip` (read in place: a stored entry is copied straight out of it, with no temp dir) or a `.7z` — and put each held video's file into the saved-video store as its source container, so clip windows and report clips can be cut from it with nothing fetched. The id is the folder's trailing `(<id>)`, else the file's yt-dlp suffix; `"items": [{"id", "path"}]` names exact files and `"match"` narrows the folders. The pointer records where the file came from: `origin: {kind: "local-archive", archive, entry, sha256, attachedAt}`. A video that already has a saved container is left alone unless `"replace": true`; `"createRecords": true` writes a record for a video the channel does not hold (from the folder's yt-dlp `.info.json`, else its `description.txt` and name); `[LOST]` folders are listed and never attached. `"dryRun": true` lists what each folder is, and the held videos the archive has no media for, and writes nothing. The archive is never written to.
- **A curated tag can exist on some sites only.** A tag's new **Sites** field on /tags (`sites` in `transcripts/tags.json`; `pnpm ops tags` takes it in a define) names the sites it exists on. Its rules then fire, and its pins apply, only to videos on those sites' channels, and every other site drops it from its records, its counts and its `/tags.json` — where **Hidden** only hid the chip. Empty is every site, as before. Setting it, or changing the channels of those sites, re-derives the corpus's tags once at the next index update. The Eva tags are what this is for: they belong on Anilyzer alone.
- **The publish lane.** Publishing can run itself: turn it on at **/operations/publish** (the runner's Start, Drain and Stop, the hold, and the lane's settings; or `publish.enabled` in settings) and the lane checks every `checkEveryMinutes` (10) whether the index is stale; when it is — and its last update is at least `refreshEveryMinutes` (360) old — it updates it, then builds every site whose channels changed or whose data the new index moved, one stage at a time on the `publish` queue. What it may do with a site is the site's own — the **Publish policy** on the site's settings form, `site.json` `publish.auto` —: `off` (the default: left alone), `build`, `preview` (built and deployed to the preview branch `publish.previewBranch`) or `production`; the hub and the homepage have `publish.hub` and `publish.homepage`. A private site is only ever built, and a site needs its Cloudflare Pages project before it may deploy. Hold the lane and the stage running finishes and no next one starts; quiet hours (`publish.quietHours`) do the same; Drain finishes the stage and ends the runner. The lane never forces a stage: a stage that finds its target current does nothing. On /jobs every stage of one run reads `run <id> · <target>`, and a stage still queued when the editor restarts is cancelled, never re-queued — the lane works out again what is stale from what is on disk. `archilyzer publish now` runs the same plan from the command line, one stage after another in its own process.
diff --git a/editor/app/api/ops/prepare-playable/route.test.ts b/editor/app/api/ops/prepare-playable/route.test.ts
@@ -0,0 +1,66 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+// Run with:
+// pnpm -C editor exec tsx --test "app/api/ops/prepare-playable/route.test.ts"
+//
+// The body's shape and the refusals that come before any job, then one dry
+// run over a channel with no saved containers. Nothing is remuxed.
+
+const ROOT = await mkdtemp(path.join(os.tmpdir(), "prepare-playable-route-"));
+process.env.WORKER_TOKEN = "test-token";
+process.env.TRANSCRIPTS_DIR = ROOT;
+process.env.SETTINGS_FILE = path.join(ROOT, "settings.json");
+const { POST } = await import("./route");
+const { getRegistry } = await import("yt-dlp-transcript-common/jobs/registry");
+const { getPaths } = await import("yt-dlp-transcript-common/lib/paths");
+test.after(() => rm(ROOT, { recursive: true, force: true }));
+
+await mkdir(path.join(ROOT, "channels", "tism", "data", "aaaaaaaaaaa"), { recursive: true });
+await writeFile(path.join(ROOT, "channels", "tism", "config.json"), JSON.stringify({ handling: "youtube", name: "tism" }));
+
+async function post(body: Record<string, unknown>) {
+ const res = await POST(
+ new Request("http://localhost/api/ops/prepare-playable", {
+ method: "POST",
+ headers: { authorization: "Bearer test-token", "content-type": "application/json" },
+ body: JSON.stringify(body),
+ }),
+ );
+ return { status: res.status, json: (await res.json()) as { ok?: boolean; error?: string; jobId?: string } };
+}
+
+test("refusals before any job", async () => {
+ const cases: [Record<string, unknown>, RegExp][] = [
+ [{}, /"slug" is required/],
+ [{ slug: "tism", force: 1 }, /unknown key\(s\): force/],
+ [{ slug: "tism", ids: [] }, /non-empty array/],
+ [{ slug: "tism", ids: ["../x"] }, /not a video id/],
+ [{ slug: "tism", trackers: "wss://x" }, /must be an array/],
+ [{ slug: "tism", trackers: ["file:///etc/passwd"] }, /not a tracker URL/],
+ [{ slug: "tism", root: "relative" }, /must be an absolute path/],
+ [{ slug: "nope" }, /Channel "nope" not found/],
+ ];
+ for (const [body, re] of cases) {
+ const r = await post(body);
+ assert.equal(r.status, 400, JSON.stringify(body));
+ assert.match(r.json.error!, re, JSON.stringify(body));
+ }
+});
+
+test("a dry run over a channel with no saved containers is a job that says so", async () => {
+ const r = await post({ slug: "tism", dryRun: true, root: path.join(ROOT, "playable"), trackers: ["wss://tracker.example/announce"] });
+ assert.equal(r.status, 200, JSON.stringify(r.json));
+ for (let i = 0; i < 100; i++) {
+ const s = getRegistry().get(r.json.jobId!)?.status;
+ if (s === "done" || s === "failed") break;
+ await new Promise((res) => setTimeout(res, 50));
+ }
+ assert.equal(getRegistry().get(r.json.jobId!)?.status, "done");
+ const log = await readFile(path.join(getPaths().jobsDir, `${r.json.jobId}.log`), "utf8");
+ assert.match(log, /0 video\(s\) with a saved container/);
+ assert.match(log, /summary: \{"dryRun":true/);
+});
diff --git a/editor/app/api/ops/prepare-playable/route.ts b/editor/app/api/ops/prepare-playable/route.ts
@@ -0,0 +1,41 @@
+import { preparePlayableAction } from "../../../channels/[slug]/playableActions";
+import { OpsInputError, jobResponse, ops, optBool, optString, reqSlug, reqStringArray, reqVideoId, type OpsBody } from "../_lib";
+
+export const dynamic = "force-dynamic";
+
+// POST { slug, ids?: string[], trackers?: string[], root?, dryRun? }
+// -> { ok: true, jobId }
+//
+// Playable copies and their torrents (release 21 D3): each of the channel's
+// saved containers remuxed losslessly (`-c copy`) into mp4 (+faststart) or
+// webm, and one single-file torrent per copy, under the playable root —
+// `playable/` beside the saved-video store, unless `root` names another — as
+// one `prepare-playable` job (controller/preparePlayable.ts). `ids` narrows
+// it to those videos; `trackers` is the announce list written into each
+// .torrent (none by default; it is outside the info dict, so the infohash
+// does not depend on it). A video already prepared from the same source
+// (by sha256) is skipped. `dryRun` logs each decision and writes nothing.
+function optIds(body: OpsBody): string[] | undefined {
+ if (body.ids === undefined) return undefined;
+ return reqStringArray(body, "ids").map((id) => reqVideoId({ id }, "id"));
+}
+
+function optTrackers(body: OpsBody): string[] | undefined {
+ if (body.trackers === undefined) return undefined;
+ if (!Array.isArray(body.trackers)) throw new OpsInputError('"trackers" must be an array of announce URLs');
+ if (body.trackers.length === 0) return [];
+ return reqStringArray(body, "trackers");
+}
+
+export async function POST(request: Request) {
+ return ops(request, ["slug", "ids", "trackers", "root", "dryRun"], async (body) =>
+ jobResponse(
+ await preparePlayableAction(reqSlug(body, "slug"), {
+ ids: optIds(body),
+ trackers: optTrackers(body),
+ root: optString(body, "root"),
+ dryRun: optBool(body, "dryRun"),
+ }),
+ ),
+ );
+}
diff --git a/editor/app/channels/[slug]/playableActions.ts b/editor/app/channels/[slug]/playableActions.ts
@@ -0,0 +1,67 @@
+"use server";
+
+import path from "node:path";
+import { safeRevalidate } from "../../lib/safeRevalidate";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels";
+import { preparePlayable } from "yt-dlp-transcript-common/controller/preparePlayable";
+import {
+ runManagedFunction,
+ type StreamActionResult,
+} from "yt-dlp-transcript-common/jobs/streamCommand";
+
+// PLAYABLE COPIES AND THEIR TORRENTS (release 21 D3): the `prepare-playable`
+// job, as `pnpm ops prepare-playable` posts it (`/api/ops/prepare-playable`).
+// The work and its rules are common/controller/preparePlayable.ts.
+//
+// ONE QUEUE MACHINE-WIDE: a remux reads and writes gigabytes on the archive
+// drive, and two channels' worth at once only thrash it.
+const PLAYABLE_QUEUE = "playable";
+
+export async function preparePlayableAction(
+ slug: string,
+ opts: { ids?: string[]; trackers?: string[]; root?: string; dryRun?: boolean },
+): Promise<StreamActionResult> {
+ const paths = getPaths();
+ const channelConfig = await readChannelConfig(paths, slug);
+ if (!channelConfig) {
+ return { ok: false, error: `Channel "${slug}" not found` };
+ }
+ if (opts.root !== undefined && !path.isAbsolute(opts.root)) {
+ return { ok: false, error: `"root" must be an absolute path` };
+ }
+ for (const tr of opts.trackers ?? []) {
+ if (!/^(wss?|https?|udp):\/\/[^\s]+$/i.test(tr)) {
+ return { ok: false, error: `"${tr}" is not a tracker URL (wss://, ws://, https://, http:// or udp://)` };
+ }
+ }
+ return runManagedFunction({
+ kind: "prepare-playable",
+ queueKey: PLAYABLE_QUEUE,
+ paths,
+ channelSlug: slug,
+ fn: async (onLog, signal, _setProgress, ctx) => {
+ const result = await preparePlayable({
+ paths,
+ slug,
+ channelConfig,
+ ...(opts.root ? { root: opts.root } : {}),
+ ...(opts.ids ? { ids: opts.ids } : {}),
+ ...(opts.trackers ? { trackers: opts.trackers } : {}),
+ ...(opts.dryRun ? { dryRun: true } : {}),
+ onLog,
+ signal,
+ drainSignal: ctx.drainSignal,
+ });
+ if (!opts.dryRun && result.prepared.length > 0) {
+ safeRevalidate([`/channels/${slug}`]);
+ }
+ if (result.failed.length > 0 && result.prepared.length === 0) {
+ throw new Error(`${result.failed.length} failed, none prepared`);
+ }
+ if (result.stopped && result.stopped !== "drained" && result.stopped !== "cancelled") {
+ throw new Error(`stopped: ${result.stopped}`);
+ }
+ },
+ });
+}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
@@ -38,6 +38,9 @@ importers:
cmdk:
specifier: ^1.1.1
version: 1.1.1(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)
+ create-torrent:
+ specifier: ^6.1.3
+ version: 6.1.3
d3-scale:
specifier: ^4.0.2
version: 4.0.2
@@ -68,6 +71,9 @@ importers:
p-limit:
specifier: ^7.3.0
version: 7.3.0
+ parse-torrent:
+ specifier: ^11.0.24
+ version: 11.0.24
radix-ui:
specifier: ^1.6.0
version: 1.6.0(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)
@@ -2320,6 +2326,10 @@ packages:
'@tanstack/virtual-core@3.14.0':
resolution: {integrity: sha512-JLANqGy/D6k4Ujmh8Tr25lGimuOXNiaVyXaCAZS0W+1390sADdGnyUdSWNIfd49gebtIxGMij4IktRVzrdr12Q==}
+ '@thaunknown/thirty-two@1.0.6':
+ resolution: {integrity: sha512-Uz2kFb0cNeHg7hds8BtI/umcwzaSMUdbkeQgZrIZQbp3gWRFjrvrvBHNEB6SQYChVxC3rWHovKx1Ya/vSRRMYA==}
+ engines: {node: '>=0.2.6'}
+
'@tybys/wasm-util@0.10.1':
resolution: {integrity: sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==}
@@ -2541,6 +2551,10 @@ packages:
'@zeit/schemas@2.36.0':
resolution: {integrity: sha512-7kjMwcChYEzMKjeex9ZFXkt1AyNov9R5HZtjBKVsmVpw7pa7ZtlCGvCBC2vnnXctaYN+aRI61HjIqeetZW5ROg==}
+ abort-controller@3.0.0:
+ resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==}
+ engines: {node: '>=6.5'}
+
acorn-jsx@5.3.2:
resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==}
peerDependencies:
@@ -2659,9 +2673,20 @@ packages:
engines: {node: '>=6.0.0'}
hasBin: true
+ bencode@4.0.1:
+ resolution: {integrity: sha512-/bCR3FvgfB8AZk4LlowShU+nfaTNRM2vruJ9aejDa5LpgkCoRjFyEauFiZFxbpxVxQbBFapYCN42Er2dkE/Qgw==}
+ engines: {node: '>=12.20.0'}
+
+ bep53-range@2.0.1:
+ resolution: {integrity: sha512-VWN7j7sqUssWD6ROZe2m709YL3u5A1IAx1qvAbH3DAhszHt4WPvCzJLRJ+mzmZn1qWIb4mpflwdn95qezz3n8g==}
+ engines: {node: '>=12.20.0'}
+
blake3-wasm@2.1.5:
resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==}
+ block-iterator@1.1.2:
+ resolution: {integrity: sha512-yAHUP44v2K25xLPdrgVTgwtuQctlullzjczu9CoUZom5AP3g4p1R1+aWHjS1GHG9JtcSUVUnbEPiuXiW5YZ24w==}
+
bowser@2.14.1:
resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==}
@@ -2784,6 +2809,14 @@ packages:
resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==}
engines: {node: '>=18'}
+ create-torrent@6.1.3:
+ resolution: {integrity: sha512-/XARuf6/TmOLICFkUU7pf4KPL/vlIFyk2xOJjQ4np8Xzl8xLt9W7KJps1EVnZyUCMHbAm5xYfCoDrgVw3F4FxA==}
+ engines: {node: '>=12'}
+ hasBin: true
+
+ cross-fetch-ponyfill@1.0.3:
+ resolution: {integrity: sha512-uOBkDhUAGAbx/FEzNKkOfx3w57H8xReBBXoZvUnOKTI0FW0Xvrj3GrYv2iZXUqlffC1LMGfQzhmBM/ke+6eTDA==}
+
cross-spawn@7.0.6:
resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==}
engines: {node: '>= 8'}
@@ -2838,6 +2871,10 @@ packages:
damerau-levenshtein@1.0.8:
resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==}
+ data-uri-to-buffer@4.0.1:
+ resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==}
+ engines: {node: '>= 12'}
+
data-view-buffer@1.0.2:
resolution: {integrity: sha512-EmKO5V3OLXh1rtK2wgXRansaK1/mtVdTUEiEI0W8RkvgT05kfxaH29PliLnpLP73yYO6142Q72QNa8Wx/A5CqQ==}
engines: {node: '>= 0.4'}
@@ -3109,6 +3146,10 @@ packages:
resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==}
engines: {node: '>=0.10.0'}
+ event-target-shim@5.0.1:
+ resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==}
+ engines: {node: '>=6'}
+
eventemitter3@4.0.7:
resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==}
@@ -3149,6 +3190,9 @@ packages:
fast-levenshtein@2.0.6:
resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==}
+ fast-readable-async-iterator@2.0.0:
+ resolution: {integrity: sha512-8Sld+DuyWRIftl86ZguJxR2oXCBccOiJxrY/Rj9/7ZBynW8pYMWzIcqxFL1da+25jaWJZVa+HHX/8SsA21JdTA==}
+
fast-uri@3.1.0:
resolution: {integrity: sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==}
@@ -3164,6 +3208,10 @@ packages:
picomatch:
optional: true
+ fetch-blob@3.2.0:
+ resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}
+ engines: {node: ^12.20 || >= 14.13}
+
figures@6.1.0:
resolution: {integrity: sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==}
engines: {node: '>=18'}
@@ -3194,6 +3242,10 @@ packages:
resolution: {integrity: sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==}
engines: {node: '>= 0.4'}
+ formdata-polyfill@4.0.10:
+ resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
+ engines: {node: '>=12.20.0'}
+
fs-extra@11.3.4:
resolution: {integrity: sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==}
engines: {node: '>=14.14'}
@@ -3238,6 +3290,10 @@ packages:
resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==}
engines: {node: '>= 0.4'}
+ get-stdin@10.0.0:
+ resolution: {integrity: sha512-eWSePJ4zXFdqz+/Lyfopob4rIcoF/U2XfE8nJc7iZV6lnebWc9k7DoQQpX+2a9jc0AOvBsXvbe5YkjXl/MHbpg==}
+ engines: {node: '>=20'}
+
get-stream@6.0.1:
resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==}
engines: {node: '>=10'}
@@ -3401,6 +3457,9 @@ packages:
resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==}
engines: {node: '>=0.10.0'}
+ is-file@1.0.0:
+ resolution: {integrity: sha512-ZGMuc+xA8mRnrXtmtf2l/EkIW2zaD2LSBWlaOVEF6yH4RTndHob65V4SwWWdtGKVthQfXPVKsXqw4TDUjbVxVQ==}
+
is-finalizationregistry@1.1.1:
resolution: {integrity: sha512-1pC6N8qWJbWoPtEjgcL2xyhQOP491EQjeUo3qTKcmV8YSDDJrOepfG8pcC7h/QgnQHYSv0mJ3Z/ZWxmatVrysg==}
engines: {node: '>= 0.4'}
@@ -3507,6 +3566,9 @@ packages:
resolution: {integrity: sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==}
hasBin: true
+ join-async-iterator@1.1.1:
+ resolution: {integrity: sha512-ATse+nuNeKZ9K1y27LKdvPe/GCe9R/u9dw9vI248e+vILeRK3IcJP4JUPAlSmKRCDK0cKhEwfmiw4Skqx7UnGQ==}
+
jose@6.2.3:
resolution: {integrity: sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==}
@@ -3550,6 +3612,10 @@ packages:
resolution: {integrity: sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==}
engines: {node: '>=4.0'}
+ junk@4.0.1:
+ resolution: {integrity: sha512-Qush0uP+G8ZScpGMZvHUiRfI0YBWuB3gVBYlI0v0vvOJt5FLicco+IkP0a50LqTTQhmts/m6tP5SWE+USyIvcQ==}
+ engines: {node: '>=12.20'}
+
keyv@4.5.4:
resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==}
@@ -3674,6 +3740,10 @@ packages:
magic-string@0.30.21:
resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==}
+ magnet-uri@7.0.10:
+ resolution: {integrity: sha512-PjS/xWnvq9RK6P/8+dWG7ka3mFGGt/R9xiq5A+P8a8MpE40Dw6yNyTdjCFuJHS3apn6p5bJUCzMSceW3VxezRA==}
+ engines: {node: '>=12.20.0'}
+
markdown-to-jsx@7.7.17:
resolution: {integrity: sha512-7mG/1feQ0TX5I7YyMZVDgCC/y2I3CiEhIRQIhyov9nGBP5eoVrOXXHuL5ZP8GRfxVZKRiXWJgwXkb9It+nQZfQ==}
engines: {node: '>= 10'}
@@ -3785,10 +3855,19 @@ packages:
node-addon-api@6.1.0:
resolution: {integrity: sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==}
+ node-domexception@1.0.0:
+ resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
+ engines: {node: '>=10.5.0'}
+ deprecated: Use your platform's native DOMException instead
+
node-exports-info@1.6.0:
resolution: {integrity: sha512-pyFS63ptit/P5WqUkt+UUfe+4oevH+bFeIiPPdfb0pFeYEu/1ELnJu5l+5EcTKYL5M7zaAa7S8ddywgXypqKCw==}
engines: {node: '>= 0.4'}
+ node-fetch@3.3.2:
+ resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==}
+ engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+
node-gyp-build-optional-packages@5.2.2:
resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==}
hasBin: true
@@ -3840,6 +3919,9 @@ packages:
resolution: {integrity: sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==}
engines: {node: '>= 0.8'}
+ once@1.4.0:
+ resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
+
onetime@5.1.2:
resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==}
engines: {node: '>=6'}
@@ -3875,6 +3957,11 @@ packages:
resolution: {integrity: sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw==}
engines: {node: '>=18'}
+ parse-torrent@11.0.24:
+ resolution: {integrity: sha512-6nNWYiJq6egD8pMqZYL1mfnAly9ZH32DHkAYgZuAfwWij52ce7e3BtrWGVw2q1Bpb5Us8YQVGN7ML/47z/bVIA==}
+ engines: {node: '>=12.20.0'}
+ hasBin: true
+
path-exists@4.0.0:
resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==}
engines: {node: '>=8'}
@@ -3913,6 +4000,9 @@ packages:
resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==}
engines: {node: '>=12'}
+ piece-length@2.0.1:
+ resolution: {integrity: sha512-dBILiDmm43y0JPISWEmVGKBETQjwJe6mSU9GND+P9KW0SJGUwoU/odyH1nbalOP9i8WSYuqf1lQnaj92Bhw+Ug==}
+
pkce-challenge@5.0.1:
resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==}
engines: {node: '>=16.20.0'}
@@ -4376,6 +4466,9 @@ packages:
engines: {node: '>=14.17'}
hasBin: true
+ uint8-util@2.3.2:
+ resolution: {integrity: sha512-u1c0/eiT1IW+c5+ogbmE2Fu2hmtJQPkuzDmXD/4crMU4FPMjkTkFnjJ2NN/9AsZ9YWcc7yE869uc3WB8wIZEEA==}
+
unbox-primitive@1.1.0:
resolution: {integrity: sha512-nWJ91DjeOkej/TA8pXQ3myruKpKEYgqvpw9lz4OPHj/NWFNluYrjbz9j01CJ8yKQd2g4jFoOkINCTW2I5LEEyw==}
engines: {node: '>= 0.4'}
@@ -4446,6 +4539,10 @@ packages:
weak-lru-cache@1.2.2:
resolution: {integrity: sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==}
+ web-streams-polyfill@3.3.3:
+ resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==}
+ engines: {node: '>= 8'}
+
which-boxed-primitive@1.1.1:
resolution: {integrity: sha512-TbX3mj8n0odCBFVlY8AxkqcHASw3L60jIuF8jFP78az3C2YhmGvqbHBpAjTRH2/xqYunrJ9g1jSyjCjpoWzIAA==}
engines: {node: '>= 0.4'}
@@ -4494,6 +4591,9 @@ packages:
resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==}
engines: {node: '>=12'}
+ wrappy@1.0.2:
+ resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==}
+
ws@8.21.0:
resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==}
engines: {node: '>=10.0.0'}
@@ -6333,6 +6433,10 @@ snapshots:
'@tanstack/virtual-core@3.14.0': {}
+ '@thaunknown/thirty-two@1.0.6':
+ dependencies:
+ uint8-util: 2.3.2
+
'@tybys/wasm-util@0.10.1':
dependencies:
tslib: 2.8.1
@@ -6457,7 +6561,7 @@ snapshots:
'@typescript-eslint/visitor-keys': 8.59.0
debug: 4.4.3(supports-color@10.2.2)
minimatch: 10.2.5
- semver: 7.7.4
+ semver: 7.8.5
tinyglobby: 0.2.16
ts-api-utils: 2.5.0(typescript@5.9.3)
typescript: 5.9.3
@@ -6541,6 +6645,10 @@ snapshots:
'@zeit/schemas@2.36.0': {}
+ abort-controller@3.0.0:
+ dependencies:
+ event-target-shim: 5.0.1
+
acorn-jsx@5.3.2(acorn@8.16.0):
dependencies:
acorn: 8.16.0
@@ -6674,8 +6782,16 @@ snapshots:
baseline-browser-mapping@2.10.23: {}
+ bencode@4.0.1:
+ dependencies:
+ uint8-util: 2.3.2
+
+ bep53-range@2.0.1: {}
+
blake3-wasm@2.1.5: {}
+ block-iterator@1.1.2: {}
+
bowser@2.14.1: {}
boxen@7.0.0:
@@ -6811,6 +6927,26 @@ snapshots:
cookie@1.1.1: {}
+ create-torrent@6.1.3:
+ dependencies:
+ bencode: 4.0.1
+ block-iterator: 1.1.2
+ fast-readable-async-iterator: 2.0.0
+ is-file: 1.0.0
+ join-async-iterator: 1.1.1
+ junk: 4.0.1
+ minimist: 1.2.8
+ once: 1.4.0
+ piece-length: 2.0.1
+ queue-microtask: 1.2.3
+ run-parallel: 1.2.0
+ uint8-util: 2.3.2
+
+ cross-fetch-ponyfill@1.0.3:
+ dependencies:
+ abort-controller: 3.0.0
+ node-fetch: 3.3.2
+
cross-spawn@7.0.6:
dependencies:
path-key: 3.1.1
@@ -6859,6 +6995,8 @@ snapshots:
damerau-levenshtein@1.0.8: {}
+ data-uri-to-buffer@4.0.1: {}
+
data-view-buffer@1.0.2:
dependencies:
call-bound: 1.0.4
@@ -7319,6 +7457,8 @@ snapshots:
esutils@2.0.3: {}
+ event-target-shim@5.0.1: {}
+
eventemitter3@4.0.7: {}
events@3.3.0: {}
@@ -7372,6 +7512,8 @@ snapshots:
fast-levenshtein@2.0.6: {}
+ fast-readable-async-iterator@2.0.0: {}
+
fast-uri@3.1.0: {}
fastq@1.20.1:
@@ -7382,6 +7524,11 @@ snapshots:
optionalDependencies:
picomatch: 4.0.4
+ fetch-blob@3.2.0:
+ dependencies:
+ node-domexception: 1.0.0
+ web-streams-polyfill: 3.3.3
+
figures@6.1.0:
dependencies:
is-unicode-supported: 2.1.0
@@ -7412,6 +7559,10 @@ snapshots:
dependencies:
is-callable: 1.2.7
+ formdata-polyfill@4.0.10:
+ dependencies:
+ fetch-blob: 3.2.0
+
fs-extra@11.3.4:
dependencies:
graceful-fs: 4.2.11
@@ -7461,6 +7612,8 @@ snapshots:
dunder-proto: 1.0.1
es-object-atoms: 1.1.1
+ get-stdin@10.0.0: {}
+
get-stream@6.0.1: {}
get-stream@9.0.1:
@@ -7606,6 +7759,8 @@ snapshots:
is-extglob@2.1.1: {}
+ is-file@1.0.0: {}
+
is-finalizationregistry@1.1.1:
dependencies:
call-bound: 1.0.4
@@ -7703,6 +7858,8 @@ snapshots:
jiti@2.6.1: {}
+ join-async-iterator@1.1.1: {}
+
jose@6.2.3: {}
js-tokens@4.0.0: {}
@@ -7740,6 +7897,8 @@ snapshots:
object.assign: 4.1.7
object.values: 1.2.1
+ junk@4.0.1: {}
+
keyv@4.5.4:
dependencies:
json-buffer: 3.0.1
@@ -7849,6 +8008,12 @@ snapshots:
dependencies:
'@jridgewell/sourcemap-codec': 1.5.5
+ magnet-uri@7.0.10:
+ dependencies:
+ '@thaunknown/thirty-two': 1.0.6
+ bep53-range: 2.0.1
+ uint8-util: 2.3.2
+
markdown-to-jsx@7.7.17(react@19.2.4):
optionalDependencies:
react: 19.2.4
@@ -7954,6 +8119,8 @@ snapshots:
node-addon-api@6.1.0: {}
+ node-domexception@1.0.0: {}
+
node-exports-info@1.6.0:
dependencies:
array.prototype.flatmap: 1.3.3
@@ -7961,6 +8128,12 @@ snapshots:
object.entries: 1.1.9
semver: 6.3.1
+ node-fetch@3.3.2:
+ dependencies:
+ data-uri-to-buffer: 4.0.1
+ fetch-blob: 3.2.0
+ formdata-polyfill: 4.0.10
+
node-gyp-build-optional-packages@5.2.2:
dependencies:
detect-libc: 2.1.2
@@ -8020,6 +8193,10 @@ snapshots:
on-headers@1.1.0: {}
+ once@1.4.0:
+ dependencies:
+ wrappy: 1.0.2
+
onetime@5.1.2:
dependencies:
mimic-fn: 2.1.0
@@ -8059,6 +8236,15 @@ snapshots:
parse-ms@4.0.0: {}
+ parse-torrent@11.0.24:
+ dependencies:
+ bencode: 4.0.1
+ cross-fetch-ponyfill: 1.0.3
+ get-stdin: 10.0.0
+ magnet-uri: 7.0.10
+ queue-microtask: 1.2.3
+ uint8-util: 2.3.2
+
path-exists@4.0.0: {}
path-is-inside@1.0.2: {}
@@ -8081,6 +8267,8 @@ snapshots:
picomatch@4.0.4: {}
+ piece-length@2.0.1: {}
+
pkce-challenge@5.0.1: {}
playwright-core@1.59.1: {}
@@ -8721,6 +8909,8 @@ snapshots:
typescript@5.9.3: {}
+ uint8-util@2.3.2: {}
+
unbox-primitive@1.1.0:
dependencies:
call-bound: 1.0.4
@@ -8817,6 +9007,8 @@ snapshots:
weak-lru-cache@1.2.2: {}
+ web-streams-polyfill@3.3.3: {}
+
which-boxed-primitive@1.1.1:
dependencies:
is-bigint: 1.1.0
@@ -8899,6 +9091,8 @@ snapshots:
string-width: 5.1.2
strip-ansi: 7.2.0
+ wrappy@1.0.2: {}
+
ws@8.21.0: {}
yallist@3.1.1: {}
diff --git a/scripts/archilyzer-ops.mjs b/scripts/archilyzer-ops.mjs
@@ -134,6 +134,9 @@ const ACTIONS = [
// directory, a zip read in place, a 7z ({slug, source, items?, match?,
// createRecords?, replace?, dryRun?}): one job, nothing fetched.
"attach-media",
+ // A channel's saved containers remuxed losslessly into browser-playable
+ // copies, one torrent each ({slug, ids?, trackers?, root?, dryRun?}).
+ "prepare-playable",
// A podcast channel's records completed from its RSS feed ({slug, dryRun?}):
// one fetch of the feed, no media.
"feed-metadata",
@@ -527,6 +530,14 @@ export function usage() {
" ambiguous) and the held videos with no media in it, and writes nothing.",
" The log ends with a summary: line; a re-run resumes.",
"",
+ 'prepare-playable remuxes each of a channel\'s saved containers, losslessly',
+ ' (-c copy), into a browser-playable mp4 (+faststart) or webm, and makes',
+ ' one single-file torrent per copy, under playable/ beside the saved-video',
+ ' store: {"slug"}. "ids" narrows it; "trackers": [...] is each .torrent\'s',
+ ' announce list (none by default; the infohash does not depend on it);',
+ ' "root" names another playable root. A video prepared from the same',
+ ' source (by sha256) is skipped; "dryRun": true logs each decision.',
+ "",
'fetch-windows fetches clip windows, one paced job per platform queue',
' (YouTube and Rumble side by side): {"siteId"} fetches every window the',
' site\'s published reports cite and the disk does not hold; {"items":',
