import { test } from "node:test"; import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { execa } from "execa"; import type { Paths } from "../lib/paths"; import type { ChannelConfig } from "../lib/channelConfig"; import { makeZip, type ZipFixtureEntry } from "../testing/makeZip"; import { loadSavedVideo } from "../lib/savedVideo-server"; import { savedVideoPath } from "../lib/savedVideo"; import { SAVED_VIDEOS_MARKER_FILENAME } from "../lib/savedVideoStore"; import { attachMedia, parse7zListing, type AttachMediaOptions } from "./attachMedia"; // Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test controller/attachMedia.test.ts // // A synthetic archive in the pilot's shape (release 21 D1): yt-dlp folders, // bare-id folders with description.txt, a [LOST] folder. Every byte is made // here; nothing reads a real archive or corpus. const SLUG = "demo"; const sha = (b: Buffer) => createHash("sha256").update(b).digest("hex"); const bytes = (n: number, seed: number) => Buffer.from(Array.from({ length: n }, (_, i) => (i * seed + 3) & 0xff)); const A = bytes(50_000, 7); // held, yt-dlp folder, stored const C = bytes(40_000, 11); // not held, bare id + description.txt, deflated const D = bytes(30_000, 13); // not held, yt-dlp folder with info.json const INFO_D = JSON.stringify({ id: "ddddddddddd", title: "From yt-dlp", extractor_key: "Youtube", upload_date: "20190101" }); const ENTRIES: ZipFixtureEntry[] = [ { name: "YouTube/" }, { name: "YouTube/001 - Intro - (aaaaaaaaaaa)/Intro-aaaaaaaaaaa.mkv", data: A }, { name: "YouTube/001 - Intro - (aaaaaaaaaaa)/Intro-aaaaaaaaaaa.info.json", data: '{"id":"aaaaaaaaaaa"}' }, { name: "YouTube/002 [LOST] - Gone - (bbbbbbbbbbb)/description.txt", data: "lost" }, { name: "YouTube/003 - Episode 2 - (ccccccccccc)/ccccccccccc.mp4", data: C, deflate: true }, { name: "YouTube/003 - Episode 2 - (ccccccccccc)/description.txt", data: "Published on July 13th 2018\n\nTips" }, { name: "YouTube/003 - Episode 2 - (ccccccccccc)/source.txt", data: "Video source: somewhere" }, { name: "YouTube/004 - Other - (ddddddddddd)/Other-ddddddddddd.webm", data: D }, { name: "YouTube/004 - Other - (ddddddddddd)/Other-ddddddddddd.info.json", data: INFO_D }, ]; type Ctx = { dir: string; paths: Paths; zip: string; lines: string[]; opts: (o?: Partial) => AttachMediaOptions }; async function withFixture(fn: (ctx: Ctx) => Promise, entries = ENTRIES): Promise { const dir = await mkdtemp(path.join(tmpdir(), "ttb-attach-")); const transcriptsDir = path.join(dir, "corpus"); const paths = { transcriptsDir, channelsDir: path.join(transcriptsDir, "channels"), savedVideosDir: path.join(dir, "store"), ffprobeBin: "ffprobe-not-used", } as Paths; await mkdir(paths.savedVideosDir, { recursive: true }); // Held: aaaaaaaaaaa (with media in the archive), bbbbbbbbbbb (LOST), hhhhhhhhhhh (no folder). for (const id of ["aaaaaaaaaaa", "bbbbbbbbbbb", "hhhhhhhhhhh"]) { const vd = path.join(paths.channelsDir, SLUG, "data", id); await mkdir(vd, { recursive: true }); await writeFile(path.join(vd, "metadata.info.json"), JSON.stringify({ id, title: id })); } const zip = path.join(dir, "archive", "YouTube.zip"); await mkdir(path.dirname(zip), { recursive: true }); await writeFile(zip, makeZip(entries)); const lines: string[] = []; const opts = (o: Partial = {}): AttachMediaOptions => ({ paths, slug: SLUG, channelConfig: { name: "Demo", handling: "youtube", platform: "youtube" } as unknown as ChannelConfig, source: zip, onLog: (l) => lines.push(l), deps: { probeDuration: async () => 61.5, now: () => new Date("2026-10-09T12:00:00Z") }, ...o, }); try { await fn({ dir, paths, zip, lines, opts }); } finally { await rm(dir, { recursive: true, force: true }); } } const videoDir = (paths: Paths, id: string) => path.join(paths.channelsDir, SLUG, "data", id); test("dryRun lists every class and writes nothing", async () => { await withFixture(async ({ paths, zip, opts, lines }) => { const zipBefore = sha(await readFile(zip)); const r = await attachMedia(opts({ dryRun: true })); assert.equal(r.counts.attach, 1); assert.equal(r.counts["not-held"], 2); assert.equal(r.counts.lost, 1); assert.deepEqual(r.notHeld.sort(), ["ccccccccccc", "ddddddddddd"]); assert.deepEqual(r.heldWithoutMedia, ["bbbbbbbbbbb", "hhhhhhhhhhh"]); assert.deepEqual(r.attached, []); assert.equal(await loadSavedVideo(videoDir(paths, "aaaaaaaaaaa")), null); assert.deepEqual(await readdir(paths.savedVideosDir), []); assert.equal(sha(await readFile(zip)), zipBefore, "the archive is never written"); assert.ok(lines.some((l) => l.startsWith("summary: "))); }); }); test("attaches a held video through the store, with local-archive provenance", async () => { await withFixture(async ({ paths, zip, opts }) => { const zipBefore = sha(await readFile(zip)); const r = await attachMedia(opts()); assert.deepEqual(r.attached, ["aaaaaaaaaaa"]); assert.deepEqual(r.failed, []); const pointer = await loadSavedVideo(videoDir(paths, "aaaaaaaaaaa")); assert.ok(pointer); assert.equal(pointer.file, "source-media.mkv"); assert.equal(pointer.dir, path.join(paths.savedVideosDir, SLUG, "aaaaaaaaaaa")); assert.equal(pointer.keepReason, "pin"); assert.equal(pointer.bytes, A.length); assert.equal(pointer.sha256, sha(A)); assert.deepEqual(pointer.origin, { requestedBy: "attach-media", kind: "local-archive", archive: zip, entry: "YouTube/001 - Intro - (aaaaaaaaaaa)/Intro-aaaaaaaaaaa.mkv", sha256: sha(A), attachedAt: "2026-10-09T12:00:00.000Z", }); assert.deepEqual(await readFile(savedVideoPath(pointer)), A); // No staging file left beside the store dir; nothing copied into data/. assert.deepEqual(await readdir(path.join(paths.savedVideosDir, SLUG)), ["aaaaaaaaaaa"]); assert.deepEqual((await readdir(videoDir(paths, "aaaaaaaaaaa"))).sort(), ["metadata.info.json", "saved-video.json"]); assert.equal(sha(await readFile(zip)), zipBefore); // Not held, no createRecords: listed, nothing written for them. await assert.rejects(stat(videoDir(paths, "ccccccccccc"))); }); }); test("a re-run is a no-op (already attached), and refuses over a pointer unless replace", async () => { await withFixture(async ({ paths, opts }) => { await attachMedia(opts()); const first = await loadSavedVideo(videoDir(paths, "aaaaaaaaaaa")); const again = await attachMedia(opts()); assert.deepEqual(again.attached, []); assert.deepEqual(again.alreadyAttached, ["aaaaaaaaaaa"]); assert.deepEqual(await loadSavedVideo(videoDir(paths, "aaaaaaaaaaa")), first); const replaced = await attachMedia(opts({ replace: true, deps: { now: () => new Date("2026-10-10T00:00:00Z") } })); assert.deepEqual(replaced.attached, ["aaaaaaaaaaa"]); const after = await loadSavedVideo(videoDir(paths, "aaaaaaaaaaa")); assert.equal(after?.origin?.attachedAt, "2026-10-10T00:00:00.000Z"); assert.deepEqual(await readFile(savedVideoPath(after!)), A); }); }); test("replace removes a previous container of another name", async () => { await withFixture(async ({ paths, opts }) => { // A container persisted earlier as .mp4 (a download, say). const vd = videoDir(paths, "aaaaaaaaaaa"); const storeDir = path.join(paths.savedVideosDir, SLUG, "aaaaaaaaaaa"); await mkdir(storeDir, { recursive: true }); await writeFile(path.join(storeDir, "source-media.mp4"), "old"); await writeFile( path.join(vd, "saved-video.json"), JSON.stringify({ storedAt: "x", dir: storeDir, file: "source-media.mp4", bytes: 3, keepReason: "override" }), ); const refused = await attachMedia(opts()); assert.deepEqual(refused.alreadyAttached, ["aaaaaaaaaaa"]); await attachMedia(opts({ replace: true })); assert.deepEqual((await readdir(storeDir)).sort(), ["source-media.mkv"]); }); }); test("createRecords: a record from the folder's .info.json, or from description.txt and its name", async () => { await withFixture(async ({ paths, opts }) => { const r = await attachMedia(opts({ createRecords: true })); assert.deepEqual(r.created.sort(), ["ccccccccccc", "ddddddddddd"]); assert.deepEqual(r.attached.sort(), ["aaaaaaaaaaa", "ccccccccccc", "ddddddddddd"]); const d = await readFile(path.join(videoDir(paths, "ddddddddddd"), "metadata.info.json"), "utf8"); assert.deepEqual(JSON.parse(d), JSON.parse(INFO_D)); const c = JSON.parse(await readFile(path.join(videoDir(paths, "ccccccccccc"), "metadata.info.json"), "utf8")); assert.equal(c.title, "Episode 2"); assert.equal(c.upload_date, "20180713"); assert.equal(c.duration, 61.5); assert.equal(c.webpage_url, "https://www.youtube.com/watch?v=ccccccccccc"); // The deflated entry came out whole. const pc = await loadSavedVideo(videoDir(paths, "ccccccccccc")); assert.deepEqual(await readFile(savedVideoPath(pc!)), C); assert.equal(pc?.file, "source-media.mp4"); // The channel's archive file gained a line per new record, as a download's would. const archive = await readFile(path.join(paths.channelsDir, SLUG, "archive"), "utf8"); assert.deepEqual(archive.trim().split("\n").sort(), ["youtube ccccccccccc", "youtube ddddddddddd"]); }); }); test("a .info.json for another id is refused, and nothing is left behind", async () => { const entries = ENTRIES.map((e) => e.name.endsWith("Other-ddddddddddd.info.json") ? { ...e, data: '{"id":"zzzzzzzzzzz"}' } : e, ); await withFixture(async ({ paths, opts }) => { const r = await attachMedia(opts({ createRecords: true })); assert.deepEqual(r.failed.map((x) => x.id), ["ddddddddddd"]); assert.match(r.failed[0].error, /not ddddddddddd/); await assert.rejects(stat(videoDir(paths, "ddddddddddd"))); assert.deepEqual((await readdir(path.join(paths.savedVideosDir, SLUG))).sort(), ["aaaaaaaaaaa", "ccccccccccc"]); }, entries); }); test("an explicit item attaches a named file to a named id", async () => { await withFixture(async ({ paths, opts }) => { const r = await attachMedia( opts({ items: [{ id: "hhhhhhhhhhh", path: "YouTube/004 - Other - (ddddddddddd)/Other-ddddddddddd.webm" }] }), ); assert.deepEqual(r.attached, ["hhhhhhhhhhh"]); const p = await loadSavedVideo(videoDir(paths, "hhhhhhhhhhh")); assert.deepEqual(await readFile(savedVideoPath(p!)), D); assert.equal(p?.origin?.entry, "YouTube/004 - Other - (ddddddddddd)/Other-ddddddddddd.webm"); }); }); test("a directory source attaches the same way", async () => { await withFixture(async ({ dir, paths, opts }) => { const src = path.join(dir, "unpacked"); const folder = path.join(src, "001 - Intro - (aaaaaaaaaaa)"); await mkdir(folder, { recursive: true }); await writeFile(path.join(folder, "Intro-aaaaaaaaaaa.mp4"), A); const r = await attachMedia(opts({ source: src })); assert.deepEqual(r.attached, ["aaaaaaaaaaa"]); const p = await loadSavedVideo(videoDir(paths, "aaaaaaaaaaa")); assert.equal(p?.origin?.archive, src); assert.equal(p?.origin?.entry, "001 - Intro - (aaaaaaaaaaa)/Intro-aaaaaaaaaaa.mp4"); // The directory source is read, never moved. assert.deepEqual(await readFile(path.join(folder, "Intro-aaaaaaaaaaa.mp4")), A); }); }); test("guards: a store being moved stops the run; a store root that is not there refuses", async () => { await withFixture(async ({ paths, opts }) => { // The in-corpus store path is what the marker guards; point the store there. const inCorpus = { ...paths, savedVideosDir: path.join(paths.transcriptsDir, "saved-videos") } as Paths; await mkdir(inCorpus.savedVideosDir, { recursive: true }); await writeFile( path.join(paths.transcriptsDir, SAVED_VIDEOS_MARKER_FILENAME), JSON.stringify({ target: "/elsewhere", direction: "out", startedAt: "2026-10-09T00:00:00Z", phase: "copy" }), ); const held = await attachMedia(opts({ paths: inCorpus })); assert.deepEqual(held.attached, []); assert.ok(held.stopped); const gone = { ...paths, savedVideosDir: path.join(paths.transcriptsDir, "no-such-store") } as Paths; await assert.rejects(attachMedia(opts({ paths: gone })), /is not there/); await assert.rejects(stat(gone.savedVideosDir), "an unmounted store is never materialised"); }); }); test("drain stops before the next video", async () => { await withFixture(async ({ opts }) => { const drain = new AbortController(); drain.abort(); const r = await attachMedia(opts({ createRecords: true, drainSignal: drain.signal })); assert.equal(r.stopped, "drained"); assert.deepEqual(r.attached, []); }); }); test("7z listing parse, and a .7z source when 7z is installed", async (t) => { const listing = [ "Path = /x/a.7z", "Type = 7z", "", "----------", "Path = dir", "Folder = +", "Size = 0", "", "Path = dir/001 - X - (aaaaaaaaaaa)/x.mp4", "Folder = -", "Size = 50000", "Attributes = A", "", ].join("\n"); assert.deepEqual(parse7zListing(listing), [{ path: "dir/001 - X - (aaaaaaaaaaa)/x.mp4", size: 50000 }]); const probe = await execa("7z", ["i"], { reject: false }).catch(() => null); if (!probe || probe.exitCode !== 0) { t.skip("7z is not installed"); return; } await withFixture(async ({ dir, paths, opts }) => { const src = path.join(dir, "pack"); await mkdir(path.join(src, "001 - Intro [x] - (aaaaaaaaaaa)"), { recursive: true }); await writeFile(path.join(src, "001 - Intro [x] - (aaaaaaaaaaa)", "Intro-aaaaaaaaaaa.mp4"), A); const archive = path.join(dir, "pack.7z"); await execa("7z", ["a", "-bd", archive, "."], { cwd: src }); const r = await attachMedia(opts({ source: archive })); assert.deepEqual(r.attached, ["aaaaaaaaaaa"]); const p = await loadSavedVideo(videoDir(paths, "aaaaaaaaaaa")); assert.deepEqual(await readFile(savedVideoPath(p!)), A); assert.equal(p?.sha256, sha(A)); }); });