import { test } from "node:test"; import assert from "node:assert/strict"; import { chmod, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import type { Paths } from "../lib/paths"; import type { ChannelConfig } from "../lib/channelConfig"; import { persistKept } from "./persistKept"; // Run with: // pnpm --filter yt-dlp-transcript-common exec tsx --test controller/persistKeptForceMedia.test.ts // // "PERSIST KEPT" MEANS PERSIST (release 10 slice N, review L1). On a // youtube-handling channel every kept video has a transcript or captions, so // without `forceMedia` the bulk pass ran two subtitle-shaped yt-dlp passes per // video and persisted nothing — the same silent no-op "Persist source video" // had. `persistKept.test.ts` never reaches downloadOneManaged, and the e2e // "Persist kept now" check uses a transcribe fixture, so this is the one test // that fails when persistKept stops passing the flag. A separate file because // persistKept reads settings through `getPaths()`, which caches the first // SETTINGS_FILE it sees: this process sets it before anything else can. const ID = "H64QQZuw-aA"; const VIDEO = `https://www.youtube.com/watch?v=${ID}`; // The same fake as ytdlp/forceMedia.test.ts: the prefetch copies meta.json in // as the info json, the subtitle pass writes nothing, and only a media pass // (the `source-media` output template) writes a container. const FAKE_YTDLP = `#!/usr/bin/env node const fs = require("node:fs"); const path = require("node:path"); const argv = process.argv.slice(2); const root = process.env.FAKE_ROOT; fs.appendFileSync(path.join(root, "argv.log"), JSON.stringify(argv) + "\\n"); const has = (f) => argv.includes(f); const arg = (f) => { const i = argv.indexOf(f); return i < 0 ? undefined : argv[i + 1]; }; const info = arg("--load-info-json"); const id = info ? path.basename(path.dirname(info)) : "${ID}"; const dir = path.join("data", id); fs.mkdirSync(dir, { recursive: true }); if (has("--skip-download") && has("--write-info-json") && has("--no-write-subs")) { fs.copyFileSync(path.join(root, "meta.json"), path.join(dir, "metadata.info.json")); process.exit(0); } if (has("--skip-download") && has("--write-auto-subs")) { console.log("DLOM_ARCHIVE youtube " + id); process.exit(0); } if ((arg("-o") || "").includes("source-media")) { if (fs.existsSync(path.join(root, "media-fail"))) { fs.writeFileSync(path.join(dir, "source-media.f137.mp4.part"), "half a container"); console.error("ERROR: [download] Got error: HTTP Error 403: Forbidden"); process.exit(1); } fs.writeFileSync(path.join(dir, "source-media.mp4"), "container bytes for " + id); console.log("DLOM_ARCHIVE youtube " + id); process.exit(0); } console.error("fake: unknown invocation " + argv.join(" ")); process.exit(2); `; // ONE settings file for the whole process: persistKept reads settings through // getPaths(), which caches the first SETTINGS_FILE it sees, so a per-test file // would be gone by the second test. Disk floor off, so the per-item gate cannot // turn a run into a skip on a full disk. const SETTINGS_ROOT = await mkdtemp(path.join(tmpdir(), "persist-kept-settings-")); const SETTINGS_FILE = path.join(SETTINGS_ROOT, "settings.json"); await writeFile(SETTINGS_FILE, JSON.stringify({ minFreeDiskGB: 0 })); process.env.SETTINGS_FILE = SETTINGS_FILE; process.env.TRANSCRIPTS_DIR = SETTINGS_ROOT; const TRANSCRIPT = '{"transcription":[{"text":"kept"}]}'; // A youtube-handling channel keeping its latest video, which has a transcript; // persistKept over it. `mediaFail`: the media pass dies part-way with a 403. async function runPersistKept(opts: { mediaFail?: boolean } = {}) { const root = await mkdtemp(path.join(tmpdir(), "persist-kept-force-")); process.env.FAKE_ROOT = root; const ytdlp = path.join(root, "fake-ytdlp.cjs"); await writeFile(ytdlp, FAKE_YTDLP); await chmod(ytdlp, 0o755); if (opts.mediaFail) await writeFile(path.join(root, "media-fail"), ""); const meta = { id: ID, title: "A kept video", upload_date: "20260101", duration: 60, webpage_url: VIDEO, subtitles: {}, automatic_captions: { en: [{ url: "https://example/cap" }] }, }; await writeFile(path.join(root, "meta.json"), JSON.stringify(meta)); const paths = { transcriptsDir: root, channelsDir: path.join(root, "channels"), savedVideosDir: path.join(root, "saved-videos"), ytdlpBin: ytdlp, ffmpegBin: path.join(root, "no-ffmpeg"), ffprobeBin: path.join(root, "no-ffprobe"), } as Paths; const videoDir = path.join(paths.channelsDir, "c", "data", ID); await mkdir(videoDir, { recursive: true }); await writeFile(path.join(videoDir, "metadata.info.json"), JSON.stringify(meta)); await writeFile(path.join(videoDir, "transcript.json"), TRANSCRIPT); let log = ""; const result = await persistKept({ paths, channelSlug: "c", channelConfig: { handling: "youtube", url: "https://www.youtube.com/@c/videos", keepLatest: 1, } as ChannelConfig, onLog: (s) => { log += s; }, }); const argvs = (await readFile(path.join(root, "argv.log"), "utf8")) .split("\n") .filter(Boolean) .map((l) => JSON.parse(l) as string[]); return { result, log, argvs, videoDir, cleanup: () => rm(root, { recursive: true, force: true }), }; } test("persistKept forces the media pass: a kept youtube-handling video with a transcript is persisted, with no audio", async () => { const r = await runPersistKept(); try { assert.equal(r.result.kept, 1); assert.equal(r.result.persisted, 1); assert.match(r.log, /forceMedia: downloading the source although a transcript is on disk/); // The media pass ran: three spawns, the last without --skip-download. assert.equal(r.argvs.length, 3); assert.ok(!r.argvs[2].includes("--skip-download")); // Persisted, and nothing else: the pointer is there, the container is in // the store, no audio beside the transcript, the transcript untouched. const pointer = JSON.parse( await readFile(path.join(r.videoDir, "saved-video.json"), "utf8"), ) as { dir: string; file: string; keepReason?: string }; assert.equal(pointer.keepReason, "override"); assert.equal( await readFile(path.join(pointer.dir, pointer.file), "utf8"), `container bytes for ${ID}`, ); const files = await readdir(r.videoDir); assert.deepEqual(files.filter((f) => /^(audio|source-media)\./.test(f)), []); assert.equal(await readFile(path.join(r.videoDir, "transcript.json"), "utf8"), TRANSCRIPT); } finally { await r.cleanup(); } }); // A RETURNED DOWNLOAD IS NOT A PERSISTED SOURCE (release 11 slice O3, review // low 2). A forced media pass that fails over a transcript leaves the download // `ok`, so counting every return as `persisted` claimed a container that is // not there. test("persistKept counts a failed media pass as failed, not persisted", async () => { const r = await runPersistKept({ mediaFail: true }); try { assert.equal(r.result.kept, 1); assert.equal(r.result.persisted, 0); assert.equal(r.result.failed, 1); assert.match( r.log, new RegExp(`${ID}: persist failed — ERROR: \\[download\\] Got error: HTTP Error 403: Forbidden`), ); assert.match(r.log, /Persist kept: 0 persisted, 0 already saved, 1 failed/); const files = await readdir(r.videoDir); assert.ok(!files.includes("saved-video.json")); assert.equal(await readFile(path.join(r.videoDir, "transcript.json"), "utf8"), TRANSCRIPT); } finally { await r.cleanup(); } });