import { test } from "node:test"; import assert from "node:assert/strict"; import { lstat, mkdir, stat, mkdtemp, readFile, readlink, readdir, rm, symlink, utimes, writeFile, } from "node:fs/promises"; import { existsSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { RELOCATION_MARKER_FILENAME } from "./channelMedia"; import { TIER_RELOCATION_MARKER, channelMediaLink, relocatedMediaDir, removeMediaFile, removeVideoDirMedia, tierChannelMedia, tierLinkTarget, tierMediaFile, tierVideoDir, } from "./mediaTier-server"; // Run with: // pnpm --filter yt-dlp-transcript-common exec tsx --test lib/mediaTier-server.test.ts // // Everything happens inside one mkdtemp; nothing reads a real corpus. async function fixture() { const root = await mkdtemp(path.join(tmpdir(), "media-tier-")); const channelsDir = path.join(root, "channels"); const slug = "chan"; const videoDir = path.join(channelsDir, slug, "data", "vid1"); await mkdir(videoDir, { recursive: true }); await writeFile(path.join(videoDir, "audio.mp3"), "AUDIO"); await writeFile(path.join(videoDir, "transcript.json"), "{}"); await writeFile(path.join(videoDir, "transcript.live_chat.json"), "CHAT"); await writeFile(path.join(videoDir, "source-media.mp4"), "VIDEO"); return { root, channelsDir, slug, videoDir, paths: { channelsDir } }; } test("names: the link, the relocated root, the relative link target", () => { assert.equal(channelMediaLink({ channelsDir: "/c" }, "s"), "/c/s/media"); assert.equal(relocatedMediaDir(" /mnt/p ", "s"), "/mnt/p/s/media"); assert.equal(tierLinkTarget("v", "audio.mp3"), "../../media/v/audio.mp3"); }); test("a classic channel (no media/) leaves every file real", async () => { const f = await fixture(); assert.equal(await tierMediaFile(f.videoDir, "audio.mp3"), "left"); assert.ok((await lstat(path.join(f.videoDir, "audio.mp3"))).isFile()); assert.equal(existsSync(channelMediaLink(f.paths, f.slug)), false); await rm(f.root, { recursive: true, force: true }); }); test("tiered in place: the bytes move to media/, a relative link stays", async () => { const f = await fixture(); await mkdir(channelMediaLink(f.paths, f.slug)); assert.equal(await tierMediaFile(f.videoDir, "audio.mp3"), "tiered"); const link = path.join(f.videoDir, "audio.mp3"); assert.ok((await lstat(link)).isSymbolicLink()); assert.equal(await readlink(link), "../../media/vid1/audio.mp3"); assert.equal(await readFile(link, "utf8"), "AUDIO"); assert.equal( await readFile(path.join(f.channelsDir, f.slug, "media", "vid1", "audio.mp3"), "utf8"), "AUDIO", ); // Called again: already a link. assert.equal(await tierMediaFile(f.videoDir, "audio.mp3"), "already"); // No temp link left behind. assert.deepEqual( (await readdir(f.videoDir)).filter((n) => n.includes("tierlink")), [], ); await rm(f.root, { recursive: true, force: true }); }); test("a missing name is already; text and source-media are left", async () => { const f = await fixture(); await mkdir(channelMediaLink(f.paths, f.slug)); assert.equal(await tierMediaFile(f.videoDir, "audio.m4a"), "already"); assert.equal(await tierMediaFile(f.videoDir, "transcript.json"), "left"); assert.equal(await tierMediaFile(f.videoDir, "source-media.mp4"), "left"); assert.ok((await lstat(path.join(f.videoDir, "source-media.mp4"))).isFile()); await rm(f.root, { recursive: true, force: true }); }); test("tierVideoDir tiers the audio and the raw live chat, nothing else", async () => { const f = await fixture(); await mkdir(channelMediaLink(f.paths, f.slug)); const c = await tierVideoDir(f.videoDir); assert.deepEqual(c, { tiered: 2, left: 0, already: 0 }); assert.ok((await lstat(path.join(f.videoDir, "transcript.live_chat.json"))).isSymbolicLink()); assert.ok((await lstat(path.join(f.videoDir, "transcript.json"))).isFile()); assert.ok((await lstat(path.join(f.videoDir, "source-media.mp4"))).isFile()); assert.deepEqual(await tierVideoDir(f.videoDir), { tiered: 0, left: 0, already: 2 }); await rm(f.root, { recursive: true, force: true }); }); test("relocated: the media link points at another root; EXDEV copies (with the file's times), then links", async () => { const f = await fixture(); const platter = path.join(f.root, "platter"); const target = relocatedMediaDir(platter, f.slug); await mkdir(target, { recursive: true }); await symlink(target, channelMediaLink(f.paths, f.slug)); let copies = 0; const result = await tierMediaFile(f.videoDir, "audio.mp3", { fs: { link: async () => { const err = new Error("cross-device link not permitted") as NodeJS.ErrnoException; err.code = "EXDEV"; throw err; }, copyFileAtomic: async (src, dest) => { copies += 1; await writeFile(dest, await readFile(src)); }, }, }); assert.equal(result, "tiered"); assert.equal(copies, 1); const link = path.join(f.videoDir, "audio.mp3"); assert.ok((await lstat(link)).isSymbolicLink()); assert.equal(await readFile(link, "utf8"), "AUDIO"); assert.equal(await readFile(path.join(target, "vid1", "audio.mp3"), "utf8"), "AUDIO"); // stat (the copy) and lstat (the link) agree on the mtime. assert.equal((await stat(link)).mtimeMs, (await lstat(link)).mtimeMs); await rm(f.root, { recursive: true, force: true }); }); test("a failed move is undone: the name is a real file again, no link", async () => { const f = await fixture(); await mkdir(channelMediaLink(f.paths, f.slug)); const logs: string[] = []; const result = await tierMediaFile(f.videoDir, "audio.mp3", { onLog: (s) => logs.push(s), fs: { link: async () => { const err = new Error("no space left on device") as NodeJS.ErrnoException; err.code = "ENOSPC"; throw err; }, }, }); assert.equal(result, "left"); assert.ok((await lstat(path.join(f.videoDir, "audio.mp3"))).isFile()); assert.match(logs.join(""), /left in place: no space left/); assert.deepEqual( (await readdir(f.videoDir)).filter((n) => n.includes("tierlink")), [], ); await rm(f.root, { recursive: true, force: true }); }); test("a dangling media link (unmounted drive) creates nothing and leaves the file", async () => { const f = await fixture(); const platter = path.join(f.root, "platter-unmounted"); const target = relocatedMediaDir(platter, f.slug); await symlink(target, channelMediaLink(f.paths, f.slug)); assert.equal(await tierMediaFile(f.videoDir, "audio.mp3"), "left"); assert.deepEqual(await tierVideoDir(f.videoDir), { tiered: 0, left: 2, already: 0 }); assert.deepEqual(await tierChannelMedia(f.paths, f.slug), { tiered: 0, left: 0, already: 0 }); // Nothing materialised under the missing mountpoint. assert.equal(existsSync(platter), false); assert.ok((await lstat(path.join(f.videoDir, "audio.mp3"))).isFile()); await rm(f.root, { recursive: true, force: true }); }); test("the per-video mkdir is not recursive: a media root that vanished is not rebuilt", async () => { const f = await fixture(); const platter = path.join(f.root, "platter"); const target = relocatedMediaDir(platter, f.slug); await mkdir(target, { recursive: true }); await symlink(target, channelMediaLink(f.paths, f.slug)); const calls: unknown[][] = []; // The drive goes away between the readiness check and the mkdir. const result = await tierMediaFile(f.videoDir, "audio.mp3", { fs: { mkdir: async (...args: unknown[]) => { calls.push(args); await rm(platter, { recursive: true, force: true }); return mkdir(args[0] as string); }, }, }); assert.equal(result, "left"); assert.equal(calls.length, 1); assert.equal(calls[0].length, 1, "mkdir is called without options"); assert.equal(existsSync(platter), false); assert.ok((await lstat(path.join(f.videoDir, "audio.mp3"))).isFile()); await rm(f.root, { recursive: true, force: true }); }); test("tierChannelMedia: createMediaDir makes media/ real; since skips old dirs", async () => { const f = await fixture(); const old = path.join(f.channelsDir, f.slug, "data", "vid0"); await mkdir(old); await writeFile(path.join(old, "audio.mp3"), "OLD"); const past = new Date(Date.now() - 3_600_000); await utimes(old, past, past); const since = Date.now() - 60_000; // Without createMediaDir a classic channel tiers nothing. assert.deepEqual(await tierChannelMedia(f.paths, f.slug, { since }), { tiered: 0, left: 0, already: 0, }); const c = await tierChannelMedia(f.paths, f.slug, { since, createMediaDir: true }); assert.deepEqual(c, { tiered: 2, left: 0, already: 0 }); assert.ok((await lstat(channelMediaLink(f.paths, f.slug))).isDirectory()); assert.ok((await lstat(path.join(old, "audio.mp3"))).isFile()); // Without since, the old dir too. assert.deepEqual(await tierChannelMedia(f.paths, f.slug), { tiered: 1, left: 0, already: 2 }); await rm(f.root, { recursive: true, force: true }); }); test("removeMediaFile derefs a link: the bytes in media/ go, then the link", async () => { const f = await fixture(); await mkdir(channelMediaLink(f.paths, f.slug)); await tierVideoDir(f.videoDir); const bytes = path.join(f.channelsDir, f.slug, "media", "vid1", "audio.mp3"); assert.ok(existsSync(bytes)); await removeMediaFile(f.videoDir, "audio.mp3"); assert.equal(existsSync(bytes), false); await assert.rejects(lstat(path.join(f.videoDir, "audio.mp3"))); // A real file, and a missing one. await removeMediaFile(f.videoDir, "transcript.json"); await assert.rejects(lstat(path.join(f.videoDir, "transcript.json"))); await removeMediaFile(f.videoDir, "nope"); await rm(f.root, { recursive: true, force: true }); }); test("removeMediaFile never follows a link out of the channel's media/", async () => { const f = await fixture(); const outside = path.join(f.root, "precious.mp3"); await writeFile(outside, "KEEP"); await rm(path.join(f.videoDir, "audio.mp3")); await symlink(outside, path.join(f.videoDir, "audio.mp3")); await removeMediaFile(f.videoDir, "audio.mp3"); assert.equal(await readFile(outside, "utf8"), "KEEP"); await assert.rejects(lstat(path.join(f.videoDir, "audio.mp3"))); await rm(f.root, { recursive: true, force: true }); }); test("removeVideoDirMedia clears every link's bytes and the video's media dir", async () => { const f = await fixture(); await mkdir(channelMediaLink(f.paths, f.slug)); await tierVideoDir(f.videoDir); const tierDir = path.join(f.channelsDir, f.slug, "media", "vid1"); await writeFile(path.join(tierDir, "metadata.info.json"), "{}"); // a leftover copy await removeVideoDirMedia(f.videoDir); assert.equal(existsSync(tierDir), false); // The text is the caller's to remove with the dir. assert.ok(existsSync(path.join(f.videoDir, "transcript.json"))); await rm(f.root, { recursive: true, force: true }); }); test("the link carries the file's mtime, so an lstat answers freshness without the media drive", async () => { const f = await fixture(); await mkdir(channelMediaLink(f.paths, f.slug)); const file = path.join(f.videoDir, "transcript.live_chat.json"); const when = new Date("2026-01-02T03:04:05Z"); await utimes(file, when, when); assert.equal(await tierMediaFile(f.videoDir, "transcript.live_chat.json"), "tiered"); const l = await lstat(file); assert.ok(l.isSymbolicLink()); assert.equal(l.mtimeMs, when.getTime()); await rm(f.root, { recursive: true, force: true }); }); test("a move marker on the channel: the hook writes nothing into media/", async () => { assert.equal(TIER_RELOCATION_MARKER, RELOCATION_MARKER_FILENAME); const f = await fixture(); await mkdir(channelMediaLink(f.paths, f.slug)); await writeFile(path.join(f.channelsDir, f.slug, RELOCATION_MARKER_FILENAME), "{}"); assert.equal(await tierMediaFile(f.videoDir, "audio.mp3"), "left"); assert.deepEqual(await readdir(channelMediaLink(f.paths, f.slug)), []); assert.ok((await lstat(path.join(f.videoDir, "audio.mp3"))).isFile()); await rm(f.root, { recursive: true, force: true }); }); test("the name always resolves: a crash after the bytes land leaves the real file, and the next sweep removes the stray link", async () => { const f = await fixture(); await mkdir(channelMediaLink(f.paths, f.slug)); // A process killed between placing the bytes and the rename: its temp link // is left beside the still-real file (pid 999999 is not alive). await symlink("../../media/vid1/audio.mp3", path.join(f.videoDir, ".audio.mp3.tierlink-999999")); assert.ok((await lstat(path.join(f.videoDir, "audio.mp3"))).isFile()); const c = await tierVideoDir(f.videoDir); assert.equal(c.tiered, 2); assert.deepEqual( (await readdir(f.videoDir)).filter((n) => n.includes("tierlink")), [], ); await rm(f.root, { recursive: true, force: true }); }); test("same filesystem: the bytes are hard-linked into the tier (one inode), then the name is replaced", async () => { const f = await fixture(); await mkdir(channelMediaLink(f.paths, f.slug)); const ino = (await stat(path.join(f.videoDir, "audio.mp3"))).ino; assert.equal(await tierMediaFile(f.videoDir, "audio.mp3"), "tiered"); const bytes = path.join(f.channelsDir, f.slug, "media", "vid1", "audio.mp3"); assert.equal((await stat(bytes)).ino, ino); assert.equal((await stat(bytes)).nlink, 1, "the original name no longer holds the inode"); await rm(f.root, { recursive: true, force: true }); }); test("removeMediaFile derefs a link into ANOTHER id's media dir (a renamed video dir), and drops it when empty", async () => { const f = await fixture(); const media = channelMediaLink(f.paths, f.slug); await mkdir(path.join(media, "oldid"), { recursive: true }); await writeFile(path.join(media, "oldid", "audio.m4a"), "OLD"); await symlink("../../media/oldid/audio.m4a", path.join(f.videoDir, "audio.m4a")); await removeMediaFile(f.videoDir, "audio.m4a"); assert.equal(existsSync(path.join(media, "oldid")), false); await assert.rejects(lstat(path.join(f.videoDir, "audio.m4a"))); await rm(f.root, { recursive: true, force: true }); }); test("a dead process's half-placed bytes in media// are swept with the video dir", async () => { const f = await fixture(); const tierDir = path.join(channelMediaLink(f.paths, f.slug), "vid1"); await mkdir(tierDir, { recursive: true }); await writeFile(path.join(tierDir, ".audio.mp3.tiering-999999"), "HALF"); await tierVideoDir(f.videoDir); assert.deepEqual( (await readdir(tierDir)).filter((n) => n.includes("tiering")), [], ); await rm(f.root, { recursive: true, force: true }); });