import { test } from "node:test"; import assert from "node:assert/strict"; import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { ChannelMediaUnreachableError, ChannelTextUnreadableError, assertChannelMediaReachable, assertChannelTextReadable, channelMediaStall, channelTextStall, clearRelocationMarker, inspectChannelMedia, readRelocationMarker, relocatedDataDir, RELOCATION_MARKER_FILENAME, type ChannelMediaPaths, } from "./channelMedia"; import { relocatedMediaDir } from "./mediaTier-server"; import { recordLocationHealth, resetStorageHealth } from "./storageHealth"; import { isMediaHeld, isTextHeld, HELD_REASON } from "./channelMediaHold"; // Run with: // pnpm --filter yt-dlp-transcript-common exec tsx --test lib/channelMedia.test.ts // // Everything here happens inside one mkdtemp; nothing reads a real corpus. async function withTmp( fn: (paths: ChannelMediaPaths, root: string, dir: string) => Promise, ): Promise { const dir = await mkdtemp(path.join(tmpdir(), "ttb-media-")); const paths: ChannelMediaPaths = { channelsDir: path.join(dir, "channels") }; const root = path.join(dir, "platter"); await mkdir(paths.channelsDir, { recursive: true }); try { await fn(paths, root, dir); } finally { await rm(dir, { recursive: true, force: true }); } } async function seedChannel( paths: ChannelMediaPaths, slug: string, config: Record = {}, ): Promise { const channelDir = path.join(paths.channelsDir, slug); await mkdir(channelDir, { recursive: true }); await writeFile( path.join(channelDir, "config.json"), JSON.stringify({ url: "https://example.com/c", ...config }, null, 2), ); return channelDir; } test("relocatedDataDir (retired) and relocatedMediaDir fix their suffixes", () => { assert.equal(relocatedDataDir("/mnt/p", "alpha"), "/mnt/p/alpha/data"); assert.equal(relocatedMediaDir(" /mnt/p ", "alpha"), "/mnt/p/alpha/media"); }); test("a real data dir and no media link, no mediaDir: in-place (classic)", async () => { await withTmp(async (paths) => { const channelDir = await seedChannel(paths, "alpha"); await mkdir(path.join(channelDir, "data", "v1"), { recursive: true }); const loc = await inspectChannelMedia(paths, "alpha"); assert.equal(loc.status, "in-place"); assert.equal(loc.relocated, false); assert.equal(loc.target, undefined); assert.equal(loc.dataDir, path.join(channelDir, "data")); assert.equal(loc.mediaLink, path.join(channelDir, "media")); assert.deepEqual(loc.text, { dir: path.join(channelDir, "data"), readable: true }); await assertChannelMediaReachable(paths, "alpha"); await assertChannelTextReadable(paths, "alpha"); }); }); test("a real media/ directory on the corpus disk is in-place (tiered in place)", async () => { await withTmp(async (paths) => { const channelDir = await seedChannel(paths, "alpha"); await mkdir(path.join(channelDir, "data", "v1"), { recursive: true }); await mkdir(path.join(channelDir, "media", "v1"), { recursive: true }); const loc = await inspectChannelMedia(paths, "alpha"); assert.equal(loc.status, "in-place"); assert.equal(loc.relocated, false); }); }); test("a channel that has downloaded nothing is in-place, not an error", async () => { await withTmp(async (paths) => { await seedChannel(paths, "alpha"); const loc = await inspectChannelMedia(paths, "alpha"); assert.equal(loc.status, "in-place"); assert.equal(loc.text.readable, true); await assertChannelMediaReachable(paths, "alpha"); await assertChannelTextReadable(paths, "alpha"); }); }); async function seedRelocated(paths: ChannelMediaPaths, root: string, opts: { mount?: boolean } = {}) { const target = relocatedMediaDir(root, "alpha"); if (opts.mount !== false) await mkdir(path.join(target, "v1"), { recursive: true }); const channelDir = await seedChannel(paths, "alpha", { mediaDir: target }); await mkdir(path.join(channelDir, "data", "v1"), { recursive: true }); await symlink(target, path.join(channelDir, "media")); return { target, channelDir }; } test("a media link agreeing with mediaDir and pointing at a live dir is ok", async () => { await withTmp(async (paths, root) => { const { target } = await seedRelocated(paths, root); const loc = await inspectChannelMedia(paths, "alpha"); assert.equal(loc.status, "ok"); assert.equal(loc.relocated, true); assert.equal(loc.target, target); assert.equal(loc.text.readable, true); await assertChannelMediaReachable(paths, "alpha"); await assertChannelTextReadable(paths, "alpha"); }); }); test("a dangling media link (drive not mounted): media unreachable, text readable", async () => { await withTmp(async (paths, root) => { // The link is made WITHOUT creating the target: exactly an unmounted drive. await seedRelocated(paths, root, { mount: false }); const loc = await inspectChannelMedia(paths, "alpha"); assert.equal(loc.status, "unreachable"); assert.equal(loc.relocated, true); assert.equal(loc.text.readable, true); assert.match(loc.detail ?? "", /does not exist/); await assert.rejects( () => assertChannelMediaReachable(paths, "alpha"), (err: unknown) => { assert.ok(err instanceof ChannelMediaUnreachableError); assert.equal(err.slug, "alpha"); assert.equal(err.status, "unreachable"); assert.match(err.message, /not reachable/); return true; }, ); // THE NEW RULE: the text is on the corpus disk, so it is not held. await assertChannelTextReadable(paths, "alpha"); assert.equal(isMediaHeld(loc.status), true); assert.equal(isTextHeld(loc.status), false); }); }); test("an EMPTY mountpoint is still unreachable — the link points deep", async () => { await withTmp(async (paths, root) => { await mkdir(root, { recursive: true }); await seedRelocated(paths, root, { mount: false }); const loc = await inspectChannelMedia(paths, "alpha"); assert.equal(loc.status, "unreachable"); }); }); test("a stalled media location: answered from memory, text still readable", async () => { await withTmp(async (paths, root) => { resetStorageHealth(); try { const { target } = await seedRelocated(paths, root); recordLocationHealth({ id: "platter", label: "Platter", root }, "stalled"); const loc = await inspectChannelMedia(paths, "alpha", undefined, { fresh: true }); assert.equal(loc.status, "stalled"); assert.equal(loc.text.readable, true); assert.ok(channelMediaStall({ mediaDir: target })); assert.equal(channelTextStall({ mediaDir: target }), null); await assert.rejects(() => assertChannelMediaReachable(paths, "alpha")); await assertChannelTextReadable(paths, "alpha"); } finally { resetStorageHealth(); } }); }); test("a marker makes the channel in-transition; a media move leaves the text readable", async () => { await withTmp(async (paths, root) => { const target = relocatedMediaDir(root, "alpha"); const channelDir = await seedChannel(paths, "alpha"); await mkdir(path.join(channelDir, "data"), { recursive: true }); await writeFile( path.join(channelDir, RELOCATION_MARKER_FILENAME), JSON.stringify({ target, direction: "out", startedAt: new Date().toISOString(), phase: "copy", scope: "media", }), ); const loc = await inspectChannelMedia(paths, "alpha"); assert.equal(loc.status, "in-transition"); assert.equal(loc.marker?.phase, "copy"); assert.equal(loc.marker?.direction, "out"); assert.equal(loc.marker?.scope, "media"); assert.equal(loc.target, target); assert.equal(loc.text.readable, true); await assert.rejects( () => assertChannelMediaReachable(paths, "alpha"), ChannelMediaUnreachableError, ); await assertChannelTextReadable(paths, "alpha"); const marker = await readRelocationMarker(paths, "alpha"); assert.equal(marker?.target, target); }); }); test("a tier-migration marker holds the text too", async () => { await withTmp(async (paths, root) => { const channelDir = await seedChannel(paths, "alpha"); await mkdir(path.join(channelDir, "data"), { recursive: true }); await writeFile( path.join(channelDir, RELOCATION_MARKER_FILENAME), JSON.stringify({ target: relocatedMediaDir(root, "alpha"), direction: "out", phase: "copy", scope: "tier-migration", }), ); const loc = await inspectChannelMedia(paths, "alpha"); assert.equal(loc.status, "in-transition"); assert.equal(loc.text.readable, false); await assert.rejects( () => assertChannelTextReadable(paths, "alpha"), (err: unknown) => { assert.ok(err instanceof ChannelTextUnreadableError); assert.match(err.message, /being migrated/); return true; }, ); }); }); test("LEGACY: a data link with dataDir — held by both guards, no call to the drive", async () => { await withTmp(async (paths, root) => { const target = relocatedDataDir(root, "alpha"); await mkdir(path.join(target, "v1"), { recursive: true }); const channelDir = await seedChannel(paths, "alpha", { dataDir: target }); await symlink(target, path.join(channelDir, "data")); const loc = await inspectChannelMedia(paths, "alpha"); assert.equal(loc.status, "legacy"); assert.equal(loc.relocated, true); assert.equal(loc.target, target); assert.equal(loc.text.readable, false); assert.match(loc.detail ?? "", /archilyzer storage migrate-tier alpha/); assert.equal(isMediaHeld(loc.status), true); assert.equal(isTextHeld(loc.status), true); assert.match(HELD_REASON.legacy, /migrate-tier/); await assert.rejects( () => assertChannelMediaReachable(paths, "alpha"), (err: unknown) => { assert.ok(err instanceof ChannelMediaUnreachableError); assert.match(err.message, /migrate-tier/); return true; }, ); await assert.rejects( () => assertChannelTextReadable(paths, "alpha"), (err: unknown) => { assert.ok(err instanceof ChannelTextUnreadableError); assert.equal(err.status, "legacy"); assert.match(err.message, /migrate-tier alpha/); return true; }, ); }); }); test("LEGACY: a dangling data link (unmounted) is legacy too, never 'no videos'", async () => { await withTmp(async (paths, root) => { const target = relocatedDataDir(root, "alpha"); const channelDir = await seedChannel(paths, "alpha", { dataDir: target }); await symlink(target, path.join(channelDir, "data")); assert.equal((await inspectChannelMedia(paths, "alpha")).status, "legacy"); }); }); test("LEGACY: a data link with no dataDir, and dataDir with a real data dir", async () => { await withTmp(async (paths, root) => { const target = relocatedDataDir(root, "alpha"); await mkdir(target, { recursive: true }); const a = await seedChannel(paths, "alpha"); await symlink(target, path.join(a, "data")); assert.equal((await inspectChannelMedia(paths, "alpha")).status, "legacy"); const b = await seedChannel(paths, "beta", { dataDir: relocatedDataDir(root, "beta") }); await mkdir(path.join(b, "data"), { recursive: true }); const loc = await inspectChannelMedia(paths, "beta"); assert.equal(loc.status, "legacy"); assert.equal(loc.text.readable, false); assert.ok(channelTextStall({ dataDir: relocatedDataDir(root, "beta") }) === null); }); }); test("a media link that disagrees with mediaDir is inconsistent, never guessed past", async () => { await withTmp(async (paths, root) => { const real = relocatedMediaDir(root, "alpha"); const recorded = relocatedMediaDir(path.join(root, "other"), "alpha"); await mkdir(real, { recursive: true }); const channelDir = await seedChannel(paths, "alpha", { mediaDir: recorded }); await symlink(real, path.join(channelDir, "media")); const loc = await inspectChannelMedia(paths, "alpha"); assert.equal(loc.status, "inconsistent"); assert.match(loc.detail ?? "", /points at/); assert.equal(loc.text.readable, true); await assert.rejects( () => assertChannelMediaReachable(paths, "alpha"), ChannelMediaUnreachableError, ); await assertChannelTextReadable(paths, "alpha"); }); }); test("a media link with no mediaDir is inconsistent", async () => { await withTmp(async (paths, root) => { const target = relocatedMediaDir(root, "alpha"); await mkdir(target, { recursive: true }); const channelDir = await seedChannel(paths, "alpha"); await symlink(target, path.join(channelDir, "media")); const loc = await inspectChannelMedia(paths, "alpha"); assert.equal(loc.status, "inconsistent"); assert.equal(loc.relocated, false); assert.match(loc.detail ?? "", /records no mediaDir/); }); }); test("mediaDir with a real media/ directory on disk is inconsistent", async () => { await withTmp(async (paths, root) => { const target = relocatedMediaDir(root, "alpha"); const channelDir = await seedChannel(paths, "alpha", { mediaDir: target }); await mkdir(path.join(channelDir, "media"), { recursive: true }); const loc = await inspectChannelMedia(paths, "alpha"); assert.equal(loc.status, "inconsistent"); assert.match(loc.detail ?? "", /never moved/); }); }); test("mediaDir with no media link at all is inconsistent (link gone)", async () => { await withTmp(async (paths, root) => { const target = relocatedMediaDir(root, "alpha"); await seedChannel(paths, "alpha", { mediaDir: target }); const loc = await inspectChannelMedia(paths, "alpha"); assert.equal(loc.status, "inconsistent"); assert.match(loc.detail ?? "", /symlink is missing/); }); }); test("a data/ that is a file is inconsistent and its text unreadable", async () => { await withTmp(async (paths) => { const channelDir = await seedChannel(paths, "alpha"); await writeFile(path.join(channelDir, "data"), "not a dir"); const loc = await inspectChannelMedia(paths, "alpha"); assert.equal(loc.status, "inconsistent"); assert.equal(loc.text.readable, false); await assert.rejects(() => assertChannelTextReadable(paths, "alpha"), ChannelTextUnreadableError); }); }); test("a passed config is used verbatim; config.json is only read when it is absent", async () => { await withTmp(async (paths, root) => { const target = relocatedMediaDir(root, "alpha"); await mkdir(target, { recursive: true }); // config.json on disk says NOTHING about a relocation... const channelDir = await seedChannel(paths, "alpha"); await symlink(target, path.join(channelDir, "media")); // ...so reading it itself gives "inconsistent"... assert.equal( (await inspectChannelMedia(paths, "alpha")).status, "inconsistent", ); // ...while a caller that hands over the config it already holds gets the // answer for THAT config, with no second read. const passed = await inspectChannelMedia(paths, "alpha", { mediaDir: target, }); assert.equal(passed.status, "ok"); assert.equal(passed.target, target); // An explicit null means "I have no config" and must not silently fall back // to reading the file. const nulled = await inspectChannelMedia(paths, "alpha", null); assert.equal(nulled.status, "inconsistent"); }); }); test("a blank mediaDir or dataDir means in place", async () => { await withTmp(async (paths) => { const channelDir = await seedChannel(paths, "alpha", { dataDir: " ", mediaDir: " " }); await mkdir(path.join(channelDir, "data"), { recursive: true }); assert.equal((await inspectChannelMedia(paths, "alpha")).status, "in-place"); assert.equal( (await inspectChannelMedia(paths, "alpha", { dataDir: " ", mediaDir: "" })).status, "in-place", ); }); }); test("an unreadable or missing config.json is not a relocation", async () => { await withTmp(async (paths) => { const channelDir = path.join(paths.channelsDir, "alpha"); await mkdir(path.join(channelDir, "data"), { recursive: true }); await writeFile(path.join(channelDir, "config.json"), "{ not json"); assert.equal((await inspectChannelMedia(paths, "alpha")).status, "in-place"); }); }); test("clearRelocationMarker removes the marker and touches nothing else", async () => { await withTmp(async (paths, root) => { const { target, channelDir } = await seedRelocated(paths, root); await writeFile( path.join(channelDir, RELOCATION_MARKER_FILENAME), JSON.stringify({ target, direction: "out", phase: "swap", startedAt: "" }), ); // A marker outranks everything: the channel is in transition, which is the // dead end this escape hatch exists for when the run that wrote it is gone. assert.equal( (await inspectChannelMedia(paths, "alpha")).status, "in-transition", ); await clearRelocationMarker(paths, "alpha"); assert.equal(await readRelocationMarker(paths, "alpha"), null); // The link, the config and the media are exactly as they were — what is // left is the truth the disk was already telling underneath the marker. assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok"); // Idempotent: clearing a marker that is not there is not an error. await clearRelocationMarker(paths, "alpha"); }); }); test("a scope-less marker aimed at the retired //data holds the text (the old mover)", async () => { await withTmp(async (paths, root) => { const channelDir = await seedChannel(paths, "alpha"); await mkdir(path.join(channelDir, "data"), { recursive: true }); await writeFile( path.join(channelDir, RELOCATION_MARKER_FILENAME), JSON.stringify({ target: relocatedDataDir(root, "alpha"), direction: "out", phase: "copy" }), ); assert.equal((await inspectChannelMedia(paths, "alpha", undefined, { fresh: true })).text.readable, false); await writeFile( path.join(channelDir, RELOCATION_MARKER_FILENAME), JSON.stringify({ target: relocatedMediaDir(root, "alpha"), direction: "out", phase: "copy" }), ); assert.equal((await inspectChannelMedia(paths, "alpha", undefined, { fresh: true })).text.readable, true); }); });