import { test } from "node:test"; import assert from "node:assert/strict"; import { lstat, mkdir, mkdtemp, readdir, readFile, readlink, rm, symlink, writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import type { Paths } from "../lib/paths"; import type { SiteSettings } from "../lib/settings"; import { writeDirMarker } from "./relocateDir"; import { inspectSavedVideosStore, relocateSavedVideos, relocatedSavedVideosDir, savedVideosMarkerPath, } from "./relocateSavedVideos"; // Run with: // pnpm --filter yt-dlp-transcript-common exec tsx --test controller/relocateSavedVideos.test.ts // // These exercise the REAL rsync binary (paths.rsyncBin -> "rsync"), exactly as // relocateChannelMedia.test.ts does — it is the same mover underneath // (relocateDir.ts), and a fake rsync would test the wiring and not the move. // Everything happens inside one mkdtemp; the corpus and the platter are // SIBLINGS, never nested, because a root inside the corpus is refused by design. type Harness = { paths: Paths; root: string; settings: SiteSettings; io: { read: () => SiteSettings; write: (next: SiteSettings) => Promise }; }; async function withTmp(fn: (h: Harness) => Promise): Promise { const dir = await mkdtemp(path.join(tmpdir(), "ttb-savedvideos-")); const transcriptsDir = path.join(dir, "corpus"); const paths = { transcriptsDir, channelsDir: path.join(transcriptsDir, "channels"), savedVideosDir: path.join(transcriptsDir, "saved-videos"), rsyncBin: "rsync", } as Paths; const root = path.join(dir, "platter"); await mkdir(paths.channelsDir, { recursive: true }); await mkdir(root, { recursive: true }); let settings = { minFreeDiskGB: 0, resumeMarginGB: 0, storage: { locations: [ { id: "cold", label: "Cold", root, autoRepoint: false }, ], defaultLocationId: "cold", }, } as unknown as SiteSettings; const io = { read: () => settings, write: async (next: SiteSettings) => { settings = next; }, }; try { await fn({ paths, root, get settings() { return settings; }, io, } as Harness); } finally { await rm(dir, { recursive: true, force: true }); } } async function seedStore(paths: Paths): Promise { const one = path.join(paths.savedVideosDir, "chan", "vid1"); await mkdir(one, { recursive: true }); await writeFile(path.join(one, "source-media.mp4"), "x".repeat(4096)); const two = path.join(paths.savedVideosDir, "chan", "vid2"); await mkdir(two, { recursive: true }); await writeFile(path.join(two, "source-media.mkv"), "y".repeat(2048)); } test("the store moves onto a location, leaves a link and records where it went", async () => { await withTmp(async (h) => { await seedStore(h.paths); const result = await relocateSavedVideos({ paths: h.paths, locationId: "cold", io: h.io, onLog: () => {}, }); const target = relocatedSavedVideosDir(h.root); assert.equal(result.target, target); assert.equal(result.files, 2); assert.equal(result.bytes, 4096 + 2048); // A symlink where the store was, pointing at the target. const st = await lstat(h.paths.savedVideosDir); assert.equal(st.isSymbolicLink(), true); assert.equal(await readlink(h.paths.savedVideosDir), target); // Every reader keeps working, through the link, unchanged. assert.equal( await readFile( path.join(h.paths.savedVideosDir, "chan", "vid1", "source-media.mp4"), "utf8", ), "x".repeat(4096), ); // The record is written only on success. assert.equal(h.io.read().storage.savedVideosLocationId, "cold"); // No marker, and no parked second copy on the volume the move freed. assert.equal( (await readdir(h.paths.transcriptsDir)).filter((n) => n.startsWith("saved-videos."), ).length, 0, ); assert.equal( await readDirMarkerExists(savedVideosMarkerPath(h.paths)), false, ); const store = await inspectSavedVideosStore(h.paths, h.io.read()); assert.equal(store.status, "ok"); assert.equal(store.locationId, "cold"); assert.equal(store.target, target); }); }); test("and back again: a real directory, the record cleared, the target reclaimed", async () => { await withTmp(async (h) => { await seedStore(h.paths); await relocateSavedVideos({ paths: h.paths, locationId: "cold", io: h.io, onLog: () => {}, }); const target = relocatedSavedVideosDir(h.root); const back = await relocateSavedVideos({ paths: h.paths, locationId: "", io: h.io, onLog: () => {}, }); assert.equal(back.locationId, ""); assert.equal(back.files, 2); assert.equal((await lstat(h.paths.savedVideosDir)).isDirectory(), true); assert.equal(h.io.read().storage.savedVideosLocationId, undefined); assert.equal(await exists(target), false); assert.equal( await readFile( path.join(h.paths.savedVideosDir, "chan", "vid2", "source-media.mkv"), "utf8", ), "y".repeat(2048), ); assert.equal( (await inspectSavedVideosStore(h.paths, h.io.read())).status, "in-place", ); }); }); // A KILLED COPY IS RESUMED, NOT RESTARTED. The marker says which phase the dead // run reached; rsync skips what is already correctly on the far side, so a copy // that died at 90 % costs a verify pass and not the other 90 % again. test("an interrupted copy resumes from its marker", async () => { await withTmp(async (h) => { await seedStore(h.paths); const target = relocatedSavedVideosDir(h.root); // What a killed copy leaves: a partial target and a phase-copy marker. await mkdir(path.join(target, "chan", "vid1"), { recursive: true }); await writeFile( path.join(target, "chan", "vid1", "source-media.mp4"), "x".repeat(4096), ); await writeDirMarker(savedVideosMarkerPath(h.paths), { target, direction: "out", startedAt: new Date().toISOString(), phase: "copy", }); // inspect() calls that in-transition — which is what every guard must see. assert.equal( (await inspectSavedVideosStore(h.paths, h.io.read())).status, "in-transition", ); const result = await relocateSavedVideos({ paths: h.paths, locationId: "cold", io: h.io, onLog: () => {}, }); assert.equal(result.resumed, true); assert.equal((await lstat(h.paths.savedVideosDir)).isSymbolicLink(), true); assert.equal( await readDirMarkerExists(savedVideosMarkerPath(h.paths)), false, ); }); }); // THE CHANNEL MOVER'S PIPELINE, SHARED (release 16 slice RM): a container on // the partial copy that the store no longer has — unpersisted while the move // was down — is mirrored away on resume instead of failing the counts forever. test("a resume removes from the copy what the store no longer has", async () => { await withTmp(async (h) => { await seedStore(h.paths); const target = relocatedSavedVideosDir(h.root); await mkdir(path.join(target, "chan", "gone"), { recursive: true }); await writeFile(path.join(target, "chan", "gone", "source-media.mp4"), "z"); await writeDirMarker(savedVideosMarkerPath(h.paths), { target, direction: "out", startedAt: new Date().toISOString(), phase: "copy", }); const result = await relocateSavedVideos({ paths: h.paths, locationId: "cold", io: h.io, onLog: () => {}, }); assert.equal(result.files, 2); assert.deepEqual((await readdir(path.join(target, "chan"))).sort(), [ "vid1", "vid2", ]); }); }); // The job's first step: a store move never starts over a writer. test("a store move refuses to start while something is writing into the store", async () => { await withTmp(async (h) => { await seedStore(h.paths); await assert.rejects( () => relocateSavedVideos({ paths: h.paths, locationId: "cold", io: h.io, onLog: () => {}, busy: () => "1 running download job(s) (sync).", }), /1 running download job\(s\) \(sync\)\. Nothing has been touched\./, ); assert.equal( await readDirMarkerExists(savedVideosMarkerPath(h.paths)), false, ); assert.equal((await lstat(h.paths.savedVideosDir)).isDirectory(), true); }); }); // SAME TARGET, OPPOSITE INTENT. Finishing a move-out as a move-back would swap // the wrong way round, so it is refused by name rather than resumed. test("a marker for the other direction is refused, not resumed", async () => { await withTmp(async (h) => { await seedStore(h.paths); await writeDirMarker(savedVideosMarkerPath(h.paths), { target: relocatedSavedVideosDir(h.root), direction: "back", startedAt: new Date().toISOString(), phase: "copy", }); await assert.rejects( relocateSavedVideos({ paths: h.paths, locationId: "cold", io: h.io, onLog: () => {}, }), /in flight or was interrupted/, ); }); }); test("an unknown location is refused before anything is written", async () => { await withTmp(async (h) => { await seedStore(h.paths); await assert.rejects( relocateSavedVideos({ paths: h.paths, locationId: "nope", io: h.io, onLog: () => {}, }), /no storage location "nope"/, ); assert.equal((await lstat(h.paths.savedVideosDir)).isDirectory(), true); assert.equal( await readDirMarkerExists(savedVideosMarkerPath(h.paths)), false, ); }); }); // A ROOT INSIDE THE CORPUS WOULD COPY THE STORE ONTO ITSELF and then reclaim // the only copy — the same trap relocationRootProblem exists for on a channel. test("a root inside the corpus is refused", async () => { await withTmp(async (h) => { await seedStore(h.paths); const inside = path.join(h.paths.transcriptsDir, "inside"); await mkdir(inside, { recursive: true }); const settings = h.io.read(); await h.io.write({ ...settings, storage: { ...settings.storage, locations: [ ...settings.storage.locations, { id: "bad", label: "Bad", root: inside, autoRepoint: false }, ], }, }); await assert.rejects( relocateSavedVideos({ paths: h.paths, locationId: "bad", io: h.io, onLog: () => {}, }), /inside the corpus/, ); assert.equal((await lstat(h.paths.savedVideosDir)).isDirectory(), true); }); }); // THE RECORD IS A RECORD, NOT AN INTENTION. Settings that claim a location // while the disk says otherwise is `inconsistent` and is never guessed past. test("settings and disk disagreeing reads as inconsistent", async () => { await withTmp(async (h) => { await seedStore(h.paths); const settings = h.io.read(); await h.io.write({ ...settings, storage: { ...settings.storage, savedVideosLocationId: "cold" }, }); const store = await inspectSavedVideosStore(h.paths, h.io.read()); assert.equal(store.status, "inconsistent"); assert.match(store.detail ?? "", /is not a symlink/); }); }); async function exists(p: string): Promise { try { await lstat(p); return true; } catch { return false; } } async function readDirMarkerExists(p: string): Promise { return exists(p); } // A STORE THAT HAS NEVER EXISTED IS NOT AN RSYNC ERROR. Nothing has ever been // persisted on a corpus whose keep-latest window is unset, so there is no // `saved-videos` directory at all — and rsync exits 23 on a missing source, // which the first cut reported as "rsync failed (exit 23)" AFTER writing the // marker, leaving the store stuck in transition over a directory that was never // there. test("a store that has never existed moves cleanly, leaving an empty one", async () => { await withTmp(async (h) => { // Deliberately no seedStore. const result = await relocateSavedVideos({ paths: h.paths, locationId: "cold", io: h.io, onLog: () => {}, }); assert.equal(result.files, 0); const target = relocatedSavedVideosDir(h.root); assert.equal(await readlink(h.paths.savedVideosDir), target); assert.equal(h.io.read().storage.savedVideosLocationId, "cold"); assert.equal(await exists(savedVideosMarkerPath(h.paths)), false); // And the first persist after the move lands on the platter, through the // link, with nothing special asked of it. await mkdir(path.join(h.paths.savedVideosDir, "chan", "vid9"), { recursive: true, }); await writeFile( path.join(h.paths.savedVideosDir, "chan", "vid9", "source-media.mp4"), "z", ); assert.equal( await readFile(path.join(target, "chan", "vid9", "source-media.mp4"), "utf8"), "z", ); }); }); // THE TARGET IS DELETED ONLY WHEN THIS RUN CAN VOUCH FOR THE COPY STANDING IN // FOR IT. "`store` is a real directory" is not evidence: it is also what // `rsync --copy-links` of the corpus produces (WORKTREES.md documents that as // the way to carry a store into a shard) and what `inconsistent` looks like. // The old code rm -rf'd the relocated copy on the strength of it. test("move back refuses to delete a target the local copy cannot account for", async () => { await withTmp(async (h) => { // The state: settings say the store is on cold and the target holds two // files, but `saved-videos` is a REAL directory holding only one. const target = relocatedSavedVideosDir(h.root); await mkdir(path.join(target, "chan", "vid1"), { recursive: true }); await writeFile( path.join(target, "chan", "vid1", "source-media.mp4"), "x".repeat(4096), ); await mkdir(path.join(target, "chan", "vid2"), { recursive: true }); await writeFile( path.join(target, "chan", "vid2", "source-media.mkv"), "y".repeat(2048), ); await mkdir(path.join(h.paths.savedVideosDir, "chan", "vid1"), { recursive: true, }); await writeFile( path.join(h.paths.savedVideosDir, "chan", "vid1", "source-media.mp4"), "x".repeat(4096), ); const settings = h.io.read(); await h.io.write({ ...settings, storage: { ...settings.storage, savedVideosLocationId: "cold" }, }); const lines: string[] = []; const result = await relocateSavedVideos({ paths: h.paths, locationId: "", io: h.io, onLog: (l) => lines.push(l), }); // The reversible half succeeded: the record is cleared and the store is a // real directory. The DELETE is what was refused. assert.equal(h.io.read().storage.savedVideosLocationId, undefined); assert.equal(await isDir(target), true); assert.match(lines.join("\n"), /was NOT deleted/); assert.equal(result.locationId, ""); // No stuck marker over it. assert.equal(await exists(savedVideosMarkerPath(h.paths)), false); }); }); test("move back does delete the target once the local copy accounts for it", async () => { await withTmp(async (h) => { await seedStore(h.paths); await relocateSavedVideos({ paths: h.paths, locationId: "cold", io: h.io, onLog: () => {}, }); const target = relocatedSavedVideosDir(h.root); await relocateSavedVideos({ paths: h.paths, locationId: "", io: h.io, onLog: () => {}, }); assert.equal(await exists(target), false); }); }); // A RERUN MUST NOT TRIP OVER THE LAST ATTEMPT'S PARKED COPY. A fixed // `saved-videos.relocated` name meant a rename onto a non-empty directory // (ENOTEMPTY); the timestamped name plus a prefix sweep is the channel mover's // answer, and it is the one that does not orphan an earlier attempt's copy on // the volume the move exists to free. test("a stale parked copy is swept, not renamed onto", async () => { await withTmp(async (h) => { await seedStore(h.paths); // What an earlier, crashed attempt left beside the store. const stale = path.join( path.dirname(h.paths.savedVideosDir), "saved-videos.relocated-1600000000000", ); await mkdir(path.join(stale, "chan", "old"), { recursive: true }); await writeFile(path.join(stale, "chan", "old", "source-media.mp4"), "old"); await relocateSavedVideos({ paths: h.paths, locationId: "cold", io: h.io, onLog: () => {}, }); // Both the stale copy and this run's parked one are gone. const left = ( await readdir(path.dirname(h.paths.savedVideosDir)) ).filter((n) => n.startsWith("saved-videos.")); assert.deepEqual(left, []); }); }); // A ROOT THAT IS A SYMLINK BACK INTO THE CORPUS is the lexical check's blind // spot, and it is the one that copies the store onto itself and then reclaims // the only copy. test("a root that only RESOLVES inside the corpus is refused", async () => { await withTmp(async (h) => { await seedStore(h.paths); const inside = path.join(h.paths.transcriptsDir, "inside"); await mkdir(inside, { recursive: true }); // A path outside the corpus that is a link to a path inside it. const sneaky = path.join(path.dirname(h.paths.transcriptsDir), "sneaky"); await symlink(inside, sneaky); const settings = h.io.read(); await h.io.write({ ...settings, storage: { ...settings.storage, locations: [ ...settings.storage.locations, { id: "sneaky", label: "Sneaky", root: sneaky, autoRepoint: false }, ], }, }); await assert.rejects( relocateSavedVideos({ paths: h.paths, locationId: "sneaky", io: h.io, onLog: () => {}, }), /inside the corpus/, ); assert.equal(await isDir(h.paths.savedVideosDir), true); assert.equal(await exists(savedVideosMarkerPath(h.paths)), false); }); }); async function isDir(p: string): Promise { try { return (await lstat(p)).isDirectory(); } catch { return false; } }