commit 3af3d5ed17fd14679ae7b7e717bf297410e71035
parent 473e7f908bf1db9ec1571295c9cf4aec95da505b
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sun, 20 Sep 2026 18:12:09 -0400
saved videos: the store is movable, by the channel mover's own code
`relocateDir.ts` is `relocateChannelMedia.ts`'s core lifted out unchanged —
measure, copy with --partial, verify with a dry run AND a re-measure, tolerate
directory-mtime drift exactly once, dispatch every step past the copy on what is
ON DISK. What stayed behind is everything specific to a channel (config.dataDir,
the parked siblings, the social refusal, inspectChannelMedia), because none of
it generalises. The channel mover's 24 tests pass against the factored core.
On it: `relocateSavedVideos`. The store is the one large thing in the corpus no
channel move could ever reach — `plans/storage-locations.md` recorded it as
"follow-up, not here"; this is the follow-up. `<store>` becomes a symlink to
`<root>/saved-videos`, `savedVideoRoot()` is untouched and every reader follows
the link. The record of where it went is `settings.storage.savedVideosLocationId`
— written on success, after the link exists, and NOT fallen back to another
location when the named one is deleted: `defaultLocationId` is a preference,
this is a statement about where bytes are.
/storage grows a card with the store's size, its location, Move / Move back /
Resume / Clear marker, on the shared relocation queue key so it can never race
a channel move or a re-point — and a store move now freezes the location rows
for the same reason a re-point does.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
14 files changed, 1839 insertions(+), 301 deletions(-)
diff --git a/common/controller/relocateChannelMedia.ts b/common/controller/relocateChannelMedia.ts
@@ -2,26 +2,30 @@ import path from "node:path";
import {
access,
constants as fsConstants,
- lstat,
mkdir,
readdir,
- readFile,
- readlink,
realpath,
rename,
rm,
stat,
symlink,
unlink,
- writeFile,
} from "node:fs/promises";
-import { execa } from "execa";
-import {
- formatRsyncProgressDetail,
- parseRsyncProgress,
- rsyncProgressFraction,
-} from "../jobs/progressParsers";
import type { Paths } from "../lib/paths";
+import {
+ clearDirMarker,
+ COPY_ARGS,
+ isDirectory,
+ linkOrDirState,
+ makeProgressSink,
+ measureTree,
+ pathExists,
+ readDirMarker,
+ rsyncTree,
+ verifyCopy,
+ writeDirMarker,
+ type RelocationProgress,
+} from "./relocateDir";
import { isSocialChannel } from "../lib/channelConfig";
import { getFreeBytes } from "../lib/diskSpace";
import { getSettings } from "../lib/settings";
@@ -72,28 +76,10 @@ export type RelocateChannelMediaResult = {
retried: boolean;
};
-// WHAT A COPY IN FLIGHT LOOKS LIKE FROM OUTSIDE, once per rsync redraw.
-//
-// rsync has always printed this (the copy runs with `--info=progress2`) and it
-// has always gone nowhere but the job log, as several thousand carriage-return
-// redraws of one line. This is the same fact as a value: the job turns it into
-// the task bar on /jobs and the panel's own read-out, and the controller logs
-// one line per decile so the log afterwards says how it went without holding
-// every frame of it.
-//
-// `fraction` is against the MEASURED tree, never rsync's own percentage — see
-// parseRsyncProgress for why that one walks backwards under incremental
-// recursion.
-export type RelocationProgress = {
- bytes: number;
- totalBytes: number;
- fraction: number;
- rate: string;
- etaSeconds: number;
- // "12.3 GB of 45.6 GB · 27 % · 110.50MB/s · ETA 5:32". One wording, shared by
- // the task detail and the decile log line.
- detail: string;
-};
+// The progress shape is the shared mover's — re-exported because callers
+// (the editor's relocate job) reach for it through this module, which is the
+// one they already import.
+export type { RelocationProgress } from "./relocateDir";
type RelocateOpts = {
paths: Paths;
@@ -109,37 +95,6 @@ type RelocateOpts = {
signal?: AbortSignal;
};
-// Turn rsync's redraws into `RelocationProgress`, and into one log line per
-// decile. The decile latch is why this is a closure and not a function: "every
-// 10 %" is a fact about the run, and a copy that resumes at 80 % must not
-// re-announce the eight deciles it did not do.
-function makeProgressSink(opts: {
- totalBytes: number;
- log: (m: string) => void;
- onProgress?: (p: RelocationProgress) => void;
-}): (line: string) => void {
- let lastDecile = -1;
- return (line: string) => {
- const raw = parseRsyncProgress(line);
- if (!raw) return;
- const fraction = rsyncProgressFraction(raw, opts.totalBytes);
- const detail = formatRsyncProgressDetail(raw, opts.totalBytes, formatBytes);
- opts.onProgress?.({
- bytes: raw.bytes,
- totalBytes: opts.totalBytes,
- fraction,
- rate: raw.rate,
- etaSeconds: raw.etaSeconds,
- detail,
- });
- const decile = Math.min(10, Math.floor(fraction * 10));
- if (decile > lastDecile) {
- lastDecile = decile;
- opts.log(`Copying… ${detail}`);
- }
- };
-}
-
export type RelocationPreview = {
slug: string;
target: string;
@@ -154,57 +109,6 @@ export type RelocationPreview = {
existingPartial: boolean;
};
-// One walk, used by the preview, the space check and the verify. Follows no
-// symlinks (a relocated channel is never the SOURCE of another relocation).
-async function measureTree(
- dir: string,
-): Promise<{ bytes: number; files: number }> {
- let bytes = 0;
- let files = 0;
- const stack = [dir];
- while (stack.length > 0) {
- const cur = stack.pop() as string;
- let entries;
- try {
- entries = await readdir(cur, { withFileTypes: true });
- } catch {
- continue;
- }
- for (const e of entries) {
- const p = path.join(cur, e.name);
- if (e.isDirectory()) {
- stack.push(p);
- } else if (e.isFile()) {
- try {
- const st = await stat(p);
- bytes += st.size;
- files++;
- } catch {
- /* vanished mid-walk; the verify is what catches real drift */
- }
- }
- }
- }
- return { bytes, files };
-}
-
-async function pathExists(p: string): Promise<boolean> {
- try {
- await stat(p);
- return true;
- } catch {
- return false;
- }
-}
-
-async function isDirectory(p: string): Promise<boolean> {
- try {
- return (await stat(p)).isDirectory();
- } catch {
- return false;
- }
-}
-
// Resolved through every symlink when the path exists, lexically when it does
// not. A destination root that does not exist yet is refused elsewhere; a root
// that exists and is a symlink back into the corpus is exactly what this is for.
@@ -279,35 +183,6 @@ export async function relocationRootProblem(opts: {
return null;
}
-// WHAT `data/` ACTUALLY IS RIGHT NOW. lstat, never stat: a dangling symlink —
-// the exact state a half-finished move or an unmounted drive leaves behind —
-// reads as ABSENT through stat, and the code that then tries to create the link
-// fails with EEXIST on a path it was just told was not there.
-//
-// Every phase past the copy dispatches on this rather than on what the previous
-// phase is supposed to have done, which is what makes a rerun idempotent: the
-// marker says how far the last run GOT, the disk says what is actually there,
-// and only the disk is evidence.
-type DataDirState =
- | { kind: "missing" }
- | { kind: "real-dir" }
- | { kind: "link"; linkTarget: string }
- | { kind: "other" };
-
-async function dataDirState(p: string): Promise<DataDirState> {
- let st;
- try {
- st = await lstat(p);
- } catch {
- return { kind: "missing" };
- }
- if (st.isSymbolicLink()) {
- return { kind: "link", linkTarget: await readlink(p).catch(() => "") };
- }
- if (st.isDirectory()) return { kind: "real-dir" };
- return { kind: "other" };
-}
-
// Every leftover a crashed run can have parked next to `data/`, in one list.
// The reclaim phase sweeps ALL of them rather than the one name the run that is
// finishing happens to hold: a crash between the config write and the reclaim
@@ -331,165 +206,25 @@ async function sweepParked(
}
}
+// The channel's marker, at `channels/<slug>/.relocating.json`. The FILE is the
+// contract — see relocateDir.ts — and these three are the channel's name for it.
async function writeMarker(
paths: Paths,
slug: string,
marker: RelocationMarker,
): Promise<void> {
- const file = relocationMarkerPath(paths, slug);
- const tmp = `${file}.tmp-${process.pid}`;
- await writeFile(tmp, JSON.stringify(marker, null, 2) + "\n");
- await rename(tmp, file);
+ await writeDirMarker(relocationMarkerPath(paths, slug), marker);
}
async function clearMarker(paths: Paths, slug: string): Promise<void> {
- await rm(relocationMarkerPath(paths, slug), { force: true });
+ await clearDirMarker(relocationMarkerPath(paths, slug));
}
async function readMarkerRaw(
paths: Paths,
slug: string,
): Promise<RelocationMarker | null> {
- try {
- const raw = await readFile(relocationMarkerPath(paths, slug), "utf8");
- const r = JSON.parse(raw) as Partial<RelocationMarker>;
- if (typeof r.target !== "string") return null;
- return {
- target: r.target,
- direction: r.direction === "back" ? "back" : "out",
- startedAt: typeof r.startedAt === "string" ? r.startedAt : "",
- phase:
- r.phase === "swap" || r.phase === "reclaim"
- ? r.phase
- : ("copy" as RelocationPhase),
- };
- } catch {
- return null;
- }
-}
-
-// rsync, exactly as backupSavedVideos does it: the real binary from
-// paths.rsyncBin, cancellable, streaming its output into the job log.
-async function rsyncTree(opts: {
- paths: Paths;
- src: string;
- dest: string;
- args: string[];
- log: (m: string) => void;
- // Fed every `--info=progress2` redraw. When set, those redraws are kept OUT
- // of the log: one 131 GB copy is several thousand frames of one line, and the
- // sink writes a decile line instead. Everything rsync says that is not a
- // progress frame still goes to the log verbatim.
- progress?: (line: string) => void;
- signal?: AbortSignal;
-}): Promise<{ exitCode: number; output: string }> {
- // Trailing slash on src: copy the CONTENTS, so <src>/ -> <dest>/ and not
- // <dest>/data/. Getting this wrong is a silently nested corpus.
- const args = [...opts.args, `${opts.src}/`, `${opts.dest}/`];
- opts.log(`$ ${opts.paths.rsyncBin} ${args.join(" ")}`);
- const child = execa(opts.paths.rsyncBin, args, {
- cancelSignal: opts.signal,
- all: true,
- buffer: false,
- reject: false,
- });
- let output = "";
- child.all?.on("data", (c: Buffer) => {
- const text = c.toString("utf8");
- // The verify reads this whole buffer back (driftLines), so it is
- // accumulated verbatim whatever the log ends up holding.
- output += text;
- if (!opts.progress) {
- opts.log(text);
- return;
- }
- // rsync rewrites the progress line in place with carriage returns, so one
- // chunk carries many frames. Split on both, exactly as taskHooks does for
- // yt-dlp.
- let passedThrough = "";
- for (const part of text.split(/[\r\n]+/)) {
- if (part.trim() === "") continue;
- if (parseRsyncProgress(part) === null) {
- passedThrough += `${part}\n`;
- continue;
- }
- opts.progress(part);
- }
- if (passedThrough) opts.log(passedThrough);
- });
- const result = await child;
- return { exitCode: result.exitCode ?? 1, output };
-}
-
-// A DIRECTORY MTIME IS NOT CONTENT. `.d..t` is rsync's itemization for "this is
-// a directory and only its modification time differs" — nothing to send, and no
-// byte of the copy is in question. It is what the omnimirror move hit
-// (2026-09-13): a sidecar written into one video directory while the copy was
-// already past it bumped that directory's mtime on the SOURCE and left the
-// target's behind, the verify saw one drift line, and a 131 GB copy refused at
-// the last step with nothing actually wrong.
-const DIR_MTIME_ONLY = /^\.d\.\.t/;
-
-function driftLines(output: string): string[] {
- return output
- .split("\n")
- .map((l) => l.trim())
- .filter((l) => l.length > 0 && !l.startsWith("sending incremental"))
- .filter((l) => !/^(sent|total size|$)/.test(l));
-}
-
-// What a copy has to clear before the swap: rsync itself agrees there is
-// nothing left to send, AND the two trees measure the same. The dry run alone
-// would accept a target that is byte-identical for the wrong reason; the counts
-// alone would accept two trees of equal size with different contents.
-async function verifyCopy(opts: {
- paths: Paths;
- src: string;
- dest: string;
- log: (m: string) => void;
- signal?: AbortSignal;
-}): Promise<{ bytes: number; files: number; retried: boolean }> {
- let retried = false;
- // ONE retry, never a loop: if a second pass does not settle it, something is
- // still writing into the tree and the answer is to refuse, not to chase it.
- for (;;) {
- const { exitCode, output } = await rsyncTree({
- ...opts,
- args: ["-a", "--dry-run", "--itemize-changes"],
- });
- if (exitCode !== 0) {
- throw new Error(`Verification rsync failed (exit ${exitCode})`);
- }
- const drift = driftLines(output);
- if (drift.length === 0) break;
- if (!retried && drift.every((l) => DIR_MTIME_ONLY.test(l))) {
- retried = true;
- opts.log(
- `Verification found ${drift.length} directory timestamp(s) differing and ` +
- `no content drift — running one more rsync pass to settle them.`,
- );
- const again = await rsyncTree({ ...opts, args: ["-a"] });
- if (again.exitCode !== 0) {
- throw new Error(
- `Verification rsync failed (exit ${again.exitCode}). ` +
- `The source has NOT been touched.`,
- );
- }
- continue;
- }
- throw new Error(
- `Verification failed: ${drift.length} file(s) still differ ` +
- `(first: ${drift[0]}). The source has NOT been touched.`,
- );
- }
- const [a, b] = await Promise.all([measureTree(opts.src), measureTree(opts.dest)]);
- if (a.files !== b.files || a.bytes !== b.bytes) {
- throw new Error(
- `Verification failed: source has ${a.files} file(s)/${formatBytes(a.bytes)}, ` +
- `target has ${b.files}/${formatBytes(b.bytes)}. The source has NOT been touched.`,
- );
- }
- return { ...a, retried };
+ return readDirMarker(relocationMarkerPath(paths, slug));
}
// What the operator sees before committing to a move. Cheap enough to run on a
@@ -754,10 +489,10 @@ async function moveOut(args: {
// --partial keeps an aborted transfer resumable; -a preserves mtimes, which
// is what makes the LMDB index a no-op afterwards.
const { exitCode } = await rsyncTree({
- paths,
+ rsyncBin: paths.rsyncBin,
src: dataDir,
dest: target,
- args: ["-a", "--partial", "--info=progress2"],
+ args: COPY_ARGS,
log,
progress: makeProgressSink({
totalBytes: measured.bytes,
@@ -776,7 +511,7 @@ async function moveOut(args: {
log("Verifying the copy…");
verifyRetried ||= (
- await verifyCopy({ paths, src: dataDir, dest: target, log, signal })
+ await verifyCopy({ rsyncBin: paths.rsyncBin, src: dataDir, dest: target, log, signal })
).retried;
phase = "swap";
}
@@ -792,7 +527,7 @@ async function moveOut(args: {
// EVERY STEP BELOW OBSERVES THE DISK INSTEAD OF ASSUMING THE LAST ONE RAN.
// The marker says how far the previous attempt got; it cannot say how far
// it got THROUGH a phase, and a crash lands between any two syscalls.
- const state = await dataDirState(dataDir);
+ const state = await linkOrDirState(dataDir);
const parked = (await parkedSiblings(channelDir)).filter((p) =>
path.basename(p).startsWith("data.relocated-"),
);
@@ -806,7 +541,7 @@ async function moveOut(args: {
if (verifySrc) {
log("Verifying the copy…");
verifyRetried ||= (
- await verifyCopy({ paths, src: verifySrc, dest: target, log, signal })
+ await verifyCopy({ rsyncBin: paths.rsyncBin, src: verifySrc, dest: target, log, signal })
).retried;
}
@@ -822,7 +557,7 @@ async function moveOut(args: {
);
}
- const after = await dataDirState(dataDir);
+ const after = await linkOrDirState(dataDir);
if (
after.kind === "link" &&
path.resolve(after.linkTarget) !== path.resolve(target)
@@ -974,10 +709,10 @@ async function moveBack(args: {
phase: "copy",
});
const { exitCode } = await rsyncTree({
- paths,
+ rsyncBin: paths.rsyncBin,
src: target,
dest: incoming,
- args: ["-a", "--partial", "--info=progress2"],
+ args: COPY_ARGS,
log,
progress: makeProgressSink({
totalBytes: measured.bytes,
@@ -995,7 +730,7 @@ async function moveBack(args: {
if (exitCode !== 0) throw new Error(`rsync failed (exit ${exitCode})`);
log("Verifying the copy…");
verifyRetried ||= (
- await verifyCopy({ paths, src: target, dest: incoming, log, signal })
+ await verifyCopy({ rsyncBin: paths.rsyncBin, src: target, dest: incoming, log, signal })
).retried;
phase = "swap";
}
@@ -1013,7 +748,7 @@ async function moveBack(args: {
// the case that never happens: a crash AFTER the rename left `data/` a real
// directory, the swallowed unlink then failed on it, and the rename ENOENTed
// on an `incoming` that no longer existed — forever, on every rerun.
- const state = await dataDirState(dataDir);
+ const state = await linkOrDirState(dataDir);
if (state.kind === "real-dir") {
// The rename already committed. Nothing to swap; the leftovers are the
// reclaim's business.
diff --git a/common/controller/relocateDir.ts b/common/controller/relocateDir.ts
@@ -0,0 +1,355 @@
+import path from "node:path";
+import {
+ lstat,
+ readdir,
+ readFile,
+ readlink,
+ rename,
+ rm,
+ stat,
+ writeFile,
+} from "node:fs/promises";
+import { execa } from "execa";
+import {
+ formatRsyncProgressDetail,
+ parseRsyncProgress,
+ rsyncProgressFraction,
+} from "../jobs/progressParsers";
+import { formatBytes } from "../lib/format";
+import type {
+ RelocationDirection,
+ RelocationMarker,
+ RelocationPhase,
+} from "../lib/channelMedia";
+
+// MOVING A DIRECTORY TO ANOTHER DRIVE AND LEAVING A SYMLINK — the mechanism,
+// with no opinion about whose directory it is.
+//
+// This is `relocateChannelMedia.ts`'s core, lifted out unchanged so a SECOND
+// thing can be moved by it: the saved-video store. What stayed behind in that
+// file is everything specific to a channel — `config.dataDir`, the parked
+// `data.relocated-*` siblings, the social-channel refusal, `inspectChannelMedia`
+// — because none of it generalises and pretending it did would be the worse
+// abstraction.
+//
+// What IS shared is the part the omnimirror incident and three rounds of resume
+// bugs were paid for: measure the tree, copy with `--partial` so an abort is
+// resumable, verify with a dry run AND a re-measure, tolerate directory-mtime
+// drift exactly once, and dispatch every step past the copy on what is ON DISK
+// rather than on what the previous step was supposed to have done.
+//
+// THE SOURCE IS NEVER TOUCHED UNTIL THE COPY VERIFIES. That is the invariant
+// every caller inherits and none may weaken.
+
+// ---------------------------------------------------------------------------
+// Measuring
+// ---------------------------------------------------------------------------
+
+// One walk, used by the preview, the space check and the verify. Follows no
+// symlinks (a relocated tree is never the SOURCE of another relocation).
+export async function measureTree(
+ dir: string,
+): Promise<{ bytes: number; files: number }> {
+ let bytes = 0;
+ let files = 0;
+ const stack = [dir];
+ while (stack.length > 0) {
+ const cur = stack.pop() as string;
+ let entries;
+ try {
+ entries = await readdir(cur, { withFileTypes: true });
+ } catch {
+ continue;
+ }
+ for (const e of entries) {
+ const p = path.join(cur, e.name);
+ if (e.isDirectory()) {
+ stack.push(p);
+ } else if (e.isFile()) {
+ try {
+ const st = await stat(p);
+ bytes += st.size;
+ files++;
+ } catch {
+ /* vanished mid-walk; the verify is what catches real drift */
+ }
+ }
+ }
+ }
+ return { bytes, files };
+}
+
+export async function pathExists(p: string): Promise<boolean> {
+ try {
+ await stat(p);
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+export async function isDirectory(p: string): Promise<boolean> {
+ try {
+ return (await stat(p)).isDirectory();
+ } catch {
+ return false;
+ }
+}
+
+// WHAT A PATH ACTUALLY IS RIGHT NOW. lstat, never stat: a dangling symlink —
+// the exact state a half-finished move or an unmounted drive leaves behind —
+// reads as ABSENT through stat, and the code that then tries to create the link
+// fails with EEXIST on a path it was just told was not there.
+export type LinkOrDirState =
+ | { kind: "missing" }
+ | { kind: "real-dir" }
+ | { kind: "link"; linkTarget: string }
+ | { kind: "other" };
+
+export async function linkOrDirState(p: string): Promise<LinkOrDirState> {
+ let st;
+ try {
+ st = await lstat(p);
+ } catch {
+ return { kind: "missing" };
+ }
+ if (st.isSymbolicLink()) {
+ return { kind: "link", linkTarget: await readlink(p).catch(() => "") };
+ }
+ if (st.isDirectory()) return { kind: "real-dir" };
+ return { kind: "other" };
+}
+
+// ---------------------------------------------------------------------------
+// The marker
+// ---------------------------------------------------------------------------
+
+// THE MARKER FILE IS A CONTRACT WITH THE PROCESS THAT COMES NEXT, and it is
+// deliberately identical in shape for a channel and for the saved-video store:
+// `{ target, direction, startedAt, phase }`. A live editor mid-move has already
+// written these; a later binary must resume them, so nothing here may gain a
+// REQUIRED field. See common/lib/channelMedia.ts, which is where the type lives
+// and where the channel guards read it.
+export type DirRelocationMarker = RelocationMarker;
+
+export async function writeDirMarker(
+ file: string,
+ marker: DirRelocationMarker,
+): Promise<void> {
+ const tmp = `${file}.tmp-${process.pid}`;
+ await writeFile(tmp, JSON.stringify(marker, null, 2) + "\n");
+ await rename(tmp, file);
+}
+
+export async function clearDirMarker(file: string): Promise<void> {
+ await rm(file, { force: true });
+}
+
+export async function readDirMarker(
+ file: string,
+): Promise<DirRelocationMarker | null> {
+ try {
+ const raw = await readFile(file, "utf8");
+ const r = JSON.parse(raw) as Partial<DirRelocationMarker>;
+ if (typeof r.target !== "string") return null;
+ return {
+ target: r.target,
+ direction: (r.direction === "back" ? "back" : "out") as RelocationDirection,
+ startedAt: typeof r.startedAt === "string" ? r.startedAt : "",
+ phase:
+ r.phase === "swap" || r.phase === "reclaim"
+ ? r.phase
+ : ("copy" as RelocationPhase),
+ };
+ } catch {
+ return null;
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Progress
+// ---------------------------------------------------------------------------
+
+// WHAT A COPY IN FLIGHT LOOKS LIKE FROM OUTSIDE, once per rsync redraw.
+//
+// `fraction` is against the MEASURED tree, never rsync's own percentage — see
+// parseRsyncProgress for why that one walks backwards under incremental
+// recursion.
+export type RelocationProgress = {
+ bytes: number;
+ totalBytes: number;
+ fraction: number;
+ rate: string;
+ etaSeconds: number;
+ // "12.3 GB of 45.6 GB · 27 % · 110.50MB/s · ETA 5:32". One wording, shared by
+ // the task detail and the decile log line.
+ detail: string;
+};
+
+// Turn rsync's redraws into `RelocationProgress`, and into one log line per
+// decile. The decile latch is why this is a closure and not a function: "every
+// 10 %" is a fact about the run, and a copy that resumes at 80 % must not
+// re-announce the eight deciles it did not do.
+export function makeProgressSink(opts: {
+ totalBytes: number;
+ log: (m: string) => void;
+ onProgress?: (p: RelocationProgress) => void;
+}): (line: string) => void {
+ let lastDecile = -1;
+ return (line: string) => {
+ const raw = parseRsyncProgress(line);
+ if (!raw) return;
+ const fraction = rsyncProgressFraction(raw, opts.totalBytes);
+ const detail = formatRsyncProgressDetail(raw, opts.totalBytes, formatBytes);
+ opts.onProgress?.({
+ bytes: raw.bytes,
+ totalBytes: opts.totalBytes,
+ fraction,
+ rate: raw.rate,
+ etaSeconds: raw.etaSeconds,
+ detail,
+ });
+ const decile = Math.min(10, Math.floor(fraction * 10));
+ if (decile > lastDecile) {
+ lastDecile = decile;
+ opts.log(`Copying… ${detail}`);
+ }
+ };
+}
+
+// ---------------------------------------------------------------------------
+// rsync
+// ---------------------------------------------------------------------------
+
+// rsync, exactly as backupSavedVideos does it: the real binary from
+// paths.rsyncBin, cancellable, streaming its output into the job log.
+export async function rsyncTree(opts: {
+ rsyncBin: string;
+ src: string;
+ dest: string;
+ args: string[];
+ log: (m: string) => void;
+ // Fed every `--info=progress2` redraw. When set, those redraws are kept OUT
+ // of the log: one 131 GB copy is several thousand frames of one line, and the
+ // sink writes a decile line instead. Everything rsync says that is not a
+ // progress frame still goes to the log verbatim.
+ progress?: (line: string) => void;
+ signal?: AbortSignal;
+}): Promise<{ exitCode: number; output: string }> {
+ // Trailing slash on src: copy the CONTENTS, so <src>/ -> <dest>/ and not
+ // <dest>/data/. Getting this wrong is a silently nested corpus.
+ const args = [...opts.args, `${opts.src}/`, `${opts.dest}/`];
+ opts.log(`$ ${opts.rsyncBin} ${args.join(" ")}`);
+ const child = execa(opts.rsyncBin, args, {
+ cancelSignal: opts.signal,
+ all: true,
+ buffer: false,
+ reject: false,
+ });
+ let output = "";
+ child.all?.on("data", (c: Buffer) => {
+ const text = c.toString("utf8");
+ // The verify reads this whole buffer back (driftLines), so it is
+ // accumulated verbatim whatever the log ends up holding.
+ output += text;
+ if (!opts.progress) {
+ opts.log(text);
+ return;
+ }
+ // rsync rewrites the progress line in place with carriage returns, so one
+ // chunk carries many frames. Split on both, exactly as taskHooks does for
+ // yt-dlp.
+ let passedThrough = "";
+ for (const part of text.split(/[\r\n]+/)) {
+ if (part.trim() === "") continue;
+ if (parseRsyncProgress(part) === null) {
+ passedThrough += `${part}\n`;
+ continue;
+ }
+ opts.progress(part);
+ }
+ if (passedThrough) opts.log(passedThrough);
+ });
+ const result = await child;
+ return { exitCode: result.exitCode ?? 1, output };
+}
+
+// The copy arguments, in one place so the two movers cannot drift: `-a`
+// preserves mtimes (which is what makes the LMDB index a no-op afterwards),
+// `--partial` keeps an aborted transfer resumable, `--info=progress2` is what
+// the sink above reads.
+export const COPY_ARGS = ["-a", "--partial", "--info=progress2"];
+
+// A DIRECTORY MTIME IS NOT CONTENT. `.d..t` is rsync's itemization for "this is
+// a directory and only its modification time differs" — nothing to send, and no
+// byte of the copy is in question. It is what the omnimirror move hit
+// (2026-09-13): a sidecar written into one video directory while the copy was
+// already past it bumped that directory's mtime on the SOURCE and left the
+// target's behind, the verify saw one drift line, and a 131 GB copy refused at
+// the last step with nothing actually wrong.
+const DIR_MTIME_ONLY = /^\.d\.\.t/;
+
+function driftLines(output: string): string[] {
+ return output
+ .split("\n")
+ .map((l) => l.trim())
+ .filter((l) => l.length > 0 && !l.startsWith("sending incremental"))
+ .filter((l) => !/^(sent|total size|$)/.test(l));
+}
+
+// What a copy has to clear before the swap: rsync itself agrees there is
+// nothing left to send, AND the two trees measure the same. The dry run alone
+// would accept a target that is byte-identical for the wrong reason; the counts
+// alone would accept two trees of equal size with different contents.
+export async function verifyCopy(opts: {
+ rsyncBin: string;
+ src: string;
+ dest: string;
+ log: (m: string) => void;
+ signal?: AbortSignal;
+}): Promise<{ bytes: number; files: number; retried: boolean }> {
+ let retried = false;
+ // ONE retry, never a loop: if a second pass does not settle it, something is
+ // still writing into the tree and the answer is to refuse, not to chase it.
+ for (;;) {
+ const { exitCode, output } = await rsyncTree({
+ ...opts,
+ args: ["-a", "--dry-run", "--itemize-changes"],
+ });
+ if (exitCode !== 0) {
+ throw new Error(`Verification rsync failed (exit ${exitCode})`);
+ }
+ const drift = driftLines(output);
+ if (drift.length === 0) break;
+ if (!retried && drift.every((l) => DIR_MTIME_ONLY.test(l))) {
+ retried = true;
+ opts.log(
+ `Verification found ${drift.length} directory timestamp(s) differing and ` +
+ `no content drift — running one more rsync pass to settle them.`,
+ );
+ const again = await rsyncTree({ ...opts, args: ["-a"] });
+ if (again.exitCode !== 0) {
+ throw new Error(
+ `Verification rsync failed (exit ${again.exitCode}). ` +
+ `The source has NOT been touched.`,
+ );
+ }
+ continue;
+ }
+ throw new Error(
+ `Verification failed: ${drift.length} file(s) still differ ` +
+ `(first: ${drift[0]}). The source has NOT been touched.`,
+ );
+ }
+ const [a, b] = await Promise.all([
+ measureTree(opts.src),
+ measureTree(opts.dest),
+ ]);
+ if (a.files !== b.files || a.bytes !== b.bytes) {
+ throw new Error(
+ `Verification failed: source has ${a.files} file(s)/${formatBytes(a.bytes)}, ` +
+ `target has ${b.files}/${formatBytes(b.bytes)}. The source has NOT been touched.`,
+ );
+ }
+ return { ...a, retried };
+}
diff --git a/common/controller/relocateSavedVideos.test.ts b/common/controller/relocateSavedVideos.test.ts
@@ -0,0 +1,315 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ lstat,
+ mkdir,
+ mkdtemp,
+ readdir,
+ readFile,
+ readlink,
+ rm,
+ writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import type { SiteSettings } from "../lib/settings";
+import { writeDirMarker } from "./relocateDir";
+import {
+ inspectSavedVideosStore,
+ relocateSavedVideos,
+ relocatedSavedVideosDir,
+ savedVideosMarkerPath,
+} from "./relocateSavedVideos";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/relocateSavedVideos.test.ts
+//
+// These exercise the REAL rsync binary (paths.rsyncBin -> "rsync"), exactly as
+// relocateChannelMedia.test.ts does — it is the same mover underneath
+// (relocateDir.ts), and a fake rsync would test the wiring and not the move.
+// Everything happens inside one mkdtemp; the corpus and the platter are
+// SIBLINGS, never nested, because a root inside the corpus is refused by design.
+
+type Harness = {
+ paths: Paths;
+ root: string;
+ settings: SiteSettings;
+ io: { read: () => SiteSettings; write: (next: SiteSettings) => Promise<void> };
+};
+
+async function withTmp(fn: (h: Harness) => Promise<void>): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-savedvideos-"));
+ const transcriptsDir = path.join(dir, "corpus");
+ const paths = {
+ transcriptsDir,
+ channelsDir: path.join(transcriptsDir, "channels"),
+ savedVideosDir: path.join(transcriptsDir, "saved-videos"),
+ rsyncBin: "rsync",
+ } as Paths;
+ const root = path.join(dir, "platter");
+ await mkdir(paths.channelsDir, { recursive: true });
+ await mkdir(root, { recursive: true });
+ let settings = {
+ minFreeDiskGB: 0,
+ resumeMarginGB: 0,
+ storage: {
+ locations: [
+ { id: "cold", label: "Cold", root, autoRepoint: false },
+ ],
+ defaultLocationId: "cold",
+ },
+ } as unknown as SiteSettings;
+ const io = {
+ read: () => settings,
+ write: async (next: SiteSettings) => {
+ settings = next;
+ },
+ };
+ try {
+ await fn({
+ paths,
+ root,
+ get settings() {
+ return settings;
+ },
+ io,
+ } as Harness);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+async function seedStore(paths: Paths): Promise<void> {
+ const one = path.join(paths.savedVideosDir, "chan", "vid1");
+ await mkdir(one, { recursive: true });
+ await writeFile(path.join(one, "source-media.mp4"), "x".repeat(4096));
+ const two = path.join(paths.savedVideosDir, "chan", "vid2");
+ await mkdir(two, { recursive: true });
+ await writeFile(path.join(two, "source-media.mkv"), "y".repeat(2048));
+}
+
+test("the store moves onto a location, leaves a link and records where it went", async () => {
+ await withTmp(async (h) => {
+ await seedStore(h.paths);
+ const result = await relocateSavedVideos({
+ paths: h.paths,
+ locationId: "cold",
+ io: h.io,
+ onLog: () => {},
+ });
+ const target = relocatedSavedVideosDir(h.root);
+ assert.equal(result.target, target);
+ assert.equal(result.files, 2);
+ assert.equal(result.bytes, 4096 + 2048);
+
+ // A symlink where the store was, pointing at the target.
+ const st = await lstat(h.paths.savedVideosDir);
+ assert.equal(st.isSymbolicLink(), true);
+ assert.equal(await readlink(h.paths.savedVideosDir), target);
+ // Every reader keeps working, through the link, unchanged.
+ assert.equal(
+ await readFile(
+ path.join(h.paths.savedVideosDir, "chan", "vid1", "source-media.mp4"),
+ "utf8",
+ ),
+ "x".repeat(4096),
+ );
+ // The record is written only on success.
+ assert.equal(h.io.read().storage.savedVideosLocationId, "cold");
+ // No marker, and no parked second copy on the volume the move freed.
+ assert.equal(
+ (await readdir(h.paths.transcriptsDir)).filter((n) =>
+ n.startsWith("saved-videos."),
+ ).length,
+ 0,
+ );
+ assert.equal(
+ await readDirMarkerExists(savedVideosMarkerPath(h.paths)),
+ false,
+ );
+
+ const store = await inspectSavedVideosStore(h.paths, h.io.read());
+ assert.equal(store.status, "ok");
+ assert.equal(store.locationId, "cold");
+ assert.equal(store.target, target);
+ });
+});
+
+test("and back again: a real directory, the record cleared, the target reclaimed", async () => {
+ await withTmp(async (h) => {
+ await seedStore(h.paths);
+ await relocateSavedVideos({
+ paths: h.paths,
+ locationId: "cold",
+ io: h.io,
+ onLog: () => {},
+ });
+ const target = relocatedSavedVideosDir(h.root);
+ const back = await relocateSavedVideos({
+ paths: h.paths,
+ locationId: "",
+ io: h.io,
+ onLog: () => {},
+ });
+ assert.equal(back.locationId, "");
+ assert.equal(back.files, 2);
+ assert.equal((await lstat(h.paths.savedVideosDir)).isDirectory(), true);
+ assert.equal(h.io.read().storage.savedVideosLocationId, undefined);
+ assert.equal(await exists(target), false);
+ assert.equal(
+ await readFile(
+ path.join(h.paths.savedVideosDir, "chan", "vid2", "source-media.mkv"),
+ "utf8",
+ ),
+ "y".repeat(2048),
+ );
+ assert.equal(
+ (await inspectSavedVideosStore(h.paths, h.io.read())).status,
+ "in-place",
+ );
+ });
+});
+
+// A KILLED COPY IS RESUMED, NOT RESTARTED. The marker says which phase the dead
+// run reached; rsync skips what is already correctly on the far side, so a copy
+// that died at 90 % costs a verify pass and not the other 90 % again.
+test("an interrupted copy resumes from its marker", async () => {
+ await withTmp(async (h) => {
+ await seedStore(h.paths);
+ const target = relocatedSavedVideosDir(h.root);
+ // What a killed copy leaves: a partial target and a phase-copy marker.
+ await mkdir(path.join(target, "chan", "vid1"), { recursive: true });
+ await writeFile(
+ path.join(target, "chan", "vid1", "source-media.mp4"),
+ "x".repeat(4096),
+ );
+ await writeDirMarker(savedVideosMarkerPath(h.paths), {
+ target,
+ direction: "out",
+ startedAt: new Date().toISOString(),
+ phase: "copy",
+ });
+ // inspect() calls that in-transition — which is what every guard must see.
+ assert.equal(
+ (await inspectSavedVideosStore(h.paths, h.io.read())).status,
+ "in-transition",
+ );
+
+ const result = await relocateSavedVideos({
+ paths: h.paths,
+ locationId: "cold",
+ io: h.io,
+ onLog: () => {},
+ });
+ assert.equal(result.resumed, true);
+ assert.equal((await lstat(h.paths.savedVideosDir)).isSymbolicLink(), true);
+ assert.equal(
+ await readDirMarkerExists(savedVideosMarkerPath(h.paths)),
+ false,
+ );
+ });
+});
+
+// SAME TARGET, OPPOSITE INTENT. Finishing a move-out as a move-back would swap
+// the wrong way round, so it is refused by name rather than resumed.
+test("a marker for the other direction is refused, not resumed", async () => {
+ await withTmp(async (h) => {
+ await seedStore(h.paths);
+ await writeDirMarker(savedVideosMarkerPath(h.paths), {
+ target: relocatedSavedVideosDir(h.root),
+ direction: "back",
+ startedAt: new Date().toISOString(),
+ phase: "copy",
+ });
+ await assert.rejects(
+ relocateSavedVideos({
+ paths: h.paths,
+ locationId: "cold",
+ io: h.io,
+ onLog: () => {},
+ }),
+ /in flight or was interrupted/,
+ );
+ });
+});
+
+test("an unknown location is refused before anything is written", async () => {
+ await withTmp(async (h) => {
+ await seedStore(h.paths);
+ await assert.rejects(
+ relocateSavedVideos({
+ paths: h.paths,
+ locationId: "nope",
+ io: h.io,
+ onLog: () => {},
+ }),
+ /no storage location "nope"/,
+ );
+ assert.equal((await lstat(h.paths.savedVideosDir)).isDirectory(), true);
+ assert.equal(
+ await readDirMarkerExists(savedVideosMarkerPath(h.paths)),
+ false,
+ );
+ });
+});
+
+// A ROOT INSIDE THE CORPUS WOULD COPY THE STORE ONTO ITSELF and then reclaim
+// the only copy — the same trap relocationRootProblem exists for on a channel.
+test("a root inside the corpus is refused", async () => {
+ await withTmp(async (h) => {
+ await seedStore(h.paths);
+ const inside = path.join(h.paths.transcriptsDir, "inside");
+ await mkdir(inside, { recursive: true });
+ const settings = h.io.read();
+ await h.io.write({
+ ...settings,
+ storage: {
+ ...settings.storage,
+ locations: [
+ ...settings.storage.locations,
+ { id: "bad", label: "Bad", root: inside, autoRepoint: false },
+ ],
+ },
+ });
+ await assert.rejects(
+ relocateSavedVideos({
+ paths: h.paths,
+ locationId: "bad",
+ io: h.io,
+ onLog: () => {},
+ }),
+ /inside the corpus/,
+ );
+ assert.equal((await lstat(h.paths.savedVideosDir)).isDirectory(), true);
+ });
+});
+
+// THE RECORD IS A RECORD, NOT AN INTENTION. Settings that claim a location
+// while the disk says otherwise is `inconsistent` and is never guessed past.
+test("settings and disk disagreeing reads as inconsistent", async () => {
+ await withTmp(async (h) => {
+ await seedStore(h.paths);
+ const settings = h.io.read();
+ await h.io.write({
+ ...settings,
+ storage: { ...settings.storage, savedVideosLocationId: "cold" },
+ });
+ const store = await inspectSavedVideosStore(h.paths, h.io.read());
+ assert.equal(store.status, "inconsistent");
+ assert.match(store.detail ?? "", /is not a symlink/);
+ });
+});
+
+async function exists(p: string): Promise<boolean> {
+ try {
+ await lstat(p);
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+async function readDirMarkerExists(p: string): Promise<boolean> {
+ return exists(p);
+}
diff --git a/common/controller/relocateSavedVideos.ts b/common/controller/relocateSavedVideos.ts
@@ -0,0 +1,572 @@
+import path from "node:path";
+import {
+ access,
+ constants as fsConstants,
+ mkdir,
+ readdir,
+ rename,
+ rm,
+ symlink,
+ unlink,
+} from "node:fs/promises";
+import type { Paths } from "../lib/paths";
+import { getFreeBytes } from "../lib/diskSpace";
+import { formatBytes } from "../lib/format";
+import {
+ getSettings,
+ writeSettings,
+ type SiteSettings,
+} from "../lib/settings";
+import type { RelocationMarker, RelocationPhase } from "../lib/channelMedia";
+import {
+ clearDirMarker,
+ COPY_ARGS,
+ isDirectory,
+ linkOrDirState,
+ makeProgressSink,
+ measureTree,
+ readDirMarker,
+ rsyncTree,
+ verifyCopy,
+ writeDirMarker,
+ type RelocationProgress,
+} from "./relocateDir";
+
+// MOVING THE SAVED-VIDEO STORE TO ANOTHER DRIVE.
+//
+// The store (`transcripts/saved-videos`) is where a persisted source container
+// goes when the per-download retention rule decides to keep one — the single
+// largest thing in the corpus that is not a channel's `data/`, and the one
+// directory a channel move could never reach. `plans/storage-locations.md`
+// recorded it as "follow-up, not here"; this is the follow-up.
+//
+// SAME MECHANISM, SAME INVARIANT. `<store>` becomes an absolute SYMLINK to
+// `<root>/saved-videos` and the copy is verified before the source is touched,
+// exactly as a channel's `data/` does it — `relocateDir.ts` is literally the
+// same code. Nothing that reads the store has to know: `savedVideoRoot()`
+// (lib/savedVideo.ts) returns `paths.savedVideosDir` as it always did and the
+// kernel follows the link.
+//
+// WHAT IS DIFFERENT FROM A CHANNEL, and it is one thing: the record of where it
+// went is `settings.storage.savedVideosLocationId`, not a per-channel config
+// field. So the swap's last step is a settings write rather than a
+// writeChannelConfig, and a rollback has one file to put back instead of n.
+//
+// THE PER-CHANNEL OVERRIDE IS NOT MOVED. `ChannelConfig.savedVideosDir` lets a
+// channel point its store somewhere else entirely; a channel that has one is
+// not in this store and this move neither reads nor rewrites it. That is
+// deliberate — it is an absolute path the operator set, and silently
+// re-anchoring it would be this function deciding something it was not asked.
+
+export const SAVED_VIDEOS_DIRNAME = "saved-videos";
+export const SAVED_VIDEOS_MARKER_FILENAME = ".relocating-saved-videos.json";
+
+export function savedVideosMarkerPath(paths: Paths): string {
+ return path.join(paths.transcriptsDir, SAVED_VIDEOS_MARKER_FILENAME);
+}
+
+// The store's home on a location: `<root>/saved-videos`. Flat, beside the
+// channels' `<slug>/data` dirs, and not configurable for the same reason
+// `relocatedDataDir` is not — a mover recognises a target by its shape.
+export function relocatedSavedVideosDir(root: string): string {
+ return path.join(root.trim(), SAVED_VIDEOS_DIRNAME);
+}
+
+export type SavedVideosStoreStatus =
+ // The store is a real directory under the corpus.
+ | "in-place"
+ // Relocated, link and settings agree, and the target is a reachable dir.
+ | "ok"
+ // Relocated, but the target is not there — almost always an unmounted drive.
+ | "unreachable"
+ // A move is in flight, or one was interrupted: the marker is present.
+ | "in-transition"
+ // Disk and settings disagree, in either direction. Never guessed past.
+ | "inconsistent";
+
+export type SavedVideosStore = {
+ // Always `paths.savedVideosDir` — the path every reader uses, moved or not.
+ dir: string;
+ // `settings.storage.savedVideosLocationId`, or "" for in place.
+ locationId: string;
+ // Where the bytes actually are (the link target), when it is relocated.
+ target?: string;
+ status: SavedVideosStoreStatus;
+ detail?: string;
+ marker?: RelocationMarker;
+};
+
+// WHAT THE STORE IS RIGHT NOW, from the disk first and the settings second.
+//
+// The same shape and the same discipline as `inspectChannelMedia`: lstat (never
+// stat, so a dangling link is not read as absent), and any disagreement between
+// the link and the record is `inconsistent` rather than a guess.
+export async function inspectSavedVideosStore(
+ paths: Paths,
+ settings?: SiteSettings,
+): Promise<SavedVideosStore> {
+ const s = settings ?? getSettings();
+ const dir = paths.savedVideosDir;
+ const locationId = s.storage.savedVideosLocationId ?? "";
+ const marker = await readDirMarker(savedVideosMarkerPath(paths));
+ const state = await linkOrDirState(dir);
+ const loc = s.storage.locations.find((l) => l.id === locationId);
+ const expected = loc ? relocatedSavedVideosDir(loc.root) : "";
+
+ if (marker) {
+ return {
+ dir,
+ locationId,
+ ...(state.kind === "link" ? { target: state.linkTarget } : {}),
+ status: "in-transition",
+ detail: `a move (${marker.direction}) to ${marker.target} is in flight or was interrupted at phase "${marker.phase}"`,
+ marker,
+ };
+ }
+ if (state.kind === "link") {
+ const target = state.linkTarget;
+ if (expected && path.resolve(target) !== path.resolve(expected)) {
+ return {
+ dir,
+ locationId,
+ target,
+ status: "inconsistent",
+ detail: `the store links to ${target}, but settings record location "${locationId}" at ${expected}`,
+ };
+ }
+ if (!expected) {
+ return {
+ dir,
+ locationId,
+ target,
+ status: "inconsistent",
+ detail: `the store links to ${target}, but no storage location is recorded for it`,
+ };
+ }
+ if (!(await isDirectory(target))) {
+ return {
+ dir,
+ locationId,
+ target,
+ status: "unreachable",
+ detail: `${target} is not reachable (is the drive mounted?)`,
+ };
+ }
+ return { dir, locationId, target, status: "ok" };
+ }
+ if (expected) {
+ return {
+ dir,
+ locationId,
+ status: "inconsistent",
+ detail: `settings record the store on location "${locationId}", but ${dir} is not a symlink`,
+ };
+ }
+ if (state.kind === "other") {
+ return {
+ dir,
+ locationId: "",
+ status: "inconsistent",
+ detail: `${dir} is neither a directory nor a symlink`,
+ };
+ }
+ return { dir, locationId: "", status: "in-place" };
+}
+
+export type SavedVideosRelocateResult = {
+ // "" when the store was moved back in place.
+ locationId: string;
+ target: string;
+ bytes: number;
+ files: number;
+ resumed: boolean;
+ retried: boolean;
+};
+
+type Opts = {
+ paths: Paths;
+ // "" moves the store back in place; any other value is a location id.
+ locationId: string;
+ onLog?: (line: string) => void;
+ onProgress?: (p: RelocationProgress) => void;
+ signal?: AbortSignal;
+ // Injectable for unit tests, exactly as the storage controller does it.
+ io?: { read: () => SiteSettings; write: (next: SiteSettings) => Promise<void> };
+};
+
+const BYTES_PER_GB = 1024 ** 3;
+
+export async function relocateSavedVideos(
+ opts: Opts,
+): Promise<SavedVideosRelocateResult> {
+ const io = opts.io ?? { read: getSettings, write: writeSettings };
+ const log = opts.onLog ?? ((m: string) => console.log(m));
+ const paths = opts.paths;
+ const markerFile = savedVideosMarkerPath(paths);
+ const existing = await readDirMarker(markerFile);
+ const wantBack = opts.locationId.trim() === "";
+ const direction = wantBack ? "back" : "out";
+
+ // A MARKER FOR THE OTHER DIRECTION IS NEVER RESUMED INTO THIS ONE. Same
+ // target, opposite intent — the channel mover refuses this for the reason
+ // that finishing a move-out as a move-back swaps the wrong way round.
+ if (existing && existing.direction !== direction) {
+ throw new Error(
+ `A saved-video store move (${existing.direction}) to ${existing.target} is ` +
+ `in flight or was interrupted — finish that before moving ${direction}.`,
+ );
+ }
+ return wantBack
+ ? moveStoreBack({ ...opts, io, log, markerFile, resume: existing })
+ : moveStoreOut({ ...opts, io, log, markerFile, resume: existing });
+}
+
+type Inner = Opts & {
+ io: NonNullable<Opts["io"]>;
+ log: (m: string) => void;
+ markerFile: string;
+ resume: RelocationMarker | null;
+};
+
+async function stampMarker(
+ file: string,
+ target: string,
+ direction: "out" | "back",
+ phase: RelocationPhase,
+): Promise<void> {
+ await writeDirMarker(file, {
+ target,
+ direction,
+ startedAt: new Date().toISOString(),
+ phase,
+ });
+}
+
+async function moveStoreOut(a: Inner): Promise<SavedVideosRelocateResult> {
+ const { paths, log, markerFile, io } = a;
+ const settings = io.read();
+ const loc = settings.storage.locations.find(
+ (l) => l.id === a.locationId.trim(),
+ );
+ if (!loc) {
+ throw new Error(
+ `There is no storage location "${a.locationId}". Add it on /storage.`,
+ );
+ }
+ const store = paths.savedVideosDir;
+ const target = relocatedSavedVideosDir(loc.root);
+ if (a.resume && a.resume.target !== target) {
+ throw new Error(
+ `A move to ${a.resume.target} is already in progress — finish or clear it ` +
+ `before moving to ${target}.`,
+ );
+ }
+
+ // PREFLIGHT. Everything that can refuse does so before a byte is written and
+ // before the marker exists.
+ if (!(await isDirectory(loc.root))) {
+ throw new Error(
+ `The destination root ${loc.root} does not exist or is not a directory ` +
+ `(is the drive mounted?)`,
+ );
+ }
+ try {
+ await access(loc.root, fsConstants.W_OK);
+ } catch {
+ throw new Error(`The destination root ${loc.root} is not writable`);
+ }
+ if (isWithin(paths.transcriptsDir, loc.root)) {
+ throw new Error(
+ `The destination root ${loc.root} is inside the corpus at ` +
+ `${paths.transcriptsDir} — the store would be copied onto itself and ` +
+ `then reclaimed. Pick a directory on the other drive.`,
+ );
+ }
+ const state = await linkOrDirState(store);
+ if (!a.resume && state.kind !== "real-dir" && state.kind !== "missing") {
+ throw new Error(
+ `${store} is not a real directory (it is a ${state.kind}) — the store is ` +
+ `already moved, or something else is there.`,
+ );
+ }
+
+ let phase: RelocationPhase = a.resume?.phase ?? "copy";
+ const measured = await measureTree(store);
+ let retried = false;
+ log(
+ `Moving the saved-video store: ${measured.files} file(s), ` +
+ `${formatBytes(measured.bytes)} -> ${target}`,
+ );
+
+ if (phase === "copy") {
+ const marginGB =
+ settings.minFreeDiskGB > 0 ? settings.resumeMarginGB : 0;
+ const needed = measured.bytes + marginGB * BYTES_PER_GB;
+ const free = await getFreeBytes(loc.root);
+ if (free < needed) {
+ throw new Error(
+ `Not enough space on ${loc.root}: ${formatBytes(free)} free, ` +
+ `${formatBytes(measured.bytes)} to move` +
+ (marginGB > 0
+ ? ` plus a ${marginGB} GB resume margin = ${formatBytes(needed)} required`
+ : " required"),
+ );
+ }
+ await mkdir(target, { recursive: true });
+ await stampMarker(markerFile, target, "out", "copy");
+ const { exitCode } = await rsyncTree({
+ rsyncBin: paths.rsyncBin,
+ src: store,
+ dest: target,
+ args: COPY_ARGS,
+ log,
+ progress: makeProgressSink({
+ totalBytes: measured.bytes,
+ log,
+ onProgress: a.onProgress,
+ }),
+ signal: a.signal,
+ });
+ if (a.signal?.aborted) {
+ throw new Error(
+ `Cancelled. ${store} is untouched and the partial copy at ${target} is ` +
+ `resumable — rerun to continue.`,
+ );
+ }
+ if (exitCode !== 0) throw new Error(`rsync failed (exit ${exitCode})`);
+ log("Verifying the copy…");
+ retried ||= (
+ await verifyCopy({
+ rsyncBin: paths.rsyncBin,
+ src: store,
+ dest: target,
+ log,
+ signal: a.signal,
+ })
+ ).retried;
+ phase = "swap";
+ }
+
+ if (phase === "swap") {
+ await stampMarker(markerFile, target, "out", "swap");
+ // OBSERVE, DO NOT ASSUME — a crash lands between any two syscalls, and the
+ // marker can only say which phase it was in, never how far through it got.
+ const now = await linkOrDirState(store);
+ const parked = `${store}.relocated`;
+ if (now.kind === "real-dir") {
+ // Re-verify: a resumed run did not do the copy in this process and must
+ // not take the interrupted one's word for it.
+ log("Verifying the copy…");
+ retried ||= (
+ await verifyCopy({
+ rsyncBin: paths.rsyncBin,
+ src: store,
+ dest: target,
+ log,
+ signal: a.signal,
+ })
+ ).retried;
+ await rm(parked, { recursive: true, force: true });
+ await rename(store, parked);
+ } else if (now.kind === "other") {
+ throw new Error(
+ `${store} is neither a directory nor a symlink — refusing to replace it`,
+ );
+ }
+ const after = await linkOrDirState(store);
+ if (
+ after.kind === "link" &&
+ path.resolve(after.linkTarget) !== path.resolve(target)
+ ) {
+ throw new Error(
+ `${store} already points at ${after.linkTarget}, not ${target}`,
+ );
+ }
+ if (after.kind === "missing") await symlink(target, store);
+ // THE RECORD IS WRITTEN ONLY NOW, on success, after the link exists.
+ const latest = io.read();
+ await io.write({
+ ...latest,
+ storage: { ...latest.storage, savedVideosLocationId: loc.id },
+ });
+ log(`Swapped: ${store} -> ${target}`);
+ await stampMarker(markerFile, target, "out", "reclaim");
+ }
+
+ await reclaimParked(paths, log);
+ await clearDirMarker(markerFile);
+ log(
+ `Done. ${formatBytes(measured.bytes)} now on "${loc.label || loc.id}"; ` +
+ `${formatBytes(await getFreeBytes(paths.transcriptsDir))} free on the corpus volume.`,
+ );
+ return {
+ locationId: loc.id,
+ target,
+ bytes: measured.bytes,
+ files: measured.files,
+ resumed: Boolean(a.resume),
+ retried,
+ };
+}
+
+async function moveStoreBack(a: Inner): Promise<SavedVideosRelocateResult> {
+ const { paths, log, markerFile, io } = a;
+ const settings = io.read();
+ const store = paths.savedVideosDir;
+ const state = await linkOrDirState(store);
+ // The link is the first source of truth; the marker is the second, because a
+ // crash after the swap leaves settings cleared and only the marker naming the
+ // target still holding the bytes.
+ const target =
+ state.kind === "link"
+ ? state.linkTarget
+ : (a.resume?.target ??
+ (() => {
+ const loc = settings.storage.locations.find(
+ (l) => l.id === (settings.storage.savedVideosLocationId ?? ""),
+ );
+ return loc ? relocatedSavedVideosDir(loc.root) : "";
+ })());
+ if (!target) {
+ throw new Error(
+ "The saved-video store is not relocated — it is already in place.",
+ );
+ }
+
+ const incoming = `${store}.incoming`;
+ let phase: RelocationPhase = a.resume?.phase ?? "copy";
+ if (phase === "copy" && !(await isDirectory(target))) {
+ throw new Error(
+ `The relocated store at ${target} is not reachable (is the drive mounted?)`,
+ );
+ }
+ let retried = false;
+ const measured = (await isDirectory(target))
+ ? await measureTree(target)
+ : (await isDirectory(incoming))
+ ? await measureTree(incoming)
+ : { bytes: 0, files: 0 };
+ log(
+ `Moving the saved-video store back in place: ${measured.files} file(s), ` +
+ `${formatBytes(measured.bytes)} <- ${target}`,
+ );
+
+ if (phase === "copy") {
+ const marginGB =
+ settings.minFreeDiskGB > 0 ? settings.resumeMarginGB : 0;
+ const already = (await isDirectory(incoming))
+ ? (await measureTree(incoming)).bytes
+ : 0;
+ const needed =
+ Math.max(0, measured.bytes - already) + marginGB * BYTES_PER_GB;
+ const free = await getFreeBytes(paths.transcriptsDir);
+ if (free < needed) {
+ throw new Error(
+ `Not enough space on the corpus volume: ${formatBytes(free)} free, ` +
+ `${formatBytes(Math.max(0, measured.bytes - already))} still to move back` +
+ (marginGB > 0
+ ? ` plus a ${marginGB} GB resume margin = ${formatBytes(needed)} required`
+ : " required"),
+ );
+ }
+ await mkdir(incoming, { recursive: true });
+ await stampMarker(markerFile, target, "back", "copy");
+ const { exitCode } = await rsyncTree({
+ rsyncBin: paths.rsyncBin,
+ src: target,
+ dest: incoming,
+ args: COPY_ARGS,
+ log,
+ progress: makeProgressSink({
+ totalBytes: measured.bytes,
+ log,
+ onProgress: a.onProgress,
+ }),
+ signal: a.signal,
+ });
+ if (a.signal?.aborted) {
+ throw new Error(
+ `Cancelled. ${target} is untouched and ${incoming} is resumable — rerun to continue.`,
+ );
+ }
+ if (exitCode !== 0) throw new Error(`rsync failed (exit ${exitCode})`);
+ log("Verifying the copy…");
+ retried ||= (
+ await verifyCopy({
+ rsyncBin: paths.rsyncBin,
+ src: target,
+ dest: incoming,
+ log,
+ signal: a.signal,
+ })
+ ).retried;
+ phase = "swap";
+ }
+
+ if (phase === "swap") {
+ await stampMarker(markerFile, target, "back", "swap");
+ const now = await linkOrDirState(store);
+ if (now.kind === "real-dir") {
+ log(`${store} is already a real directory — the swap had completed`);
+ } else if (now.kind === "other") {
+ throw new Error(
+ `${store} is neither a directory nor a symlink — refusing to replace it`,
+ );
+ } else {
+ if (!(await isDirectory(incoming))) {
+ throw new Error(
+ `Cannot finish the move back: ${store} is not a directory and there ` +
+ `is no verified copy at ${incoming}`,
+ );
+ }
+ // unlink, not rm -r: `store` is the LINK here, and removing it
+ // recursively would be the one way this design eats the media.
+ if (now.kind !== "missing") await unlink(store);
+ await rename(incoming, store);
+ log(`Swapped: ${store} is a real directory again`);
+ }
+ const latest = io.read();
+ if ((latest.storage.savedVideosLocationId ?? "") !== "") {
+ const { savedVideosLocationId: _dropped, ...storage } = latest.storage;
+ await io.write({ ...latest, storage });
+ }
+ await stampMarker(markerFile, target, "back", "reclaim");
+ }
+
+ await rm(target, { recursive: true, force: true });
+ await reclaimParked(paths, log);
+ await clearDirMarker(markerFile);
+ log(`Done. ${formatBytes(measured.bytes)} back in place.`);
+ return {
+ locationId: "",
+ target,
+ bytes: measured.bytes,
+ files: measured.files,
+ resumed: Boolean(a.resume),
+ retried,
+ };
+}
+
+// Every leftover a crashed run can have parked beside the store, swept on every
+// path — the channel mover's `sweepParked`, for the reason it exists there: a
+// crash between the settings write and the reclaim marker otherwise orphans a
+// full second copy of the store on the volume the move exists to free.
+async function reclaimParked(
+ paths: Paths,
+ log: (m: string) => void,
+): Promise<void> {
+ const parent = path.dirname(paths.savedVideosDir);
+ const base = path.basename(paths.savedVideosDir);
+ const names = await readdir(parent).catch(() => [] as string[]);
+ for (const n of names) {
+ if (n !== `${base}.relocated` && n !== `${base}.incoming`) continue;
+ const p = path.join(parent, n);
+ log(`Reclaiming ${p}`);
+ await rm(p, { recursive: true, force: true });
+ }
+}
+
+function isWithin(parent: string, child: string): boolean {
+ const rel = path.relative(parent, child);
+ return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
+}
diff --git a/common/jobs/jobKinds.ts b/common/jobs/jobKinds.ts
@@ -386,6 +386,24 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
queueKeyStrategy: "custom",
needsMedia: false,
},
+ // THE SAVED-VIDEO STORE, ONTO A LOCATION AND BACK. Same mechanism as the
+ // channel move (relocateDir.ts is literally the same code) over one directory
+ // that belongs to no channel — so no `channelSlug`, and `needsMedia: false`
+ // for the relocation's reason: this kind is what FIXES a store on a drive
+ // that is not there.
+ //
+ // ON THE RELOCATION QUEUE KEY, with the channel move and the re-point. All
+ // three rewrite symlinks under the same roots and all three run their space
+ // check when they START; the registry caps a key at concurrency 1, so one at
+ // a time across the three kinds is the whole point.
+ "relocate-saved-videos": {
+ kind: "relocate-saved-videos",
+ label: "Move the saved-video store",
+ drainable: false,
+ replayable: false,
+ queueKeyStrategy: "custom",
+ needsMedia: false,
+ },
// THE SAME LINKS, WITHOUT THE BYTES. A re-point rewrites every channel
// symlink on one storage location plus the location's root, for the case the
// relocation above cannot help with: the media never moved, the DISK did, and
diff --git a/common/jobs/snapshotScheduler.ts b/common/jobs/snapshotScheduler.ts
@@ -60,6 +60,9 @@ const NO_REGEN_KINDS = new Set<string>([
// has one — so "move out, then move back" would be blocked by a report
// nobody needed.
"relocate-channel-media",
+ // Moving the saved-video store changes where bytes are, not what any channel
+ // has — every count in every report is identical afterwards.
+ "relocate-saved-videos",
// A RE-POINT DOES NOT EVEN MOVE THE BYTES. It rewrites n symlinks and n
// `dataDir` fields so the media that came back on a different mountpoint is
// reachable again; every count in every report is what it was before. It
diff --git a/common/lib/settings.ts b/common/lib/settings.ts
@@ -959,7 +959,26 @@ export function sanitizeStorage(value: unknown): StorageSettings {
const defaultLocationId = locations.some((l) => l.id === wanted)
? wanted
: (locations[0]?.id ?? "");
- return { locations, defaultLocationId };
+ // THE SAVED-VIDEO STORE'S LOCATION IS NOT FALLEN BACK, and the asymmetry
+ // with `defaultLocationId` above is deliberate. That one is a PREFERENCE, so
+ // picking another location when the named one is gone is helpful. This one is
+ // a RECORD OF WHERE BYTES ARE: pointing it at a different location because
+ // the recorded one was deleted would claim the store had moved when nothing
+ // had. A dangling id sanitizes to "" — "in place" — which is what the disk
+ // says as soon as anybody looks, and the symlink (if any) keeps working
+ // regardless, because the store is reached through it and not through this.
+ const savedWanted =
+ typeof r.savedVideosLocationId === "string"
+ ? r.savedVideosLocationId.trim()
+ : "";
+ const savedVideosLocationId = locations.some((l) => l.id === savedWanted)
+ ? savedWanted
+ : "";
+ return {
+ locations,
+ defaultLocationId,
+ ...(savedVideosLocationId ? { savedVideosLocationId } : {}),
+ };
}
// 4 hours. Measured: videos over this are 8.2% of the corpus by count but hold
diff --git a/common/lib/storageLocations.ts b/common/lib/storageLocations.ts
@@ -57,6 +57,16 @@ export type StorageSettings = {
locations: StorageLocation[];
// The location prefilled as the destination of a move. "" = no default.
defaultLocationId: string;
+ // WHERE THE SAVED-VIDEO STORE IS, by location id. "" = in place, under the
+ // corpus at `paths.savedVideosDir`.
+ //
+ // A RECORD OF WHAT IS ON DISK, never an intention — the same contract as a
+ // channel's `config.dataDir`. It is written by the move, on success, after
+ // the copy has verified and the symlink is in place; nothing else writes it,
+ // and a reader that disagrees with the disk trusts the disk. Optional so an
+ // older settings.json parses (and an older binary that drops it leaves a
+ // store that still works, because the symlink is what every reader follows).
+ savedVideosLocationId?: string;
};
// Strip trailing slashes so "/mnt/platter/" and "/mnt/platter" are one root.
diff --git a/common/views/storage.ts b/common/views/storage.ts
@@ -7,6 +7,7 @@ import type {
LocationRollup,
MemoizedProbe,
} from "../controller/storageLocations";
+import type { SavedVideosStoreStatus } from "../controller/relocateSavedVideos";
import type { RegistryReader } from "./inputs";
// THE /storage PAYLOAD — one row per storage location, and for each row the
@@ -102,8 +103,43 @@ export type StorageRow = {
actions: StorageActionView[];
};
+// THE SAVED-VIDEO STORE, AS A ROW OF ITS OWN.
+//
+// It is the one large thing in the corpus that is not a channel's `data/`, so
+// no channel move can ever reach it — and until now nothing could move it at
+// all. It is NOT a location (nothing lives "on" it) and it is not a channel, so
+// it gets its own block under the locations rather than being forced into
+// either table.
+export type SavedVideosView = {
+ // `paths.savedVideosDir` — the path every reader uses, moved or not.
+ dir: string;
+ // Where the bytes actually are: the link target, or `dir` when in place.
+ at: string;
+ // The location it is on, or "" for the corpus volume.
+ locationId: string;
+ locationLabel: string;
+ status: SavedVideosStoreStatus;
+ statusLabel: string;
+ detail?: string;
+ bytes: number;
+ files: number;
+ // Destinations the Move control offers: every configured location it is not
+ // already on, plus "" (in place) when it is somewhere else.
+ destinations: Array<{ id: string; label: string }>;
+ // Why every control is off, or null. A move of the store shares the relocate
+ // queue key with channel moves and the re-point, so one running anywhere
+ // freezes this too.
+ busy: string | null;
+ // A marker is present AND nothing is running — the state Resume and Clear
+ // exist for. Separate from `busy`, which the marker itself sets: reading the
+ // hatch off `busy === null` would hide it in exactly the state it is for.
+ canResume: boolean;
+};
+
export type StorageRowsPayload = {
rows: StorageRow[];
+ // Absent when the caller did not ask for it (a unit test of the rows alone).
+ savedVideos?: SavedVideosView;
// Media bytes per row id, INCLUDING "internal". The same numbers the rows
// carry, lifted out so a caller that wants the totals (the /channels meter
// bridge) does not have to re-fold the rows.
@@ -118,8 +154,24 @@ export type StorageRowsPayload = {
udisksctlAvailable: boolean;
};
+// What the shell measured about the store. Every fact arrives as an argument,
+// as everywhere in views/: the walk, the lstat and the marker read are the
+// shell's.
+export type SavedVideosInputs = {
+ dir: string;
+ at: string;
+ locationId: string;
+ status: SavedVideosStoreStatus;
+ detail?: string;
+ bytes: number;
+ files: number;
+ hasMarker: boolean;
+};
+
export type StorageRowsInputs = {
locations: readonly StorageLocation[];
+ // Omit to leave `savedVideos` off the payload entirely.
+ savedVideos?: SavedVideosInputs;
// The corpus volume as a row. Absent → no internal row (a caller that only
// wants the configured locations). `freeBytes` is a statfs of the root, taken
// by the shell, because nothing in views/ may touch a disk.
@@ -149,6 +201,16 @@ export const STORAGE_STATUS_LABEL: Record<StorageLocationStatus, string> = {
};
export const REPOINT_JOB_KIND = "repoint-storage-location";
+export const SAVED_VIDEOS_JOB_KIND = "relocate-saved-videos";
+
+export const SAVED_VIDEOS_STATUS_LABEL: Record<SavedVideosStoreStatus, string> =
+ {
+ "in-place": "In place",
+ ok: "Relocated · reachable",
+ unreachable: "Relocated · unreachable",
+ "in-transition": "Move in flight",
+ inconsistent: "Inconsistent",
+ };
function identityLine(identity: StorageIdentity): string | null {
if (!identity.known) return null;
@@ -202,16 +264,27 @@ export function storageBytesText(bytes: number, unknown: number): string {
// shared `relocate` queue key, which the registry caps at concurrency 1, so
// "one is running" is a fact about the machine and not about a row.
function runningRepoint(registry: RegistryReader): string | null {
+ // TWO KINDS, ONE QUEUE. A re-point and a saved-video store move both rewrite
+ // symlinks under the same roots and both share `relocationQueueKey()`, which
+ // the registry caps at concurrency 1 — so either one running is a fact about
+ // the machine and freezes every row on this page, not just its own. (A
+ // CHANNEL move is on that key too and deliberately does NOT freeze this page:
+ // it touches one channel's `data/`, never a location's root or the store, and
+ // /storage has been usable during one since locations shipped.)
const job = registry
.list()
.find(
(j) =>
- j.kind === REPOINT_JOB_KIND &&
+ (j.kind === REPOINT_JOB_KIND || j.kind === SAVED_VIDEOS_JOB_KIND) &&
(j.status === "running" || j.status === "queued"),
);
if (!job) return null;
+ const what =
+ job.kind === REPOINT_JOB_KIND
+ ? "A storage re-point"
+ : "A saved-video store move";
return (
- `A storage re-point is ${job.status} (job ${job.id}). One runs at a ` +
+ `${what} is ${job.status} (job ${job.id}). One runs at a ` +
`time — wait for it to finish, or cancel it on /jobs.`
);
}
@@ -288,6 +361,9 @@ export function buildStorageRows(i: StorageRowsInputs): StorageRowsPayload {
: configured;
return {
rows,
+ ...(i.savedVideos
+ ? { savedVideos: savedVideosView(i, i.savedVideos, busy) }
+ : {}),
bytesOnLocation,
bytesInPlace: bytesOnLocation[INTERNAL_ROW_ID] ?? 0,
defaultLocationId: i.defaultLocationId,
@@ -295,6 +371,48 @@ export function buildStorageRows(i: StorageRowsInputs): StorageRowsPayload {
};
}
+// THE STORE'S OWN ROW. `busy` is the page's — one relocation runs at a time
+// across all three kinds on the shared queue key, so a channel move in flight
+// freezes this too, and saying so is better than a button that refuses.
+function savedVideosView(
+ i: StorageRowsInputs,
+ sv: SavedVideosInputs,
+ pageBusy: string | null,
+): SavedVideosView {
+ const on = i.locations.find((l) => l.id === sv.locationId);
+ const busy =
+ pageBusy ??
+ (sv.status === "in-transition"
+ ? (sv.detail ?? "A move of the store is in flight or was interrupted.")
+ : null);
+ // "In place" is offered only when it is NOT in place, and a location is
+ // offered only when it is not already the one it is on. An option that would
+ // be a no-op is an option that refuses.
+ const destinations: Array<{ id: string; label: string }> = [];
+ if (sv.locationId !== "") {
+ destinations.push({ id: "", label: "Internal (in place)" });
+ }
+ for (const l of i.locations) {
+ if (l.id !== sv.locationId) {
+ destinations.push({ id: l.id, label: l.label || l.id });
+ }
+ }
+ return {
+ dir: sv.dir,
+ at: sv.at,
+ locationId: sv.locationId,
+ locationLabel: on ? on.label || on.id : "Internal (in place)",
+ status: sv.status,
+ statusLabel: SAVED_VIDEOS_STATUS_LABEL[sv.status],
+ ...(sv.detail ? { detail: sv.detail } : {}),
+ bytes: sv.bytes,
+ files: sv.files,
+ destinations,
+ busy,
+ canResume: sv.hasMarker && pageBusy === null,
+ };
+}
+
// THE CORPUS VOLUME AS A ROW, AND IT IS FIRST.
//
// It is first because it is the row with the problem: 523 GB on a disk with 67
diff --git a/editor/app/storage/actions.ts b/editor/app/storage/actions.ts
@@ -1,5 +1,6 @@
"use server";
+import path from "node:path";
import { revalidatePath } from "next/cache";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import {
@@ -19,8 +20,14 @@ import {
recordProbedIdentity,
resetStorageProbeMemo,
} from "yt-dlp-transcript-common/controller/storageLocations";
+import {
+ clearDirMarker,
+ readDirMarker,
+} from "yt-dlp-transcript-common/controller/relocateDir";
+import { savedVideosMarkerPath } from "yt-dlp-transcript-common/controller/relocateSavedVideos";
import { channelMediaBusyReason } from "../channels/lib/mediaBusy";
import { enqueueRepointJob } from "./lib/repointJob";
+import { enqueueSavedVideosRelocation } from "./lib/savedVideosJob";
// THE SIX THINGS AN OPERATOR MAY DO TO A STORAGE LOCATION.
//
@@ -276,3 +283,74 @@ export async function repointStorageLocationAction(
}
return enqueueRepointJob({ locationId: id, newRoot: root });
}
+
+// ---------------------------------------------------------------------------
+// The saved-video store
+// ---------------------------------------------------------------------------
+
+// MOVE THE STORE, RESUME AN INTERRUPTED MOVE, OR THROW ITS MARKER AWAY — the
+// same three verbs a channel's Storage panel has, for the same three states,
+// and for the reason that file gives: without the last two, a killed copy
+// leaves a marker nothing will ever clear and the only fix is deleting a
+// dotfile over SSH.
+//
+// The store is not a channel, so `channelMediaBusyReason` has nothing to ask
+// about it. What guards it instead is the shared relocation queue key: the
+// registry caps it at concurrency 1, so a second move waits rather than racing,
+// and the controller re-reads the disk at every phase regardless.
+export async function relocateSavedVideosAction(
+ locationId: string,
+): Promise<StreamActionResult> {
+ return enqueueSavedVideosRelocation({ locationId });
+}
+
+// RESUME IS THE MARKER'S DIRECTION, NOT THE OPERATOR'S. A marker records which
+// way the interrupted run was going; resuming it the other way would swap the
+// wrong way round (the controller refuses that by name, and this never asks it
+// to). "out" needs the location it was going to, which is the one the settings
+// still record — the record is written on SUCCESS, so mid-move it still names
+// where the store was, and for a resumed move-out that is where it is going.
+export async function resumeSavedVideosRelocationAction(): Promise<StreamActionResult> {
+ const paths = getPaths();
+ const marker = await readDirMarker(savedVideosMarkerPath(paths));
+ if (!marker) {
+ return {
+ ok: false,
+ error:
+ "The saved-video store has no relocation marker — there is no " +
+ "interrupted move to resume.",
+ };
+ }
+ if (marker.direction === "back") {
+ return enqueueSavedVideosRelocation({ locationId: "" });
+ }
+ // The target is `<root>/saved-videos`, so the root is its parent — checked
+ // rather than assumed, exactly as resumeRelocationAction checks a channel's.
+ const root = path.dirname(marker.target);
+ const loc = getSettings().storage.locations.find((l) => l.root === root);
+ if (!loc) {
+ return {
+ ok: false,
+ error:
+ `The marker points at ${marker.target}, whose root ${root} is not a ` +
+ `configured storage location. Add it on /storage, or clear the marker ` +
+ `and start again.`,
+ };
+ }
+ return enqueueSavedVideosRelocation({ locationId: loc.id });
+}
+
+// THE LAST RESORT, and the only one of the three that is not a move. It removes
+// the marker file and NOTHING else: no link, no settings, no bytes. Whatever
+// the store reads as afterwards is the truth the disk was already telling
+// underneath it — which may well be `inconsistent`, and that is the honest
+// answer rather than a repair nobody asked for.
+export async function clearSavedVideosMarkerAction(): Promise<LocationResult> {
+ try {
+ await clearDirMarker(savedVideosMarkerPath(getPaths()));
+ } catch (e) {
+ return { ok: false, error: (e as Error).message };
+ }
+ revalidatePath("/storage");
+ return { ok: true, note: "Marker cleared. Nothing was moved." };
+}
diff --git a/editor/app/storage/buildStorage.ts b/editor/app/storage/buildStorage.ts
@@ -8,6 +8,12 @@ import {
channelsOnLocation,
probeAllLocations,
} from "yt-dlp-transcript-common/controller/storageLocations";
+import { measureTree } from "yt-dlp-transcript-common/controller/relocateDir";
+import {
+ inspectSavedVideosStore,
+ savedVideosMarkerPath,
+} from "yt-dlp-transcript-common/controller/relocateSavedVideos";
+import { pathExists } from "yt-dlp-transcript-common/controller/relocateDir";
import {
buildStorageRows,
type StorageRowsPayload,
@@ -53,8 +59,30 @@ export async function buildStorage(): Promise<StorageRowsPayload> {
locations.length > 0 ? udisksctlAvailable(paths) : Promise.resolve(false),
getFreeBytes(paths.channelsDir),
]);
+ // THE STORE IS WALKED, and that is affordable because of what it holds: one
+ // persisted container per PINNED or kept-latest video, not one per video. It
+ // is a few dozen files on the production corpus, against 523 GB of channel
+ // media that is never walked here (that comes off the reports). If the store
+ // ever grows to corpus scale, this is the line that has to change — and
+ // `listSavedVideos` (which reads the pointers, each carrying its own `bytes`)
+ // is the cheaper answer waiting.
+ const store = await inspectSavedVideosStore(paths, settings);
+ const storeMeasured =
+ store.status === "unreachable" || store.status === "in-transition"
+ ? { bytes: 0, files: 0 }
+ : await measureTree(paths.savedVideosDir);
return buildStorageRows({
locations,
+ savedVideos: {
+ dir: store.dir,
+ at: store.target ?? store.dir,
+ locationId: store.locationId,
+ status: store.status,
+ ...(store.detail ? { detail: store.detail } : {}),
+ bytes: storeMeasured.bytes,
+ files: storeMeasured.files,
+ hasMarker: await pathExists(savedVideosMarkerPath(paths)),
+ },
// The corpus volume, always: it is where every unrelocated channel's media
// is, and it is the row the operator is actually trying to empty.
internal: {
diff --git a/editor/app/storage/components/SavedVideosStoreCard.tsx b/editor/app/storage/components/SavedVideosStoreCard.tsx
@@ -0,0 +1,199 @@
+"use client";
+
+import Link from "next/link";
+import { useState } from "react";
+import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog";
+import { formatBytes } from "yt-dlp-transcript-common/lib/format";
+import type { SavedVideosView } from "yt-dlp-transcript-common/views/storage";
+import { cancelJobAction } from "../../jobs/actions";
+import {
+ clearSavedVideosMarkerAction,
+ relocateSavedVideosAction,
+ resumeSavedVideosRelocationAction,
+} from "../actions";
+
+// THE SAVED-VIDEO STORE — the one large thing in the corpus no channel move
+// could ever reach.
+//
+// It gets a card rather than a row in the locations table because it is not a
+// location: nothing lives "on" it, it has no volume identity of its own and
+// there is nothing to re-point. What it has is a place (in the corpus, or on a
+// location) and one verb for changing it.
+//
+// ⚠️ NO RUN PANEL IS EVER UNMOUNTED BY ITS OWN RESULT. StreamActionLog holds
+// its streamed log in React state and calls router.refresh() the instant a run
+// ends (plans/FACTS.md) — and that refresh re-renders this card with the store
+// on its new location: a different destination list, and for a Resume the very
+// marker the button was offered for now gone. The Move panel is therefore
+// rendered UNCONDITIONALLY (only its `disabled` changes), and the Resume
+// section follows the documented `ranHere` shape — a flag set inside its own
+// trigger, so it stays for its log with the now-cleared condition fed to
+// `disabled` rather than becoming a second Run button.
+
+export function SavedVideosStoreCard({ store }: { store: SavedVideosView }) {
+ const [destId, setDestId] = useState(() => store.destinations[0]?.id ?? "");
+ const [ranResumeHere, setRanResumeHere] = useState(false);
+ const [note, setNote] = useState<string | null>(null);
+ const [error, setError] = useState<string | null>(null);
+
+ // DERIVED, not the raw state: a finished move changes the destination list
+ // under this card, and a select still naming the location the store is now
+ // ON would be offering a no-op.
+ const valid = store.destinations.some((d) => d.id === destId);
+ const effectiveId = valid ? destId : (store.destinations[0]?.id ?? "");
+ const canMove =
+ store.busy === null &&
+ store.destinations.length > 0 &&
+ store.status !== "in-transition" &&
+ store.status !== "inconsistent";
+
+ return (
+ <article
+ aria-label="saved video store"
+ className="flex flex-col gap-3 rounded-xl border border-border bg-card px-4 py-3"
+ >
+ <div className="flex flex-wrap items-center gap-3">
+ <h2 className="text-base font-semibold">Saved-video store</h2>
+ <span
+ aria-label="saved videos status"
+ className={`rounded-full border px-2 py-0.5 text-xs font-medium ${
+ store.status === "in-place" || store.status === "ok"
+ ? "border-border bg-muted"
+ : "border-destructive/50 bg-destructive/5 text-destructive"
+ }`}
+ >
+ {store.statusLabel}
+ </span>
+ </div>
+
+ <p className="text-sm text-muted-foreground max-w-3xl">
+ Where persisted source containers live. No channel move reaches it — it
+ belongs to the corpus, not to a channel — so it moves on its own, by the
+ same mechanism: the copy is verified before the source is touched, and a
+ symlink is left behind so{" "}
+ <Link href="/saved-videos" className="underline">
+ every reader
+ </Link>{" "}
+ keeps working unchanged.
+ </p>
+
+ <dl className="grid grid-cols-[max-content_1fr] gap-x-4 gap-y-1 text-sm">
+ <dt className="text-muted-foreground">On</dt>
+ <dd aria-label="saved videos location">{store.locationLabel}</dd>
+ <dt className="text-muted-foreground">Path</dt>
+ <dd className="font-mono text-xs break-all" aria-label="saved videos path">
+ {store.at}
+ {store.at !== store.dir ? ` (read as ${store.dir})` : ""}
+ </dd>
+ <dt className="text-muted-foreground">Size</dt>
+ <dd aria-label="saved videos bytes">
+ {store.status === "unreachable" || store.status === "in-transition"
+ ? "—"
+ : `${formatBytes(store.bytes)} in ${store.files.toLocaleString()} file(s)`}
+ </dd>
+ </dl>
+
+ {store.detail && (
+ <p
+ role="status"
+ aria-label="saved videos detail"
+ className="text-sm rounded border border-border bg-muted px-3 py-2"
+ >
+ {store.detail}
+ </p>
+ )}
+ {store.busy && (
+ <p role="status" className="text-sm rounded border border-border bg-muted px-3 py-2">
+ {store.busy}
+ </p>
+ )}
+ {note && (
+ <p role="status" aria-label="saved videos result" className="text-sm">
+ {note}
+ </p>
+ )}
+ {error && (
+ <p role="alert" aria-label="saved videos error" className="text-sm text-destructive">
+ {error}
+ </p>
+ )}
+
+ <section className="flex flex-col gap-2">
+ <label className="flex flex-col gap-1 text-sm max-w-md">
+ <span className="font-medium">Move the store to…</span>
+ <select
+ aria-label="saved videos destination"
+ value={effectiveId}
+ disabled={!canMove}
+ onChange={(e) => setDestId(e.target.value)}
+ className="rounded border border-border bg-card px-2 py-1 text-sm"
+ >
+ {store.destinations.map((d) => (
+ <option key={d.id || "internal"} value={d.id}>
+ {d.label}
+ </option>
+ ))}
+ </select>
+ </label>
+ <StreamActionLog
+ key="saved-videos-move-log"
+ trigger={() => relocateSavedVideosAction(effectiveId)}
+ cancelAction={cancelJobAction}
+ buttonLabel="Move the store"
+ runningLabel="Moving the store…"
+ label="Move the store"
+ disabled={!canMove}
+ />
+ </section>
+
+ {/* THE TWO WAYS OUT OF A MARKER WHOSE RUN IS GONE — finish it, or throw
+ it away. Offered only when a marker is present AND nothing is running:
+ with a live job the marker is not stale, it belongs to that run. */}
+ {(store.canResume || ranResumeHere) && (
+ <section className="flex flex-col gap-2 rounded border border-destructive/50 bg-destructive/5 px-3 py-2">
+ <h3 className="text-sm font-semibold">
+ Finish the interrupted move, or clear its marker
+ </h3>
+ {store.canResume && (
+ <p className="text-xs text-muted-foreground">
+ <strong>Resume</strong> runs the same move again from where it
+ stopped — whatever already copied correctly is not copied twice,
+ and the source is not touched until the copy verifies. That is the
+ usual answer. <strong>Clear marker</strong> removes the marker file
+ and nothing else: no link, no settings, no bytes.
+ </p>
+ )}
+ <div>
+ <button
+ type="button"
+ aria-label="clear saved videos marker"
+ disabled={!store.canResume}
+ onClick={async () => {
+ setError(null);
+ setNote(null);
+ const r = await clearSavedVideosMarkerAction();
+ if (r.ok) setNote(r.note ?? null);
+ else setError(r.error);
+ }}
+ className="px-3 py-1.5 rounded-md border border-destructive text-destructive text-sm font-medium disabled:opacity-50"
+ >
+ Clear marker
+ </button>
+ </div>
+ <StreamActionLog
+ key="saved-videos-resume-log"
+ trigger={() => {
+ setRanResumeHere(true);
+ return resumeSavedVideosRelocationAction();
+ }}
+ cancelAction={cancelJobAction}
+ buttonLabel="Resume move"
+ runningLabel="Resuming…"
+ label="Resume move"
+ disabled={!store.canResume}
+ />
+ </section>
+ )}
+ </article>
+ );
+}
diff --git a/editor/app/storage/components/StorageLocationsTable.tsx b/editor/app/storage/components/StorageLocationsTable.tsx
@@ -20,6 +20,7 @@ import {
repointStorageLocationAction,
} from "../actions";
import { LocationForm } from "./LocationForm";
+import { SavedVideosStoreCard } from "./SavedVideosStoreCard";
// THE LOCATIONS, ONE CARD EACH, AND WHAT MAY BE DONE TO THEM.
//
@@ -68,6 +69,13 @@ export function StorageLocationsTable({ payload }: { payload: StorageRowsPayload
))}
</section>
+ {/* AFTER THE LOCATIONS, BEFORE THE FORM. The store is a thing that lives
+ on a location, so it reads after the list of them — and before "Add a
+ location", which is the page's trailing affordance. */}
+ {payload.savedVideos && (
+ <SavedVideosStoreCard store={payload.savedVideos} />
+ )}
+
<section className="flex flex-col gap-2">
<h2 className="text-base font-semibold">Add a location</h2>
{adding ? (
diff --git a/editor/app/storage/lib/savedVideosJob.ts b/editor/app/storage/lib/savedVideosJob.ts
@@ -0,0 +1,80 @@
+import { revalidatePath } from "next/cache";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { relocationQueueKey } from "yt-dlp-transcript-common/lib/queueKeys";
+import {
+ runManagedFunction,
+ type StreamActionResult,
+} from "yt-dlp-transcript-common/jobs/streamCommand";
+import { formatBytes } from "yt-dlp-transcript-common/lib/format";
+import { relocateSavedVideos } from "yt-dlp-transcript-common/controller/relocateSavedVideos";
+
+// ONE ENQUEUE OF THE SAVED-VIDEO STORE MOVE, for the two callers that have one:
+// the Move control on /storage and its Resume.
+//
+// Deliberately NOT in `actions.ts` — that file carries "use server", where
+// every non-type export is a server action, so a shared helper exported from it
+// would put an unguarded enqueue on the wire under its own endpoint. Same
+// reason `lib/repointJob.ts` and `channels/lib/relocationJob.ts` exist.
+//
+// THE QUEUE KEY IS THE SHARED RELOCATION ONE. A store move, a channel move and
+// a re-point all rewrite symlinks under the same roots and all run their space
+// check when they START; the registry caps a key at concurrency 1, so one at a
+// time across the three is the whole point of sharing it.
+//
+// THE PROGRESS IS THE CHANNEL MOVE'S, for the reason the code is: it is the
+// same rsync, reported the same way, so /jobs draws the same bar.
+export async function enqueueSavedVideosRelocation(opts: {
+ // "" moves the store back in place.
+ locationId: string;
+}): Promise<StreamActionResult> {
+ const paths = getPaths();
+ const back = opts.locationId.trim() === "";
+ return runManagedFunction({
+ kind: "relocate-saved-videos",
+ queueKey: relocationQueueKey(),
+ paths,
+ fn: async (onLog, signal, _setProgress, ctx) => {
+ const taskId = "relocate:saved-videos";
+ let started = false;
+ const result = await relocateSavedVideos({
+ paths,
+ locationId: opts.locationId,
+ onLog,
+ // Added on the FIRST frame, not at start: the preflight and a resumed
+ // run's verify pass transfer nothing, and a bar sitting at 0 % through
+ // them would be claiming a copy had begun.
+ onProgress: (p) => {
+ if (!started) {
+ started = true;
+ ctx?.addTask({
+ id: taskId,
+ label: back
+ ? "saved-video store → in place"
+ : `saved-video store → ${opts.locationId}`,
+ kind: "relocate",
+ startedAt: Date.now(),
+ });
+ }
+ ctx?.updateTask(taskId, { fraction: p.fraction, detail: p.detail });
+ },
+ signal,
+ }).finally(() => {
+ if (started) ctx?.removeTask(taskId);
+ });
+ onLog(
+ `${back ? "Moved back" : "Moved"} ${result.files} file(s) / ` +
+ `${formatBytes(result.bytes)} — ${result.target}` +
+ (result.resumed ? " (resumed an interrupted move)" : "") +
+ (result.retried
+ ? " (one directory timestamp settled by a second pass)"
+ : ""),
+ );
+ // No snapshot regen — the kind is in NO_REGEN_KINDS. The move changes
+ // where the store's bytes are, not what any channel has: every count in
+ // every report is identical afterwards. What DOES change is what these
+ // pages read per render.
+ revalidatePath("/storage");
+ revalidatePath("/saved-videos");
+ },
+ });
+}
