commit 7ab623f66af7d496eaccb8bfcba6d0365aee1bb6
parent cea69e7bb246d609c66d92300d1d4be0109c57e4
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sun, 20 Sep 2026 22:05:30 -0400
Merge main into feat/fetch-window-via-editor
Three add/add conflicts, all "keep both":
- persistSourceVideo's opts gained `origin?` here and `paths?` on main. They
answer different questions — who asked for the container, and whether the
saved-video store is mid-move — so both stay, and the call in
downloadOneManaged passes both.
- plans/FACTS.md: two sections appended at the same place.
Noted while resolving: main's new `mediaBytes` loop skips sub-directories on
the stated grounds that "a video dir is flat", which `clips/` makes untrue —
so a fetched window is on the platter but not in `snapshot.totalMediaBytes`.
Recorded in FACTS with the anchor; not fixed here.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
78 files changed, 8340 insertions(+), 396 deletions(-)
diff --git a/RUNNING_IN_DOCKER.md b/RUNNING_IN_DOCKER.md
@@ -224,6 +224,55 @@ boot self-updates it. Or, once:
docker compose exec editor yt-dlp -U
```
+### Driving the editor without a browser
+
+Every editor gesture is a server action, which is fine for a person and hostile
+to a script: there is no URL to POST to. `/api/ops/*` is a thin layer over the
+**same actions** — one route per gesture, no rule of its own — so a shell, a cron
+job or an agent can run the archive without Playwright.
+
+It is gated by the **same `WORKER_TOKEN`** as `/api/worker/*`, deliberately: that
+variable already means "this instance takes instructions from something that is
+not the browser in front of it". Unset on the server and every route answers
+**503** (the surface is off until you opt in); wrong or missing on the caller and
+it answers **401**.
+
+```sh
+export ARCHILYZER_EDITOR_URL=http://localhost:3001
+export WORKER_TOKEN=<the same secret the editor is running with>
+
+pnpm ops sync --json '{"slug":"the-quartering"}' --wait
+pnpm ops metadata-scan --json '{"slug":"the-quartering"}'
+pnpm ops channel-config --json '{"slug":"the-quartering","patch":{"downloadFilterExclude":"rerun"}}'
+pnpm ops channel-priority --json '{"slugs":["the-quartering"],"operation":"download","tier":"paused"}'
+pnpm ops lane --json '{"lane":"download","held":true}'
+pnpm ops refresh-report --json '{"all":true}'
+pnpm ops get channel the-quartering
+pnpm ops list # every action name
+```
+
+Three things to know before you script against it:
+
+- **A job-starting action returns a `jobId` and does not stream.** The job may
+ sit in a platform queue behind other work for hours, so "started" is the
+ honest answer; `--wait` follows `/api/jobs/<id>/log` to the end and exits with
+ the job's status.
+- **Unknown body keys are a 400.** A misspelled `downloadFilterExclude` would
+ otherwise save cleanly and leave a channel downloading everything.
+- **`channel-config` patch keys are the CONFIGURE FORM's field names**, not
+ `config.json`'s — `downloadFilterInclude` / `downloadFilterExclude` rather than
+ a `downloadFilter` object. That is what routes them through the form's own
+ validators, so a bad regex is refused here with the sentence the form shows.
+ `""` clears a field, exactly as clearing the input does.
+
+The read side needs no new routes for jobs: `/api/jobs/active`,
+`/api/jobs/<id>/log`, `/api/scheduler/status` and `/api/auto-queue/status`
+already exist. `GET /api/ops/channel/<slug>` is the one addition — config,
+report totals, bucket sizes, priority and, the part no directory listing can
+tell you, whether the channel's media is actually **reachable**. Add `--counts`
+(`?counts=1`) for the live on-disk counts; it is opt-in because it walks every
+video directory, eleven thousand of them on the largest channel here.
+
### Booting without resuming work
`editor/instrumentation.ts` arms the sync heartbeat and every enabled auto-queue
diff --git a/SETUP.md b/SETUP.md
@@ -273,7 +273,7 @@ any of them via environment variables before launching:
| `PARAKEET_CLI` / `PARAKEET_MODEL` / `PARAKEET_STITCH_BIN` | `parakeet-cli` / — / `scripts/parakeet-stitch.mjs` | parakeet.cpp CLI, model, and wrapper. |
| `FFMPEG_BIN` / `FFPROBE_BIN` | `ffmpeg` / `ffprobe` (PATH) | Audio transcode + duration checks. |
| `RSYNC_BIN` | `rsync` (PATH) | Saved-video backup. |
-| `WORKER_TOKEN` | — | Bearer token for the remote-worker transcription API (set on both ends when used). |
+| `WORKER_TOKEN` | — | Bearer token for the remote-worker transcription API (set on both ends when used), and for the `/api/ops/*` HTTP layer over the editor's actions — see [RUNNING_IN_DOCKER.md](RUNNING_IN_DOCKER.md#driving-the-editor-without-a-browser) and `pnpm ops`. Unset means both surfaces are off. |
Feature-area docs cover their own env vars: [SCHEDULED_SYNC.md](SCHEDULED_SYNC.md)
(`SYNC_HEARTBEAT_SECONDS`, `SYNC_TICK_URL`, `SYNC_TICK_TOKEN`) and
diff --git a/common/controller/channelSnapshot.ts b/common/controller/channelSnapshot.ts
@@ -318,6 +318,17 @@ export type ChannelSnapshot = {
// must default to 0 — and MUST render that as "—", not "0", because a zero
// here would claim a measurement nobody took.
totalAudioBytes?: number;
+ // EVERY byte under `data/<id>/` for every video — audio, transcripts, cues,
+ // metadata, thumbnails, a persisted container. The figure `/storage` and the
+ // `/channels` Size column are priced in, and the one a relocation carries;
+ // `totalAudioBytes` is a fraction of it and is about what a CLEANUP could
+ // reclaim, which is a different question.
+ //
+ // Optional, and the distinction is load-bearing: a snapshot written before
+ // this field existed lacks it, and a reader MUST render that as "size unknown
+ // until Refresh report", never as 0 — a zero would rank a 400 GB channel
+ // bottom of a "free up N GB" list.
+ totalMediaBytes?: number;
// WHY the audio that isn't reclaimable isn't reclaimable, in the sweep's own
// order (cleanAudioFromTranscribed's discover loop). A video leaves at the
// FIRST gate it hits, so these are an attribution and never overlapping sets:
@@ -741,13 +752,31 @@ export async function generateChannelSnapshot(
limit(async () => {
const dir = path.join(dataDir, id);
const files = await readVideoFiles(dir, { checkUntranscribable: true });
- // Sizes of the real audio files, used to estimate how much disk a
- // cleanup would reclaim. Best-effort: skip any file we can't stat.
+ // EVERY FILE IN THE DIR, STATTED ONCE, feeding two numbers.
+ //
+ // `audioSizes` is what it always was: the real audio files, keyed by
+ // name, for the cleanup reclaim estimate. `mediaBytes` is new and is
+ // every byte this video dir holds — audio, transcripts, sidecars,
+ // thumbnails, a persisted container — because THAT is the number a
+ // relocation moves and a volume holds, and the audio total is only a
+ // fraction of it (a channel's transcripts, cues and metadata are not
+ // free).
+ //
+ // NOT A SECOND WALK: the readdir is `files.entries`, already in hand,
+ // and the audio stats this loop replaces were being paid anyway. What
+ // it adds is a stat per NON-audio entry — six to ten per video, warm
+ // inode cache, on a pass that already reads several sidecars per video.
+ // Sub-directories are counted as nothing rather than recursed: a video
+ // dir is flat, and a walk here would be the second walk this avoids.
+ const audioSet = new Set(files.audioFiles);
const audioSizes: Record<string, number> = {};
- for (const name of files.audioFiles) {
+ let mediaBytes = 0;
+ for (const name of files.entries) {
try {
const st = await stat(path.join(dir, name));
- audioSizes[name] = st.size;
+ if (!st.isFile()) continue;
+ mediaBytes += st.size;
+ if (audioSet.has(name)) audioSizes[name] = st.size;
} catch {
// ignore — file vanished or is unreadable
}
@@ -820,6 +849,7 @@ export async function generateChannelSnapshot(
files,
backfill,
audioSizes,
+ mediaBytes,
nativeId,
availability,
effectiveAvailability,
@@ -931,6 +961,10 @@ export async function generateChannelSnapshot(
// hand on the perVideo entry, so this adds ZERO I/O to a pass that runs over
// ~79,000 videos.
let totalAudioBytes = 0;
+ // Every byte under `data/`, not only the audio. The figure the storage
+ // surfaces are priced in: how much a volume is holding for this channel, and
+ // how much a move would carry.
+ let totalMediaBytes = 0;
const heldAudioBytes = emptyHeldAudio();
const heldAudioCounts = emptyHeldAudio();
let reclaimableAtRiskBytes = 0;
@@ -946,6 +980,7 @@ export async function generateChannelSnapshot(
id,
files,
audioSizes,
+ mediaBytes,
backfill,
effectiveAvailability,
outcome,
@@ -955,6 +990,7 @@ export async function generateChannelSnapshot(
} of perVideo) {
if (isVideoTranscribed(files)) transcribed++;
if (isVideoDownloaded(files)) downloaded++;
+ totalMediaBytes += mediaBytes;
// --- Hold attribution ---------------------------------------------------
// FIRST, before the short-circuits below: they `continue` past videos that
@@ -1410,6 +1446,7 @@ export async function generateChannelSnapshot(
foreignAudio: foreignAudioBytes,
},
totalAudioBytes,
+ totalMediaBytes,
heldAudioBytes,
heldAudioCounts,
reclaimableAtRiskBytes,
diff --git a/common/controller/relocateChannelMedia.ts b/common/controller/relocateChannelMedia.ts
@@ -2,21 +2,30 @@ import path from "node:path";
import {
access,
constants as fsConstants,
- lstat,
mkdir,
readdir,
- readFile,
- readlink,
realpath,
rename,
rm,
stat,
symlink,
unlink,
- writeFile,
} from "node:fs/promises";
-import { execa } from "execa";
import type { Paths } from "../lib/paths";
+import {
+ clearDirMarker,
+ COPY_ARGS,
+ isDirectory,
+ linkOrDirState,
+ makeProgressSink,
+ measureTree,
+ pathExists,
+ readDirMarker,
+ rsyncTree,
+ verifyCopy,
+ writeDirMarker,
+ type RelocationProgress,
+} from "./relocateDir";
import { isSocialChannel } from "../lib/channelConfig";
import { getFreeBytes } from "../lib/diskSpace";
import { getSettings } from "../lib/settings";
@@ -67,6 +76,11 @@ export type RelocateChannelMediaResult = {
retried: boolean;
};
+// The progress shape is the shared mover's — re-exported because callers
+// (the editor's relocate job) reach for it through this module, which is the
+// one they already import.
+export type { RelocationProgress } from "./relocateDir";
+
type RelocateOpts = {
paths: Paths;
slug: string;
@@ -74,6 +88,10 @@ type RelocateOpts = {
// Required for "out"; ignored for "back", which reads the target from config.
root?: string;
onLog?: (line: string) => void;
+ // Called on every rsync redraw of the COPY phase (never the verify, which
+ // transfers nothing). Optional: a bin script that only wants the log passes
+ // nothing and pays only the parse.
+ onProgress?: (p: RelocationProgress) => void;
signal?: AbortSignal;
};
@@ -91,57 +109,6 @@ export type RelocationPreview = {
existingPartial: boolean;
};
-// One walk, used by the preview, the space check and the verify. Follows no
-// symlinks (a relocated channel is never the SOURCE of another relocation).
-async function measureTree(
- dir: string,
-): Promise<{ bytes: number; files: number }> {
- let bytes = 0;
- let files = 0;
- const stack = [dir];
- while (stack.length > 0) {
- const cur = stack.pop() as string;
- let entries;
- try {
- entries = await readdir(cur, { withFileTypes: true });
- } catch {
- continue;
- }
- for (const e of entries) {
- const p = path.join(cur, e.name);
- if (e.isDirectory()) {
- stack.push(p);
- } else if (e.isFile()) {
- try {
- const st = await stat(p);
- bytes += st.size;
- files++;
- } catch {
- /* vanished mid-walk; the verify is what catches real drift */
- }
- }
- }
- }
- return { bytes, files };
-}
-
-async function pathExists(p: string): Promise<boolean> {
- try {
- await stat(p);
- return true;
- } catch {
- return false;
- }
-}
-
-async function isDirectory(p: string): Promise<boolean> {
- try {
- return (await stat(p)).isDirectory();
- } catch {
- return false;
- }
-}
-
// Resolved through every symlink when the path exists, lexically when it does
// not. A destination root that does not exist yet is refused elsewhere; a root
// that exists and is a symlink back into the corpus is exactly what this is for.
@@ -216,35 +183,6 @@ export async function relocationRootProblem(opts: {
return null;
}
-// WHAT `data/` ACTUALLY IS RIGHT NOW. lstat, never stat: a dangling symlink —
-// the exact state a half-finished move or an unmounted drive leaves behind —
-// reads as ABSENT through stat, and the code that then tries to create the link
-// fails with EEXIST on a path it was just told was not there.
-//
-// Every phase past the copy dispatches on this rather than on what the previous
-// phase is supposed to have done, which is what makes a rerun idempotent: the
-// marker says how far the last run GOT, the disk says what is actually there,
-// and only the disk is evidence.
-type DataDirState =
- | { kind: "missing" }
- | { kind: "real-dir" }
- | { kind: "link"; linkTarget: string }
- | { kind: "other" };
-
-async function dataDirState(p: string): Promise<DataDirState> {
- let st;
- try {
- st = await lstat(p);
- } catch {
- return { kind: "missing" };
- }
- if (st.isSymbolicLink()) {
- return { kind: "link", linkTarget: await readlink(p).catch(() => "") };
- }
- if (st.isDirectory()) return { kind: "real-dir" };
- return { kind: "other" };
-}
-
// Every leftover a crashed run can have parked next to `data/`, in one list.
// The reclaim phase sweeps ALL of them rather than the one name the run that is
// finishing happens to hold: a crash between the config write and the reclaim
@@ -268,142 +206,25 @@ async function sweepParked(
}
}
+// The channel's marker, at `channels/<slug>/.relocating.json`. The FILE is the
+// contract — see relocateDir.ts — and these three are the channel's name for it.
async function writeMarker(
paths: Paths,
slug: string,
marker: RelocationMarker,
): Promise<void> {
- const file = relocationMarkerPath(paths, slug);
- const tmp = `${file}.tmp-${process.pid}`;
- await writeFile(tmp, JSON.stringify(marker, null, 2) + "\n");
- await rename(tmp, file);
+ await writeDirMarker(relocationMarkerPath(paths, slug), marker);
}
async function clearMarker(paths: Paths, slug: string): Promise<void> {
- await rm(relocationMarkerPath(paths, slug), { force: true });
+ await clearDirMarker(relocationMarkerPath(paths, slug));
}
async function readMarkerRaw(
paths: Paths,
slug: string,
): Promise<RelocationMarker | null> {
- try {
- const raw = await readFile(relocationMarkerPath(paths, slug), "utf8");
- const r = JSON.parse(raw) as Partial<RelocationMarker>;
- if (typeof r.target !== "string") return null;
- return {
- target: r.target,
- direction: r.direction === "back" ? "back" : "out",
- startedAt: typeof r.startedAt === "string" ? r.startedAt : "",
- phase:
- r.phase === "swap" || r.phase === "reclaim"
- ? r.phase
- : ("copy" as RelocationPhase),
- };
- } catch {
- return null;
- }
-}
-
-// rsync, exactly as backupSavedVideos does it: the real binary from
-// paths.rsyncBin, cancellable, streaming its output into the job log.
-async function rsyncTree(opts: {
- paths: Paths;
- src: string;
- dest: string;
- args: string[];
- log: (m: string) => void;
- signal?: AbortSignal;
-}): Promise<{ exitCode: number; output: string }> {
- // Trailing slash on src: copy the CONTENTS, so <src>/ -> <dest>/ and not
- // <dest>/data/. Getting this wrong is a silently nested corpus.
- const args = [...opts.args, `${opts.src}/`, `${opts.dest}/`];
- opts.log(`$ ${opts.paths.rsyncBin} ${args.join(" ")}`);
- const child = execa(opts.paths.rsyncBin, args, {
- cancelSignal: opts.signal,
- all: true,
- buffer: false,
- reject: false,
- });
- let output = "";
- child.all?.on("data", (c: Buffer) => {
- const text = c.toString("utf8");
- output += text;
- opts.log(text);
- });
- const result = await child;
- return { exitCode: result.exitCode ?? 1, output };
-}
-
-// A DIRECTORY MTIME IS NOT CONTENT. `.d..t` is rsync's itemization for "this is
-// a directory and only its modification time differs" — nothing to send, and no
-// byte of the copy is in question. It is what the omnimirror move hit
-// (2026-09-13): a sidecar written into one video directory while the copy was
-// already past it bumped that directory's mtime on the SOURCE and left the
-// target's behind, the verify saw one drift line, and a 131 GB copy refused at
-// the last step with nothing actually wrong.
-const DIR_MTIME_ONLY = /^\.d\.\.t/;
-
-function driftLines(output: string): string[] {
- return output
- .split("\n")
- .map((l) => l.trim())
- .filter((l) => l.length > 0 && !l.startsWith("sending incremental"))
- .filter((l) => !/^(sent|total size|$)/.test(l));
-}
-
-// What a copy has to clear before the swap: rsync itself agrees there is
-// nothing left to send, AND the two trees measure the same. The dry run alone
-// would accept a target that is byte-identical for the wrong reason; the counts
-// alone would accept two trees of equal size with different contents.
-async function verifyCopy(opts: {
- paths: Paths;
- src: string;
- dest: string;
- log: (m: string) => void;
- signal?: AbortSignal;
-}): Promise<{ bytes: number; files: number; retried: boolean }> {
- let retried = false;
- // ONE retry, never a loop: if a second pass does not settle it, something is
- // still writing into the tree and the answer is to refuse, not to chase it.
- for (;;) {
- const { exitCode, output } = await rsyncTree({
- ...opts,
- args: ["-a", "--dry-run", "--itemize-changes"],
- });
- if (exitCode !== 0) {
- throw new Error(`Verification rsync failed (exit ${exitCode})`);
- }
- const drift = driftLines(output);
- if (drift.length === 0) break;
- if (!retried && drift.every((l) => DIR_MTIME_ONLY.test(l))) {
- retried = true;
- opts.log(
- `Verification found ${drift.length} directory timestamp(s) differing and ` +
- `no content drift — running one more rsync pass to settle them.`,
- );
- const again = await rsyncTree({ ...opts, args: ["-a"] });
- if (again.exitCode !== 0) {
- throw new Error(
- `Verification rsync failed (exit ${again.exitCode}). ` +
- `The source has NOT been touched.`,
- );
- }
- continue;
- }
- throw new Error(
- `Verification failed: ${drift.length} file(s) still differ ` +
- `(first: ${drift[0]}). The source has NOT been touched.`,
- );
- }
- const [a, b] = await Promise.all([measureTree(opts.src), measureTree(opts.dest)]);
- if (a.files !== b.files || a.bytes !== b.bytes) {
- throw new Error(
- `Verification failed: source has ${a.files} file(s)/${formatBytes(a.bytes)}, ` +
- `target has ${b.files}/${formatBytes(b.bytes)}. The source has NOT been touched.`,
- );
- }
- return { ...a, retried };
+ return readDirMarker(relocationMarkerPath(paths, slug));
}
// What the operator sees before committing to a move. Cheap enough to run on a
@@ -465,6 +286,7 @@ export async function relocateChannelMedia(
): Promise<RelocateChannelMediaResult> {
const { paths, slug, direction, signal } = opts;
const log = opts.onLog ?? ((m: string) => console.log(m));
+ const onProgress = opts.onProgress;
const config = await readChannelConfig(paths, slug);
if (!config) throw new Error(`Channel "${slug}" not found`);
if (isSocialChannel(config)) {
@@ -514,6 +336,7 @@ export async function relocateChannelMedia(
dataDir,
target,
log,
+ onProgress,
signal,
resumed: Boolean(resume),
phase: resume?.phase ?? "copy",
@@ -558,6 +381,7 @@ export async function relocateChannelMedia(
root,
target,
log,
+ onProgress,
signal,
// `resumed` relaxes the "must be in-place" precondition, so it must mean
// "this run continues an interrupted move OUT" and nothing looser.
@@ -575,6 +399,7 @@ async function moveOut(args: {
root: string;
target: string;
log: (m: string) => void;
+ onProgress?: (p: RelocationProgress) => void;
signal?: AbortSignal;
resumed: boolean;
phase: RelocationPhase;
@@ -655,6 +480,12 @@ async function moveOut(args: {
);
}
await mkdir(target, { recursive: true });
+ // WHAT A PREVIOUS ATTEMPT ALREADY LANDED, so the bar is about the TREE and
+ // not about this process's share of it. rsync counts only what it sends: a
+ // copy resumed at 60 % would otherwise climb 0 → 40 % and stop, having
+ // moved every remaining byte. One walk of the target, which for a fresh
+ // move is an empty directory and costs a readdir.
+ const already = (await measureTree(target)).bytes;
await writeMarker(paths, slug, {
target,
direction: "out",
@@ -664,11 +495,17 @@ async function moveOut(args: {
// --partial keeps an aborted transfer resumable; -a preserves mtimes, which
// is what makes the LMDB index a no-op afterwards.
const { exitCode } = await rsyncTree({
- paths,
+ rsyncBin: paths.rsyncBin,
src: dataDir,
dest: target,
- args: ["-a", "--partial", "--info=progress2"],
+ args: COPY_ARGS,
log,
+ progress: makeProgressSink({
+ totalBytes: measured.bytes,
+ alreadyBytes: already,
+ log,
+ onProgress: args.onProgress,
+ }),
signal,
});
if (signal?.aborted) {
@@ -681,7 +518,7 @@ async function moveOut(args: {
log("Verifying the copy…");
verifyRetried ||= (
- await verifyCopy({ paths, src: dataDir, dest: target, log, signal })
+ await verifyCopy({ rsyncBin: paths.rsyncBin, src: dataDir, dest: target, log, signal })
).retried;
phase = "swap";
}
@@ -697,7 +534,7 @@ async function moveOut(args: {
// EVERY STEP BELOW OBSERVES THE DISK INSTEAD OF ASSUMING THE LAST ONE RAN.
// The marker says how far the previous attempt got; it cannot say how far
// it got THROUGH a phase, and a crash lands between any two syscalls.
- const state = await dataDirState(dataDir);
+ const state = await linkOrDirState(dataDir);
const parked = (await parkedSiblings(channelDir)).filter((p) =>
path.basename(p).startsWith("data.relocated-"),
);
@@ -711,7 +548,7 @@ async function moveOut(args: {
if (verifySrc) {
log("Verifying the copy…");
verifyRetried ||= (
- await verifyCopy({ paths, src: verifySrc, dest: target, log, signal })
+ await verifyCopy({ rsyncBin: paths.rsyncBin, src: verifySrc, dest: target, log, signal })
).retried;
}
@@ -727,7 +564,7 @@ async function moveOut(args: {
);
}
- const after = await dataDirState(dataDir);
+ const after = await linkOrDirState(dataDir);
if (
after.kind === "link" &&
path.resolve(after.linkTarget) !== path.resolve(target)
@@ -790,6 +627,7 @@ async function moveBack(args: {
dataDir: string;
target: string;
log: (m: string) => void;
+ onProgress?: (p: RelocationProgress) => void;
signal?: AbortSignal;
resumed: boolean;
phase: RelocationPhase;
@@ -878,11 +716,19 @@ async function moveBack(args: {
phase: "copy",
});
const { exitCode } = await rsyncTree({
- paths,
+ rsyncBin: paths.rsyncBin,
src: target,
dest: incoming,
- args: ["-a", "--partial", "--info=progress2"],
+ args: COPY_ARGS,
log,
+ progress: makeProgressSink({
+ totalBytes: measured.bytes,
+ // The same figure the space check above is priced in — what is already
+ // in `data.incoming` from an interrupted run.
+ alreadyBytes: already,
+ log,
+ onProgress: args.onProgress,
+ }),
signal,
});
if (signal?.aborted) {
@@ -894,7 +740,7 @@ async function moveBack(args: {
if (exitCode !== 0) throw new Error(`rsync failed (exit ${exitCode})`);
log("Verifying the copy…");
verifyRetried ||= (
- await verifyCopy({ paths, src: target, dest: incoming, log, signal })
+ await verifyCopy({ rsyncBin: paths.rsyncBin, src: target, dest: incoming, log, signal })
).retried;
phase = "swap";
}
@@ -912,7 +758,7 @@ async function moveBack(args: {
// the case that never happens: a crash AFTER the rename left `data/` a real
// directory, the swallowed unlink then failed on it, and the rename ENOENTed
// on an `incoming` that no longer existed — forever, on every rerun.
- const state = await dataDirState(dataDir);
+ const state = await linkOrDirState(dataDir);
if (state.kind === "real-dir") {
// The rename already committed. Nothing to swap; the leftovers are the
// reclaim's business.
diff --git a/common/controller/relocateDir.test.ts b/common/controller/relocateDir.test.ts
@@ -0,0 +1,91 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { makeProgressSink, type RelocationProgress } from "./relocateDir";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/relocateDir.test.ts
+//
+// The sink, on its own: no rsync, no disk. What it has to get right is the two
+// things a bar claims — that it is about the TREE, and that it only moves one
+// way.
+
+function frame(bytes: number, percent: number): string {
+ return ` ${bytes.toLocaleString("en-US")} ${percent}% 1.03GB/s 0:00:03`;
+}
+
+function collect(opts: { totalBytes: number; alreadyBytes?: number }) {
+ const seen: RelocationProgress[] = [];
+ const lines: string[] = [];
+ const sink = makeProgressSink({
+ ...opts,
+ log: (m) => lines.push(m),
+ onProgress: (p) => seen.push(p),
+ });
+ return { sink, seen, lines };
+}
+
+test("a fresh copy's fraction is transferred over the measured tree", () => {
+ const { sink, seen } = collect({ totalBytes: 100 });
+ sink(frame(25, 25));
+ sink(frame(100, 100));
+ assert.deepEqual(
+ seen.map((p) => p.fraction),
+ [0.25, 1],
+ );
+ assert.equal(seen[0].bytes, 25);
+});
+
+// RSYNC COUNTS WHAT *IT* SENT, NOT WHAT IS THERE. Resume a copy that died at
+// 60 % and the bar climbs 0 → 40 % and stops, having moved every remaining
+// byte — which an operator reads as a copy that wedged four fifths of the way
+// through something. The caller measures the destination before the copy; this
+// is what that measurement is for.
+test("a resumed copy is offset by what is already on the far side", () => {
+ const { sink, seen } = collect({ totalBytes: 100, alreadyBytes: 60 });
+ sink(frame(0, 0));
+ sink(frame(20, 50));
+ sink(frame(40, 100));
+ assert.deepEqual(
+ seen.map((p) => p.fraction),
+ [0.6, 0.8, 1],
+ );
+ assert.deepEqual(
+ seen.map((p) => p.bytes),
+ [60, 80, 100],
+ );
+ // The detail line says the same thing, so the log and the bar cannot
+ // disagree about the same instant.
+ assert.match(seen[1].detail, /80 B of 100 B · 80 %/);
+});
+
+// A resumed copy re-sends the partial file it died inside, so
+// `already + transferred` can exceed the tree by one file's size. That is a
+// bar reading 112 %, which is a bar nobody believes again.
+test("the fraction is clamped when a partial file is re-sent", () => {
+ const { sink, seen } = collect({ totalBytes: 100, alreadyBytes: 90 });
+ sink(frame(30, 100));
+ assert.equal(seen[0].fraction, 1);
+ assert.equal(seen[0].bytes, 100);
+});
+
+// ONE LINE PER DECILE, and a resumed copy does not re-announce the deciles it
+// did not do — which is why the latch lives in the closure and not in a
+// parameter.
+test("the log gets one line per decile, starting from the resume point", () => {
+ const { sink, lines } = collect({ totalBytes: 100, alreadyBytes: 60 });
+ sink(frame(0, 0));
+ sink(frame(1, 2));
+ sink(frame(20, 50));
+ sink(frame(40, 100));
+ assert.equal(lines.length, 3);
+ assert.match(lines[0], /^Copying… 60 B of 100 B · 60 %/);
+ assert.match(lines[2], /100 B of 100 B · 100 %/);
+});
+
+test("a line that is not a frame is ignored entirely", () => {
+ const { sink, seen, lines } = collect({ totalBytes: 100 });
+ sink("sending incremental file list");
+ sink(".d..t...... 20240101_test1234567/");
+ assert.deepEqual(seen, []);
+ assert.deepEqual(lines, []);
+});
diff --git a/common/controller/relocateDir.ts b/common/controller/relocateDir.ts
@@ -0,0 +1,399 @@
+import path from "node:path";
+import {
+ lstat,
+ readdir,
+ readFile,
+ readlink,
+ rename,
+ rm,
+ stat,
+ writeFile,
+} from "node:fs/promises";
+import { execa } from "execa";
+import {
+ formatRsyncProgressDetail,
+ parseRsyncProgress,
+ rsyncProgressFraction,
+} from "../jobs/progressParsers";
+import { formatBytes } from "../lib/format";
+import type {
+ RelocationDirection,
+ RelocationMarker,
+ RelocationPhase,
+} from "../lib/channelMedia";
+
+// MOVING A DIRECTORY TO ANOTHER DRIVE AND LEAVING A SYMLINK — the mechanism,
+// with no opinion about whose directory it is.
+//
+// This is `relocateChannelMedia.ts`'s core, lifted out unchanged so a SECOND
+// thing can be moved by it: the saved-video store. What stayed behind in that
+// file is everything specific to a channel — `config.dataDir`, the parked
+// `data.relocated-*` siblings, the social-channel refusal, `inspectChannelMedia`
+// — because none of it generalises and pretending it did would be the worse
+// abstraction.
+//
+// What IS shared is the part the omnimirror incident and three rounds of resume
+// bugs were paid for: measure the tree, copy with `--partial` so an abort is
+// resumable, verify with a dry run AND a re-measure, tolerate directory-mtime
+// drift exactly once, and dispatch every step past the copy on what is ON DISK
+// rather than on what the previous step was supposed to have done.
+//
+// THE SOURCE IS NEVER TOUCHED UNTIL THE COPY VERIFIES. That is the invariant
+// every caller inherits and none may weaken.
+
+// ---------------------------------------------------------------------------
+// Measuring
+// ---------------------------------------------------------------------------
+
+// One walk, used by the preview, the space check and the verify. Follows no
+// symlinks (a relocated tree is never the SOURCE of another relocation).
+export async function measureTree(
+ dir: string,
+): Promise<{ bytes: number; files: number }> {
+ let bytes = 0;
+ let files = 0;
+ const stack = [dir];
+ while (stack.length > 0) {
+ const cur = stack.pop() as string;
+ let entries;
+ try {
+ entries = await readdir(cur, { withFileTypes: true });
+ } catch {
+ continue;
+ }
+ for (const e of entries) {
+ const p = path.join(cur, e.name);
+ if (e.isDirectory()) {
+ stack.push(p);
+ } else if (e.isFile()) {
+ try {
+ const st = await stat(p);
+ bytes += st.size;
+ files++;
+ } catch {
+ /* vanished mid-walk; the verify is what catches real drift */
+ }
+ }
+ }
+ }
+ return { bytes, files };
+}
+
+export async function pathExists(p: string): Promise<boolean> {
+ try {
+ await stat(p);
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+export async function isDirectory(p: string): Promise<boolean> {
+ try {
+ return (await stat(p)).isDirectory();
+ } catch {
+ return false;
+ }
+}
+
+// WHAT A PATH ACTUALLY IS RIGHT NOW. lstat, never stat: a dangling symlink —
+// the exact state a half-finished move or an unmounted drive leaves behind —
+// reads as ABSENT through stat, and the code that then tries to create the link
+// fails with EEXIST on a path it was just told was not there.
+export type LinkOrDirState =
+ | { kind: "missing" }
+ | { kind: "real-dir" }
+ | { kind: "link"; linkTarget: string }
+ | { kind: "other" };
+
+export async function linkOrDirState(p: string): Promise<LinkOrDirState> {
+ let st;
+ try {
+ st = await lstat(p);
+ } catch {
+ return { kind: "missing" };
+ }
+ if (st.isSymbolicLink()) {
+ return { kind: "link", linkTarget: await readlink(p).catch(() => "") };
+ }
+ if (st.isDirectory()) return { kind: "real-dir" };
+ return { kind: "other" };
+}
+
+// ---------------------------------------------------------------------------
+// The marker
+// ---------------------------------------------------------------------------
+
+// THE MARKER FILE IS A CONTRACT WITH THE PROCESS THAT COMES NEXT, and it is
+// deliberately identical in shape for a channel and for the saved-video store:
+// `{ target, direction, startedAt, phase }`. A live editor mid-move has already
+// written these; a later binary must resume them, so nothing here may gain a
+// REQUIRED field. See common/lib/channelMedia.ts, which is where the type lives
+// and where the channel guards read it.
+export type DirRelocationMarker = RelocationMarker;
+
+export async function writeDirMarker(
+ file: string,
+ marker: DirRelocationMarker,
+): Promise<void> {
+ const tmp = `${file}.tmp-${process.pid}`;
+ await writeFile(tmp, JSON.stringify(marker, null, 2) + "\n");
+ await rename(tmp, file);
+}
+
+export async function clearDirMarker(file: string): Promise<void> {
+ await rm(file, { force: true });
+}
+
+export async function readDirMarker(
+ file: string,
+): Promise<DirRelocationMarker | null> {
+ try {
+ const raw = await readFile(file, "utf8");
+ const r = JSON.parse(raw) as Partial<DirRelocationMarker>;
+ if (typeof r.target !== "string") return null;
+ return {
+ target: r.target,
+ direction: (r.direction === "back" ? "back" : "out") as RelocationDirection,
+ startedAt: typeof r.startedAt === "string" ? r.startedAt : "",
+ phase:
+ r.phase === "swap" || r.phase === "reclaim"
+ ? r.phase
+ : ("copy" as RelocationPhase),
+ };
+ } catch {
+ return null;
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Progress
+// ---------------------------------------------------------------------------
+
+// WHAT A COPY IN FLIGHT LOOKS LIKE FROM OUTSIDE, once per rsync redraw.
+//
+// `fraction` is against the MEASURED tree, never rsync's own percentage — see
+// parseRsyncProgress for why that one walks backwards under incremental
+// recursion.
+export type RelocationProgress = {
+ bytes: number;
+ totalBytes: number;
+ fraction: number;
+ rate: string;
+ etaSeconds: number;
+ // "12.3 GB of 45.6 GB · 27 % · 110.50MB/s · ETA 5:32". One wording, shared by
+ // the task detail and the decile log line.
+ detail: string;
+};
+
+// Turn rsync's redraws into `RelocationProgress`, and into one log line per
+// decile. The decile latch is why this is a closure and not a function: "every
+// 10 %" is a fact about the run, and a copy that resumes at 80 % must not
+// re-announce the eight deciles it did not do.
+export function makeProgressSink(opts: {
+ totalBytes: number;
+ // BYTES ALREADY ON THE FAR SIDE WHEN THIS RUN STARTED — a resumed copy's
+ // offset, and 0 for a fresh one.
+ //
+ // rsync counts what IT transferred, not what is there: resume a copy that
+ // died at 60 % and the bar climbs 0 → 40 % and stops, having moved every
+ // remaining byte. The operator reads that as a copy that wedged four fifths
+ // of the way through something. Adding the head start makes the fraction a
+ // statement about the TREE, which is what the bar claims to be about, and it
+ // is why the caller measures the destination before the copy rather than
+ // letting rsync's own numbers stand.
+ alreadyBytes?: number;
+ log: (m: string) => void;
+ onProgress?: (p: RelocationProgress) => void;
+}): (line: string) => void {
+ let lastDecile = -1;
+ const already = Math.max(0, opts.alreadyBytes ?? 0);
+ return (line: string) => {
+ const raw = parseRsyncProgress(line);
+ if (!raw) return;
+ // Clamped to the total: a resumed copy can re-send a partial file, so
+ // `already + transferred` may exceed the tree by the size of one file.
+ const done = Math.min(opts.totalBytes || Infinity, already + raw.bytes);
+ const shifted = { ...raw, bytes: done };
+ const fraction = rsyncProgressFraction(shifted, opts.totalBytes);
+ const detail = formatRsyncProgressDetail(
+ shifted,
+ opts.totalBytes,
+ formatBytes,
+ );
+ opts.onProgress?.({
+ bytes: done,
+ totalBytes: opts.totalBytes,
+ fraction,
+ rate: raw.rate,
+ etaSeconds: raw.etaSeconds,
+ detail,
+ });
+ const decile = Math.min(10, Math.floor(fraction * 10));
+ if (decile > lastDecile) {
+ lastDecile = decile;
+ opts.log(`Copying… ${detail}`);
+ }
+ };
+}
+
+// ---------------------------------------------------------------------------
+// rsync
+// ---------------------------------------------------------------------------
+
+// rsync, exactly as backupSavedVideos does it: the real binary from
+// paths.rsyncBin, cancellable, streaming its output into the job log.
+export async function rsyncTree(opts: {
+ rsyncBin: string;
+ src: string;
+ dest: string;
+ args: string[];
+ log: (m: string) => void;
+ // Fed every `--info=progress2` redraw. When set, those redraws are kept OUT
+ // of the log: one 131 GB copy is several thousand frames of one line, and the
+ // sink writes a decile line instead. Everything rsync says that is not a
+ // progress frame still goes to the log verbatim.
+ progress?: (line: string) => void;
+ signal?: AbortSignal;
+}): Promise<{ exitCode: number; output: string }> {
+ // Trailing slash on src: copy the CONTENTS, so <src>/ -> <dest>/ and not
+ // <dest>/data/. Getting this wrong is a silently nested corpus.
+ const args = [...opts.args, `${opts.src}/`, `${opts.dest}/`];
+ opts.log(`$ ${opts.rsyncBin} ${args.join(" ")}`);
+ const child = execa(opts.rsyncBin, args, {
+ cancelSignal: opts.signal,
+ all: true,
+ buffer: false,
+ reject: false,
+ });
+ let output = "";
+ // THE TAIL OF THE LAST CHUNK, HELD BACK UNTIL ITS DELIMITER ARRIVES.
+ //
+ // A chunk boundary falls wherever the pipe decides, and it lands INSIDE a
+ // progress frame often enough to matter on a long copy. Parsing the two
+ // halves separately is not merely a dropped frame: ` 30,000,` matches the
+ // number, the percent and the rate of nothing, and the leading ` 30` of
+ // the next chunk can parse as a frame reporting THIRTY BYTES — so the bar
+ // jumps back to 0 % and the decile latch has already spent its line. So the
+ // trailing segment (the one with no delimiter after it) is carried into the
+ // next chunk, which is what every line-oriented stream reader has to do.
+ let pending = "";
+ child.all?.on("data", (c: Buffer) => {
+ const text = c.toString("utf8");
+ // The verify reads this whole buffer back (driftLines), so it is
+ // accumulated verbatim whatever the log ends up holding.
+ output += text;
+ if (!opts.progress) {
+ opts.log(text);
+ return;
+ }
+ // rsync rewrites the progress line in place with carriage returns, so one
+ // chunk carries many frames. Split on both, exactly as taskHooks does for
+ // yt-dlp — but keep the last segment back unless the chunk ended on a
+ // delimiter, because only then is it a whole line.
+ const segments = (pending + text).split(/[\r\n]+/);
+ pending = /[\r\n]$/.test(text) ? "" : (segments.pop() ?? "");
+ let passedThrough = "";
+ for (const part of segments) {
+ if (part.trim() === "") continue;
+ if (parseRsyncProgress(part) === null) {
+ passedThrough += `${part}\n`;
+ continue;
+ }
+ opts.progress(part);
+ }
+ if (passedThrough) opts.log(passedThrough);
+ });
+ const result = await child;
+ // THE LAST LINE HAS NO DELIMITER AFTER IT. rsync's final `100%` frame is
+ // exactly that, and it is the one the decile latch needs to reach 10.
+ if (pending.trim() !== "") {
+ if (opts.progress && parseRsyncProgress(pending) !== null) {
+ opts.progress(pending);
+ } else {
+ opts.log(`${pending}\n`);
+ }
+ pending = "";
+ }
+ return { exitCode: result.exitCode ?? 1, output };
+}
+
+// The copy arguments, in one place so the two movers cannot drift: `-a`
+// preserves mtimes (which is what makes the LMDB index a no-op afterwards),
+// `--partial` keeps an aborted transfer resumable, `--info=progress2` is what
+// the sink above reads.
+export const COPY_ARGS = ["-a", "--partial", "--info=progress2"];
+
+// A DIRECTORY MTIME IS NOT CONTENT. `.d..t` is rsync's itemization for "this is
+// a directory and only its modification time differs" — nothing to send, and no
+// byte of the copy is in question. It is what the omnimirror move hit
+// (2026-09-13): a sidecar written into one video directory while the copy was
+// already past it bumped that directory's mtime on the SOURCE and left the
+// target's behind, the verify saw one drift line, and a 131 GB copy refused at
+// the last step with nothing actually wrong.
+const DIR_MTIME_ONLY = /^\.d\.\.t/;
+
+function driftLines(output: string): string[] {
+ return output
+ .split("\n")
+ .map((l) => l.trim())
+ .filter((l) => l.length > 0 && !l.startsWith("sending incremental"))
+ .filter((l) => !/^(sent|total size|$)/.test(l));
+}
+
+// What a copy has to clear before the swap: rsync itself agrees there is
+// nothing left to send, AND the two trees measure the same. The dry run alone
+// would accept a target that is byte-identical for the wrong reason; the counts
+// alone would accept two trees of equal size with different contents.
+export async function verifyCopy(opts: {
+ rsyncBin: string;
+ src: string;
+ dest: string;
+ log: (m: string) => void;
+ signal?: AbortSignal;
+}): Promise<{ bytes: number; files: number; retried: boolean }> {
+ let retried = false;
+ // ONE retry, never a loop: if a second pass does not settle it, something is
+ // still writing into the tree and the answer is to refuse, not to chase it.
+ for (;;) {
+ const { exitCode, output } = await rsyncTree({
+ ...opts,
+ args: ["-a", "--dry-run", "--itemize-changes"],
+ });
+ if (exitCode !== 0) {
+ throw new Error(`Verification rsync failed (exit ${exitCode})`);
+ }
+ const drift = driftLines(output);
+ if (drift.length === 0) break;
+ if (!retried && drift.every((l) => DIR_MTIME_ONLY.test(l))) {
+ retried = true;
+ opts.log(
+ `Verification found ${drift.length} directory timestamp(s) differing and ` +
+ `no content drift — running one more rsync pass to settle them.`,
+ );
+ const again = await rsyncTree({ ...opts, args: ["-a"] });
+ if (again.exitCode !== 0) {
+ throw new Error(
+ `Verification rsync failed (exit ${again.exitCode}). ` +
+ `The source has NOT been touched.`,
+ );
+ }
+ continue;
+ }
+ throw new Error(
+ `Verification failed: ${drift.length} file(s) still differ ` +
+ `(first: ${drift[0]}). The source has NOT been touched.`,
+ );
+ }
+ const [a, b] = await Promise.all([
+ measureTree(opts.src),
+ measureTree(opts.dest),
+ ]);
+ if (a.files !== b.files || a.bytes !== b.bytes) {
+ throw new Error(
+ `Verification failed: source has ${a.files} file(s)/${formatBytes(a.bytes)}, ` +
+ `target has ${b.files}/${formatBytes(b.bytes)}. The source has NOT been touched.`,
+ );
+ }
+ return { ...a, retried };
+}
diff --git a/common/controller/relocateSavedVideos.test.ts b/common/controller/relocateSavedVideos.test.ts
@@ -0,0 +1,497 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ lstat,
+ mkdir,
+ mkdtemp,
+ readdir,
+ readFile,
+ readlink,
+ rm,
+ symlink,
+ writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import type { SiteSettings } from "../lib/settings";
+import { writeDirMarker } from "./relocateDir";
+import {
+ inspectSavedVideosStore,
+ relocateSavedVideos,
+ relocatedSavedVideosDir,
+ savedVideosMarkerPath,
+} from "./relocateSavedVideos";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/relocateSavedVideos.test.ts
+//
+// These exercise the REAL rsync binary (paths.rsyncBin -> "rsync"), exactly as
+// relocateChannelMedia.test.ts does — it is the same mover underneath
+// (relocateDir.ts), and a fake rsync would test the wiring and not the move.
+// Everything happens inside one mkdtemp; the corpus and the platter are
+// SIBLINGS, never nested, because a root inside the corpus is refused by design.
+
+type Harness = {
+ paths: Paths;
+ root: string;
+ settings: SiteSettings;
+ io: { read: () => SiteSettings; write: (next: SiteSettings) => Promise<void> };
+};
+
+async function withTmp(fn: (h: Harness) => Promise<void>): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-savedvideos-"));
+ const transcriptsDir = path.join(dir, "corpus");
+ const paths = {
+ transcriptsDir,
+ channelsDir: path.join(transcriptsDir, "channels"),
+ savedVideosDir: path.join(transcriptsDir, "saved-videos"),
+ rsyncBin: "rsync",
+ } as Paths;
+ const root = path.join(dir, "platter");
+ await mkdir(paths.channelsDir, { recursive: true });
+ await mkdir(root, { recursive: true });
+ let settings = {
+ minFreeDiskGB: 0,
+ resumeMarginGB: 0,
+ storage: {
+ locations: [
+ { id: "cold", label: "Cold", root, autoRepoint: false },
+ ],
+ defaultLocationId: "cold",
+ },
+ } as unknown as SiteSettings;
+ const io = {
+ read: () => settings,
+ write: async (next: SiteSettings) => {
+ settings = next;
+ },
+ };
+ try {
+ await fn({
+ paths,
+ root,
+ get settings() {
+ return settings;
+ },
+ io,
+ } as Harness);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+async function seedStore(paths: Paths): Promise<void> {
+ const one = path.join(paths.savedVideosDir, "chan", "vid1");
+ await mkdir(one, { recursive: true });
+ await writeFile(path.join(one, "source-media.mp4"), "x".repeat(4096));
+ const two = path.join(paths.savedVideosDir, "chan", "vid2");
+ await mkdir(two, { recursive: true });
+ await writeFile(path.join(two, "source-media.mkv"), "y".repeat(2048));
+}
+
+test("the store moves onto a location, leaves a link and records where it went", async () => {
+ await withTmp(async (h) => {
+ await seedStore(h.paths);
+ const result = await relocateSavedVideos({
+ paths: h.paths,
+ locationId: "cold",
+ io: h.io,
+ onLog: () => {},
+ });
+ const target = relocatedSavedVideosDir(h.root);
+ assert.equal(result.target, target);
+ assert.equal(result.files, 2);
+ assert.equal(result.bytes, 4096 + 2048);
+
+ // A symlink where the store was, pointing at the target.
+ const st = await lstat(h.paths.savedVideosDir);
+ assert.equal(st.isSymbolicLink(), true);
+ assert.equal(await readlink(h.paths.savedVideosDir), target);
+ // Every reader keeps working, through the link, unchanged.
+ assert.equal(
+ await readFile(
+ path.join(h.paths.savedVideosDir, "chan", "vid1", "source-media.mp4"),
+ "utf8",
+ ),
+ "x".repeat(4096),
+ );
+ // The record is written only on success.
+ assert.equal(h.io.read().storage.savedVideosLocationId, "cold");
+ // No marker, and no parked second copy on the volume the move freed.
+ assert.equal(
+ (await readdir(h.paths.transcriptsDir)).filter((n) =>
+ n.startsWith("saved-videos."),
+ ).length,
+ 0,
+ );
+ assert.equal(
+ await readDirMarkerExists(savedVideosMarkerPath(h.paths)),
+ false,
+ );
+
+ const store = await inspectSavedVideosStore(h.paths, h.io.read());
+ assert.equal(store.status, "ok");
+ assert.equal(store.locationId, "cold");
+ assert.equal(store.target, target);
+ });
+});
+
+test("and back again: a real directory, the record cleared, the target reclaimed", async () => {
+ await withTmp(async (h) => {
+ await seedStore(h.paths);
+ await relocateSavedVideos({
+ paths: h.paths,
+ locationId: "cold",
+ io: h.io,
+ onLog: () => {},
+ });
+ const target = relocatedSavedVideosDir(h.root);
+ const back = await relocateSavedVideos({
+ paths: h.paths,
+ locationId: "",
+ io: h.io,
+ onLog: () => {},
+ });
+ assert.equal(back.locationId, "");
+ assert.equal(back.files, 2);
+ assert.equal((await lstat(h.paths.savedVideosDir)).isDirectory(), true);
+ assert.equal(h.io.read().storage.savedVideosLocationId, undefined);
+ assert.equal(await exists(target), false);
+ assert.equal(
+ await readFile(
+ path.join(h.paths.savedVideosDir, "chan", "vid2", "source-media.mkv"),
+ "utf8",
+ ),
+ "y".repeat(2048),
+ );
+ assert.equal(
+ (await inspectSavedVideosStore(h.paths, h.io.read())).status,
+ "in-place",
+ );
+ });
+});
+
+// A KILLED COPY IS RESUMED, NOT RESTARTED. The marker says which phase the dead
+// run reached; rsync skips what is already correctly on the far side, so a copy
+// that died at 90 % costs a verify pass and not the other 90 % again.
+test("an interrupted copy resumes from its marker", async () => {
+ await withTmp(async (h) => {
+ await seedStore(h.paths);
+ const target = relocatedSavedVideosDir(h.root);
+ // What a killed copy leaves: a partial target and a phase-copy marker.
+ await mkdir(path.join(target, "chan", "vid1"), { recursive: true });
+ await writeFile(
+ path.join(target, "chan", "vid1", "source-media.mp4"),
+ "x".repeat(4096),
+ );
+ await writeDirMarker(savedVideosMarkerPath(h.paths), {
+ target,
+ direction: "out",
+ startedAt: new Date().toISOString(),
+ phase: "copy",
+ });
+ // inspect() calls that in-transition — which is what every guard must see.
+ assert.equal(
+ (await inspectSavedVideosStore(h.paths, h.io.read())).status,
+ "in-transition",
+ );
+
+ const result = await relocateSavedVideos({
+ paths: h.paths,
+ locationId: "cold",
+ io: h.io,
+ onLog: () => {},
+ });
+ assert.equal(result.resumed, true);
+ assert.equal((await lstat(h.paths.savedVideosDir)).isSymbolicLink(), true);
+ assert.equal(
+ await readDirMarkerExists(savedVideosMarkerPath(h.paths)),
+ false,
+ );
+ });
+});
+
+// SAME TARGET, OPPOSITE INTENT. Finishing a move-out as a move-back would swap
+// the wrong way round, so it is refused by name rather than resumed.
+test("a marker for the other direction is refused, not resumed", async () => {
+ await withTmp(async (h) => {
+ await seedStore(h.paths);
+ await writeDirMarker(savedVideosMarkerPath(h.paths), {
+ target: relocatedSavedVideosDir(h.root),
+ direction: "back",
+ startedAt: new Date().toISOString(),
+ phase: "copy",
+ });
+ await assert.rejects(
+ relocateSavedVideos({
+ paths: h.paths,
+ locationId: "cold",
+ io: h.io,
+ onLog: () => {},
+ }),
+ /in flight or was interrupted/,
+ );
+ });
+});
+
+test("an unknown location is refused before anything is written", async () => {
+ await withTmp(async (h) => {
+ await seedStore(h.paths);
+ await assert.rejects(
+ relocateSavedVideos({
+ paths: h.paths,
+ locationId: "nope",
+ io: h.io,
+ onLog: () => {},
+ }),
+ /no storage location "nope"/,
+ );
+ assert.equal((await lstat(h.paths.savedVideosDir)).isDirectory(), true);
+ assert.equal(
+ await readDirMarkerExists(savedVideosMarkerPath(h.paths)),
+ false,
+ );
+ });
+});
+
+// A ROOT INSIDE THE CORPUS WOULD COPY THE STORE ONTO ITSELF and then reclaim
+// the only copy — the same trap relocationRootProblem exists for on a channel.
+test("a root inside the corpus is refused", async () => {
+ await withTmp(async (h) => {
+ await seedStore(h.paths);
+ const inside = path.join(h.paths.transcriptsDir, "inside");
+ await mkdir(inside, { recursive: true });
+ const settings = h.io.read();
+ await h.io.write({
+ ...settings,
+ storage: {
+ ...settings.storage,
+ locations: [
+ ...settings.storage.locations,
+ { id: "bad", label: "Bad", root: inside, autoRepoint: false },
+ ],
+ },
+ });
+ await assert.rejects(
+ relocateSavedVideos({
+ paths: h.paths,
+ locationId: "bad",
+ io: h.io,
+ onLog: () => {},
+ }),
+ /inside the corpus/,
+ );
+ assert.equal((await lstat(h.paths.savedVideosDir)).isDirectory(), true);
+ });
+});
+
+// THE RECORD IS A RECORD, NOT AN INTENTION. Settings that claim a location
+// while the disk says otherwise is `inconsistent` and is never guessed past.
+test("settings and disk disagreeing reads as inconsistent", async () => {
+ await withTmp(async (h) => {
+ await seedStore(h.paths);
+ const settings = h.io.read();
+ await h.io.write({
+ ...settings,
+ storage: { ...settings.storage, savedVideosLocationId: "cold" },
+ });
+ const store = await inspectSavedVideosStore(h.paths, h.io.read());
+ assert.equal(store.status, "inconsistent");
+ assert.match(store.detail ?? "", /is not a symlink/);
+ });
+});
+
+async function exists(p: string): Promise<boolean> {
+ try {
+ await lstat(p);
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+async function readDirMarkerExists(p: string): Promise<boolean> {
+ return exists(p);
+}
+
+// A STORE THAT HAS NEVER EXISTED IS NOT AN RSYNC ERROR. Nothing has ever been
+// persisted on a corpus whose keep-latest window is unset, so there is no
+// `saved-videos` directory at all — and rsync exits 23 on a missing source,
+// which the first cut reported as "rsync failed (exit 23)" AFTER writing the
+// marker, leaving the store stuck in transition over a directory that was never
+// there.
+test("a store that has never existed moves cleanly, leaving an empty one", async () => {
+ await withTmp(async (h) => {
+ // Deliberately no seedStore.
+ const result = await relocateSavedVideos({
+ paths: h.paths,
+ locationId: "cold",
+ io: h.io,
+ onLog: () => {},
+ });
+ assert.equal(result.files, 0);
+ const target = relocatedSavedVideosDir(h.root);
+ assert.equal(await readlink(h.paths.savedVideosDir), target);
+ assert.equal(h.io.read().storage.savedVideosLocationId, "cold");
+ assert.equal(await exists(savedVideosMarkerPath(h.paths)), false);
+ // And the first persist after the move lands on the platter, through the
+ // link, with nothing special asked of it.
+ await mkdir(path.join(h.paths.savedVideosDir, "chan", "vid9"), {
+ recursive: true,
+ });
+ await writeFile(
+ path.join(h.paths.savedVideosDir, "chan", "vid9", "source-media.mp4"),
+ "z",
+ );
+ assert.equal(
+ await readFile(path.join(target, "chan", "vid9", "source-media.mp4"), "utf8"),
+ "z",
+ );
+ });
+});
+
+// THE TARGET IS DELETED ONLY WHEN THIS RUN CAN VOUCH FOR THE COPY STANDING IN
+// FOR IT. "`store` is a real directory" is not evidence: it is also what
+// `rsync --copy-links` of the corpus produces (WORKTREES.md documents that as
+// the way to carry a store into a shard) and what `inconsistent` looks like.
+// The old code rm -rf'd the relocated copy on the strength of it.
+test("move back refuses to delete a target the local copy cannot account for", async () => {
+ await withTmp(async (h) => {
+ // The state: settings say the store is on cold and the target holds two
+ // files, but `saved-videos` is a REAL directory holding only one.
+ const target = relocatedSavedVideosDir(h.root);
+ await mkdir(path.join(target, "chan", "vid1"), { recursive: true });
+ await writeFile(
+ path.join(target, "chan", "vid1", "source-media.mp4"),
+ "x".repeat(4096),
+ );
+ await mkdir(path.join(target, "chan", "vid2"), { recursive: true });
+ await writeFile(
+ path.join(target, "chan", "vid2", "source-media.mkv"),
+ "y".repeat(2048),
+ );
+ await mkdir(path.join(h.paths.savedVideosDir, "chan", "vid1"), {
+ recursive: true,
+ });
+ await writeFile(
+ path.join(h.paths.savedVideosDir, "chan", "vid1", "source-media.mp4"),
+ "x".repeat(4096),
+ );
+ const settings = h.io.read();
+ await h.io.write({
+ ...settings,
+ storage: { ...settings.storage, savedVideosLocationId: "cold" },
+ });
+
+ const lines: string[] = [];
+ const result = await relocateSavedVideos({
+ paths: h.paths,
+ locationId: "",
+ io: h.io,
+ onLog: (l) => lines.push(l),
+ });
+ // The reversible half succeeded: the record is cleared and the store is a
+ // real directory. The DELETE is what was refused.
+ assert.equal(h.io.read().storage.savedVideosLocationId, undefined);
+ assert.equal(await isDir(target), true);
+ assert.match(lines.join("\n"), /was NOT deleted/);
+ assert.equal(result.locationId, "");
+ // No stuck marker over it.
+ assert.equal(await exists(savedVideosMarkerPath(h.paths)), false);
+ });
+});
+
+test("move back does delete the target once the local copy accounts for it", async () => {
+ await withTmp(async (h) => {
+ await seedStore(h.paths);
+ await relocateSavedVideos({
+ paths: h.paths,
+ locationId: "cold",
+ io: h.io,
+ onLog: () => {},
+ });
+ const target = relocatedSavedVideosDir(h.root);
+ await relocateSavedVideos({
+ paths: h.paths,
+ locationId: "",
+ io: h.io,
+ onLog: () => {},
+ });
+ assert.equal(await exists(target), false);
+ });
+});
+
+// A RERUN MUST NOT TRIP OVER THE LAST ATTEMPT'S PARKED COPY. A fixed
+// `saved-videos.relocated` name meant a rename onto a non-empty directory
+// (ENOTEMPTY); the timestamped name plus a prefix sweep is the channel mover's
+// answer, and it is the one that does not orphan an earlier attempt's copy on
+// the volume the move exists to free.
+test("a stale parked copy is swept, not renamed onto", async () => {
+ await withTmp(async (h) => {
+ await seedStore(h.paths);
+ // What an earlier, crashed attempt left beside the store.
+ const stale = path.join(
+ path.dirname(h.paths.savedVideosDir),
+ "saved-videos.relocated-1600000000000",
+ );
+ await mkdir(path.join(stale, "chan", "old"), { recursive: true });
+ await writeFile(path.join(stale, "chan", "old", "source-media.mp4"), "old");
+
+ await relocateSavedVideos({
+ paths: h.paths,
+ locationId: "cold",
+ io: h.io,
+ onLog: () => {},
+ });
+ // Both the stale copy and this run's parked one are gone.
+ const left = (
+ await readdir(path.dirname(h.paths.savedVideosDir))
+ ).filter((n) => n.startsWith("saved-videos."));
+ assert.deepEqual(left, []);
+ });
+});
+
+// A ROOT THAT IS A SYMLINK BACK INTO THE CORPUS is the lexical check's blind
+// spot, and it is the one that copies the store onto itself and then reclaims
+// the only copy.
+test("a root that only RESOLVES inside the corpus is refused", async () => {
+ await withTmp(async (h) => {
+ await seedStore(h.paths);
+ const inside = path.join(h.paths.transcriptsDir, "inside");
+ await mkdir(inside, { recursive: true });
+ // A path outside the corpus that is a link to a path inside it.
+ const sneaky = path.join(path.dirname(h.paths.transcriptsDir), "sneaky");
+ await symlink(inside, sneaky);
+ const settings = h.io.read();
+ await h.io.write({
+ ...settings,
+ storage: {
+ ...settings.storage,
+ locations: [
+ ...settings.storage.locations,
+ { id: "sneaky", label: "Sneaky", root: sneaky, autoRepoint: false },
+ ],
+ },
+ });
+ await assert.rejects(
+ relocateSavedVideos({
+ paths: h.paths,
+ locationId: "sneaky",
+ io: h.io,
+ onLog: () => {},
+ }),
+ /inside the corpus/,
+ );
+ assert.equal(await isDir(h.paths.savedVideosDir), true);
+ assert.equal(await exists(savedVideosMarkerPath(h.paths)), false);
+ });
+});
+
+async function isDir(p: string): Promise<boolean> {
+ try {
+ return (await lstat(p)).isDirectory();
+ } catch {
+ return false;
+ }
+}
diff --git a/common/controller/relocateSavedVideos.ts b/common/controller/relocateSavedVideos.ts
@@ -0,0 +1,730 @@
+import path from "node:path";
+import {
+ access,
+ constants as fsConstants,
+ mkdir,
+ readdir,
+ realpath,
+ rename,
+ rm,
+ symlink,
+ unlink,
+} from "node:fs/promises";
+import type { Paths } from "../lib/paths";
+import { getFreeBytes } from "../lib/diskSpace";
+import { formatBytes } from "../lib/format";
+import {
+ getSettings,
+ writeSettings,
+ type SiteSettings,
+} from "../lib/settings";
+import type { RelocationMarker, RelocationPhase } from "../lib/channelMedia";
+import {
+ relocatedSavedVideosDir,
+ savedVideosMarkerPath,
+} from "../lib/savedVideoStore";
+import {
+ clearDirMarker,
+ COPY_ARGS,
+ isDirectory,
+ linkOrDirState,
+ makeProgressSink,
+ measureTree,
+ readDirMarker,
+ rsyncTree,
+ verifyCopy,
+ writeDirMarker,
+ type RelocationProgress,
+} from "./relocateDir";
+
+// MOVING THE SAVED-VIDEO STORE TO ANOTHER DRIVE.
+//
+// The store (`transcripts/saved-videos`) is where a persisted source container
+// goes when the per-download retention rule decides to keep one — the single
+// largest thing in the corpus that is not a channel's `data/`, and the one
+// directory a channel move could never reach. `plans/storage-locations.md`
+// recorded it as "follow-up, not here"; this is the follow-up.
+//
+// SAME MECHANISM, SAME INVARIANT. `<store>` becomes an absolute SYMLINK to
+// `<root>/saved-videos` and the copy is verified before the source is touched,
+// exactly as a channel's `data/` does it — `relocateDir.ts` is literally the
+// same code. Nothing that reads the store has to know: `savedVideoRoot()`
+// (lib/savedVideo.ts) returns `paths.savedVideosDir` as it always did and the
+// kernel follows the link.
+//
+// WHAT IS DIFFERENT FROM A CHANNEL, and it is one thing: the record of where it
+// went is `settings.storage.savedVideosLocationId`, not a per-channel config
+// field. So the swap's last step is a settings write rather than a
+// writeChannelConfig, and a rollback has one file to put back instead of n.
+//
+// THE PER-CHANNEL OVERRIDE IS NOT MOVED. `ChannelConfig.savedVideosDir` lets a
+// channel point its store somewhere else entirely; a channel that has one is
+// not in this store and this move neither reads nor rewrites it. That is
+// deliberate — it is an absolute path the operator set, and silently
+// re-anchoring it would be this function deciding something it was not asked.
+
+// THE MARKER'S NAME AND ITS READER LIVE IN lib/, not here — see
+// lib/savedVideoStore.ts. `savedVideo-server.ts`, which persists a container
+// and is the thing that must be refused while a move is in flight, is lib and
+// may not import controller. Re-exported so every existing importer of this
+// module keeps working; this file WRITES the marker, it does not own it.
+export {
+ SAVED_VIDEOS_DIRNAME,
+ SAVED_VIDEOS_MARKER_FILENAME,
+ relocatedSavedVideosDir,
+ savedVideosMarkerPath,
+} from "../lib/savedVideoStore";
+
+// THE PARKED NAME IS TIMESTAMPED AND SWEPT BY PREFIX, exactly as the channel
+// mover's `data.relocated-<ts>` is, and for its reason: a rerun that renamed
+// onto a parked directory left by an earlier attempt gets ENOTEMPTY, and a
+// sweep that took only the name THIS process minted would orphan the earlier
+// one for ever — a full second copy of the store, on the volume the move exists
+// to free.
+const PARKED_PREFIX = "saved-videos.relocated-";
+const INCOMING_NAME = "saved-videos.incoming";
+
+export type SavedVideosStoreStatus =
+ // The store is a real directory under the corpus.
+ | "in-place"
+ // Relocated, link and settings agree, and the target is a reachable dir.
+ | "ok"
+ // Relocated, but the target is not there — almost always an unmounted drive.
+ | "unreachable"
+ // A move is in flight, or one was interrupted: the marker is present.
+ | "in-transition"
+ // Disk and settings disagree, in either direction. Never guessed past.
+ | "inconsistent";
+
+export type SavedVideosStore = {
+ // Always `paths.savedVideosDir` — the path every reader uses, moved or not.
+ dir: string;
+ // `settings.storage.savedVideosLocationId`, or "" for in place.
+ locationId: string;
+ // Where the bytes actually are (the link target), when it is relocated.
+ target?: string;
+ status: SavedVideosStoreStatus;
+ detail?: string;
+ marker?: RelocationMarker;
+};
+
+// WHAT THE STORE IS RIGHT NOW, from the disk first and the settings second.
+//
+// The same shape and the same discipline as `inspectChannelMedia`: lstat (never
+// stat, so a dangling link is not read as absent), and any disagreement between
+// the link and the record is `inconsistent` rather than a guess.
+export async function inspectSavedVideosStore(
+ paths: Paths,
+ settings?: SiteSettings,
+): Promise<SavedVideosStore> {
+ const s = settings ?? getSettings();
+ const dir = paths.savedVideosDir;
+ const locationId = s.storage.savedVideosLocationId ?? "";
+ const marker = await readDirMarker(savedVideosMarkerPath(paths));
+ const state = await linkOrDirState(dir);
+ const loc = s.storage.locations.find((l) => l.id === locationId);
+ const expected = loc ? relocatedSavedVideosDir(loc.root) : "";
+
+ if (marker) {
+ return {
+ dir,
+ locationId,
+ ...(state.kind === "link" ? { target: state.linkTarget } : {}),
+ status: "in-transition",
+ detail: `a move (${marker.direction}) to ${marker.target} is in flight or was interrupted at phase "${marker.phase}"`,
+ marker,
+ };
+ }
+ if (state.kind === "link") {
+ const target = state.linkTarget;
+ if (expected && path.resolve(target) !== path.resolve(expected)) {
+ return {
+ dir,
+ locationId,
+ target,
+ status: "inconsistent",
+ detail: `the store links to ${target}, but settings record location "${locationId}" at ${expected}`,
+ };
+ }
+ if (!expected) {
+ return {
+ dir,
+ locationId,
+ target,
+ status: "inconsistent",
+ detail: `the store links to ${target}, but no storage location is recorded for it`,
+ };
+ }
+ if (!(await isDirectory(target))) {
+ return {
+ dir,
+ locationId,
+ target,
+ status: "unreachable",
+ detail: `${target} is not reachable (is the drive mounted?)`,
+ };
+ }
+ return { dir, locationId, target, status: "ok" };
+ }
+ if (expected) {
+ return {
+ dir,
+ locationId,
+ status: "inconsistent",
+ detail: `settings record the store on location "${locationId}", but ${dir} is not a symlink`,
+ };
+ }
+ if (state.kind === "other") {
+ return {
+ dir,
+ locationId: "",
+ status: "inconsistent",
+ detail: `${dir} is neither a directory nor a symlink`,
+ };
+ }
+ return { dir, locationId: "", status: "in-place" };
+}
+
+export type SavedVideosRelocateResult = {
+ // "" when the store was moved back in place.
+ locationId: string;
+ target: string;
+ bytes: number;
+ files: number;
+ resumed: boolean;
+ retried: boolean;
+};
+
+type Opts = {
+ paths: Paths;
+ // "" moves the store back in place; any other value is a location id.
+ locationId: string;
+ onLog?: (line: string) => void;
+ onProgress?: (p: RelocationProgress) => void;
+ signal?: AbortSignal;
+ // Injectable for unit tests, exactly as the storage controller does it.
+ io?: { read: () => SiteSettings; write: (next: SiteSettings) => Promise<void> };
+};
+
+const BYTES_PER_GB = 1024 ** 3;
+
+export async function relocateSavedVideos(
+ opts: Opts,
+): Promise<SavedVideosRelocateResult> {
+ const io = opts.io ?? { read: getSettings, write: writeSettings };
+ const log = opts.onLog ?? ((m: string) => console.log(m));
+ const paths = opts.paths;
+ const markerFile = savedVideosMarkerPath(paths);
+ const existing = await readDirMarker(markerFile);
+ const wantBack = opts.locationId.trim() === "";
+ const direction = wantBack ? "back" : "out";
+
+ // A MARKER FOR THE OTHER DIRECTION IS NEVER RESUMED INTO THIS ONE. Same
+ // target, opposite intent — the channel mover refuses this for the reason
+ // that finishing a move-out as a move-back swaps the wrong way round.
+ if (existing && existing.direction !== direction) {
+ throw new Error(
+ `A saved-video store move (${existing.direction}) to ${existing.target} is ` +
+ `in flight or was interrupted — finish that before moving ${direction}.`,
+ );
+ }
+ return wantBack
+ ? moveStoreBack({ ...opts, io, log, markerFile, resume: existing })
+ : moveStoreOut({ ...opts, io, log, markerFile, resume: existing });
+}
+
+type Inner = Opts & {
+ io: NonNullable<Opts["io"]>;
+ log: (m: string) => void;
+ markerFile: string;
+ resume: RelocationMarker | null;
+};
+
+async function stampMarker(
+ file: string,
+ target: string,
+ direction: "out" | "back",
+ phase: RelocationPhase,
+): Promise<void> {
+ await writeDirMarker(file, {
+ target,
+ direction,
+ startedAt: new Date().toISOString(),
+ phase,
+ });
+}
+
+async function moveStoreOut(a: Inner): Promise<SavedVideosRelocateResult> {
+ const { paths, log, markerFile, io } = a;
+ const settings = io.read();
+ const loc = settings.storage.locations.find(
+ (l) => l.id === a.locationId.trim(),
+ );
+ if (!loc) {
+ throw new Error(
+ `There is no storage location "${a.locationId}". Add it on /storage.`,
+ );
+ }
+ const store = paths.savedVideosDir;
+ const target = relocatedSavedVideosDir(loc.root);
+ if (a.resume && a.resume.target !== target) {
+ throw new Error(
+ `A move to ${a.resume.target} is already in progress — finish or clear it ` +
+ `before moving to ${target}.`,
+ );
+ }
+
+ // PREFLIGHT. Everything that can refuse does so before a byte is written and
+ // before the marker exists.
+ if (!(await isDirectory(loc.root))) {
+ throw new Error(
+ `The destination root ${loc.root} does not exist or is not a directory ` +
+ `(is the drive mounted?)`,
+ );
+ }
+ try {
+ await access(loc.root, fsConstants.W_OK);
+ } catch {
+ throw new Error(`The destination root ${loc.root} is not writable`);
+ }
+ // RESOLVED THROUGH EVERY SYMLINK, on BOTH sides. A lexical comparison is
+ // exactly the check the channel mover learned not to make: `/mnt/x` can be a
+ // link back into the corpus, and the corpus dir itself is routinely a link
+ // (the worktrees' `test-transcripts`, a bind mount in a container). Miss it
+ // and the store is copied onto itself, verified against itself, and then the
+ // "source" is reclaimed — which is the only copy.
+ const [realRoot, realCorpus] = await Promise.all([
+ realOrResolved(loc.root),
+ realOrResolved(paths.transcriptsDir),
+ ]);
+ if (isWithin(realCorpus, realRoot)) {
+ throw new Error(
+ `The destination root ${loc.root} is inside the corpus at ` +
+ `${paths.transcriptsDir} — the store would be copied onto itself and ` +
+ `then reclaimed. Pick a directory on the other drive.`,
+ );
+ }
+ // Belt and braces for a root outside the corpus whose `saved-videos` level is
+ // a link back into it: the target resolves separately, because realpath of
+ // the root cannot see through a link one level down.
+ const realTarget = await realOrResolved(relocatedSavedVideosDir(realRoot));
+ const realStore = await realOrResolved(paths.savedVideosDir);
+ if (isWithin(realCorpus, realTarget) || isWithin(realStore, realTarget)) {
+ throw new Error(
+ `The destination ${target} resolves inside the corpus at ` +
+ `${paths.transcriptsDir} — the store would be copied onto itself and ` +
+ `then reclaimed. Pick a directory on the other drive.`,
+ );
+ }
+ const state = await linkOrDirState(store);
+ if (!a.resume && state.kind !== "real-dir" && state.kind !== "missing") {
+ throw new Error(
+ `${store} is not a real directory (it is a ${state.kind}) — the store is ` +
+ `already moved, or something else is there.`,
+ );
+ }
+ // A STORE THAT HAS NEVER EXISTED IS NOT AN RSYNC ERROR. Nothing has ever been
+ // persisted on a corpus whose keep-latest window is unset, so there is no
+ // `saved-videos` directory at all — and rsync exits 23 on a missing source,
+ // which the old code reported as "rsync failed (exit 23)" AFTER writing the
+ // marker, leaving the store stuck in transition over a directory that was
+ // never there. Create it empty and let the move proceed: the operator asked
+ // for the store to live on the platter, and an empty store on the platter is
+ // exactly that answer, ready for the first persist.
+ if (state.kind === "missing") {
+ await mkdir(store, { recursive: true });
+ log(`${store} did not exist yet — created it empty before the move.`);
+ }
+
+ let phase: RelocationPhase = a.resume?.phase ?? "copy";
+ const measured = await measureTree(store);
+ let retried = false;
+ log(
+ `Moving the saved-video store: ${measured.files} file(s), ` +
+ `${formatBytes(measured.bytes)} -> ${target}`,
+ );
+
+ if (phase === "copy") {
+ const marginGB =
+ settings.minFreeDiskGB > 0 ? settings.resumeMarginGB : 0;
+ const needed = measured.bytes + marginGB * BYTES_PER_GB;
+ const free = await getFreeBytes(loc.root);
+ if (free < needed) {
+ throw new Error(
+ `Not enough space on ${loc.root}: ${formatBytes(free)} free, ` +
+ `${formatBytes(measured.bytes)} to move` +
+ (marginGB > 0
+ ? ` plus a ${marginGB} GB resume margin = ${formatBytes(needed)} required`
+ : " required"),
+ );
+ }
+ await mkdir(target, { recursive: true });
+ // What a previous attempt already landed — see the channel mover.
+ const already = (await measureTree(target)).bytes;
+ await stampMarker(markerFile, target, "out", "copy");
+ const { exitCode } = await rsyncTree({
+ rsyncBin: paths.rsyncBin,
+ src: store,
+ dest: target,
+ args: COPY_ARGS,
+ log,
+ progress: makeProgressSink({
+ totalBytes: measured.bytes,
+ alreadyBytes: already,
+ log,
+ onProgress: a.onProgress,
+ }),
+ signal: a.signal,
+ });
+ if (a.signal?.aborted) {
+ throw new Error(
+ `Cancelled. ${store} is untouched and the partial copy at ${target} is ` +
+ `resumable — rerun to continue.`,
+ );
+ }
+ if (exitCode !== 0) throw new Error(`rsync failed (exit ${exitCode})`);
+ log("Verifying the copy…");
+ retried ||= (
+ await verifyCopy({
+ rsyncBin: paths.rsyncBin,
+ src: store,
+ dest: target,
+ log,
+ signal: a.signal,
+ })
+ ).retried;
+ phase = "swap";
+ }
+
+ if (phase === "swap") {
+ await stampMarker(markerFile, target, "out", "swap");
+ // OBSERVE, DO NOT ASSUME — a crash lands between any two syscalls, and the
+ // marker can only say which phase it was in, never how far through it got.
+ const now = await linkOrDirState(store);
+ const parked = path.join(
+ path.dirname(store),
+ `${PARKED_PREFIX}${Date.now()}`,
+ );
+ if (now.kind === "real-dir") {
+ // Re-verify: a resumed run did not do the copy in this process and must
+ // not take the interrupted one's word for it.
+ log("Verifying the copy…");
+ retried ||= (
+ await verifyCopy({
+ rsyncBin: paths.rsyncBin,
+ src: store,
+ dest: target,
+ log,
+ signal: a.signal,
+ })
+ ).retried;
+ await rename(store, parked);
+ } else if (now.kind === "other") {
+ throw new Error(
+ `${store} is neither a directory nor a symlink — refusing to replace it`,
+ );
+ }
+ // THE LINK IS MADE DEFENSIVELY, AND THE RECORD IS WRITTEN ONLY IF IT EXISTS.
+ //
+ // `after.kind === "missing"` was the whole condition, and it is not enough:
+ // `moveFileCrossDevice` (savedVideo-server.ts) does an unconditional
+ // `mkdir -p` of the destination's parent, so a persist firing in the
+ // instant between the rename above and this line RECREATES `saved-videos`
+ // as a real, empty directory. The old code then saw `real-dir`, made no
+ // link, and went on to record the store as living on a location it could
+ // not be reached at — a settings field pointing at bytes nothing follows.
+ // (The guard in lib/savedVideoStore.ts is what closes that window; this is
+ // what makes the outcome safe if it is ever open anyway.)
+ let after = await linkOrDirState(store);
+ if (
+ after.kind === "link" &&
+ path.resolve(after.linkTarget) !== path.resolve(target)
+ ) {
+ throw new Error(
+ `${store} already points at ${after.linkTarget}, not ${target}`,
+ );
+ }
+ if (after.kind === "real-dir") {
+ // An empty directory is the mkdir -p above and nothing else — remove it
+ // and link. A NON-empty one holds bytes this move did not copy, and
+ // deleting it is not this function's call to make.
+ const stray = await readdir(store).catch(() => ["keep"] as string[]);
+ if (stray.length > 0) {
+ throw new Error(
+ `${store} is a real directory again and is not empty (${stray.length} ` +
+ `entr(ies)) — something wrote into the store during the move. The ` +
+ `copy at ${target} is complete and untouched; move those entries ` +
+ `aside and rerun.`,
+ );
+ }
+ log(
+ `${store} was recreated as an empty directory during the swap ` +
+ `(a persist raced the move) — removing it and linking.`,
+ );
+ await rm(store, { recursive: false, force: true }).catch(async () => {
+ await rm(store, { recursive: true, force: true });
+ });
+ after = await linkOrDirState(store);
+ }
+ if (after.kind === "missing") {
+ await symlink(target, store);
+ after = await linkOrDirState(store);
+ }
+ if (after.kind !== "link") {
+ throw new Error(
+ `${store} is not a symlink after the swap (it is ${after.kind}) — ` +
+ `refusing to record the store as relocated. The copy at ${target} is ` +
+ `complete; nothing has been deleted.`,
+ );
+ }
+ // THE RECORD IS WRITTEN ONLY NOW: on success, after the link exists and has
+ // been read back. It is a statement about where bytes are, and a statement
+ // nothing can follow is worse than no statement.
+ const latest = io.read();
+ await io.write({
+ ...latest,
+ storage: { ...latest.storage, savedVideosLocationId: loc.id },
+ });
+ log(`Swapped: ${store} -> ${target}`);
+ await stampMarker(markerFile, target, "out", "reclaim");
+ }
+
+ await reclaimParked(paths, log);
+ await clearDirMarker(markerFile);
+ log(
+ `Done. ${formatBytes(measured.bytes)} now on "${loc.label || loc.id}"; ` +
+ `${formatBytes(await getFreeBytes(paths.transcriptsDir))} free on the corpus volume.`,
+ );
+ return {
+ locationId: loc.id,
+ target,
+ bytes: measured.bytes,
+ files: measured.files,
+ resumed: Boolean(a.resume),
+ retried,
+ };
+}
+
+async function moveStoreBack(a: Inner): Promise<SavedVideosRelocateResult> {
+ const { paths, log, markerFile, io } = a;
+ const settings = io.read();
+ const store = paths.savedVideosDir;
+ const state = await linkOrDirState(store);
+ // The link is the first source of truth; the marker is the second, because a
+ // crash after the swap leaves settings cleared and only the marker naming the
+ // target still holding the bytes.
+ const target =
+ state.kind === "link"
+ ? state.linkTarget
+ : (a.resume?.target ??
+ (() => {
+ const loc = settings.storage.locations.find(
+ (l) => l.id === (settings.storage.savedVideosLocationId ?? ""),
+ );
+ return loc ? relocatedSavedVideosDir(loc.root) : "";
+ })());
+ if (!target) {
+ throw new Error(
+ "The saved-video store is not relocated — it is already in place.",
+ );
+ }
+
+ const incoming = path.join(path.dirname(store), INCOMING_NAME);
+ let phase: RelocationPhase = a.resume?.phase ?? "copy";
+ if (phase === "copy" && !(await isDirectory(target))) {
+ throw new Error(
+ `The relocated store at ${target} is not reachable (is the drive mounted?)`,
+ );
+ }
+ let retried = false;
+ const measured = (await isDirectory(target))
+ ? await measureTree(target)
+ : (await isDirectory(incoming))
+ ? await measureTree(incoming)
+ : { bytes: 0, files: 0 };
+ log(
+ `Moving the saved-video store back in place: ${measured.files} file(s), ` +
+ `${formatBytes(measured.bytes)} <- ${target}`,
+ );
+
+ if (phase === "copy") {
+ const marginGB =
+ settings.minFreeDiskGB > 0 ? settings.resumeMarginGB : 0;
+ const already = (await isDirectory(incoming))
+ ? (await measureTree(incoming)).bytes
+ : 0;
+ const needed =
+ Math.max(0, measured.bytes - already) + marginGB * BYTES_PER_GB;
+ const free = await getFreeBytes(paths.transcriptsDir);
+ if (free < needed) {
+ throw new Error(
+ `Not enough space on the corpus volume: ${formatBytes(free)} free, ` +
+ `${formatBytes(Math.max(0, measured.bytes - already))} still to move back` +
+ (marginGB > 0
+ ? ` plus a ${marginGB} GB resume margin = ${formatBytes(needed)} required`
+ : " required"),
+ );
+ }
+ await mkdir(incoming, { recursive: true });
+ await stampMarker(markerFile, target, "back", "copy");
+ const { exitCode } = await rsyncTree({
+ rsyncBin: paths.rsyncBin,
+ src: target,
+ dest: incoming,
+ args: COPY_ARGS,
+ log,
+ progress: makeProgressSink({
+ totalBytes: measured.bytes,
+ // The figure the space check above is priced in.
+ alreadyBytes: already,
+ log,
+ onProgress: a.onProgress,
+ }),
+ signal: a.signal,
+ });
+ if (a.signal?.aborted) {
+ throw new Error(
+ `Cancelled. ${target} is untouched and ${incoming} is resumable — rerun to continue.`,
+ );
+ }
+ if (exitCode !== 0) throw new Error(`rsync failed (exit ${exitCode})`);
+ log("Verifying the copy…");
+ retried ||= (
+ await verifyCopy({
+ rsyncBin: paths.rsyncBin,
+ src: target,
+ dest: incoming,
+ log,
+ signal: a.signal,
+ })
+ ).retried;
+ phase = "swap";
+ }
+
+ // MAY THE TARGET BE DELETED AT THE END? Only when this run can vouch for the
+ // copy that is standing in for it.
+ //
+ // The old code reclaimed the target whenever `store` was a real directory,
+ // and "a real directory" is not evidence of anything: it is ALSO what
+ // `rsync --copy-links` of the corpus produces (WORKTREES.md documents that as
+ // the way to carry a store into a shard), and it is what the `inconsistent`
+ // state looks like — settings naming a location while the disk holds a real
+ // dir. In both cases `rm -rf target` deletes the relocated copy on the
+ // strength of a local directory nobody compared it with.
+ //
+ // Three things count as vouching, and nothing else does: this run did the
+ // swap itself from a verified `incoming`; the marker says a previous run got
+ // past the swap (`phase: "reclaim"`, which is written only after it); or the
+ // local tree measures at least as large as the target's, which is the
+ // cheapest honest answer when neither of the first two applies.
+ let vouched = a.resume?.phase === "reclaim";
+
+ if (phase === "swap") {
+ await stampMarker(markerFile, target, "back", "swap");
+ const now = await linkOrDirState(store);
+ if (now.kind === "real-dir") {
+ log(`${store} is already a real directory — the swap had completed`);
+ } else if (now.kind === "other") {
+ throw new Error(
+ `${store} is neither a directory nor a symlink — refusing to replace it`,
+ );
+ } else {
+ if (!(await isDirectory(incoming))) {
+ throw new Error(
+ `Cannot finish the move back: ${store} is not a directory and there ` +
+ `is no verified copy at ${incoming}`,
+ );
+ }
+ // unlink, not rm -r: `store` is the LINK here, and removing it
+ // recursively would be the one way this design eats the media.
+ if (now.kind !== "missing") await unlink(store);
+ await rename(incoming, store);
+ // THIS run moved a verified copy into place. Nothing is more vouched
+ // than that.
+ vouched = true;
+ log(`Swapped: ${store} is a real directory again`);
+ }
+ const latest = io.read();
+ if ((latest.storage.savedVideosLocationId ?? "") !== "") {
+ const { savedVideosLocationId: _dropped, ...storage } = latest.storage;
+ await io.write({ ...latest, storage });
+ }
+ await stampMarker(markerFile, target, "back", "reclaim");
+ }
+
+ // THE LAST MEASUREMENT BEFORE THE ONLY DESTRUCTIVE STEP. Cheap — two walks of
+ // a store that holds one container per pinned video — and it is the answer to
+ // "is what I am about to delete still the only copy".
+ if (!vouched && (await isDirectory(target))) {
+ const [here, there] = await Promise.all([
+ measureTree(store),
+ measureTree(target),
+ ]);
+ vouched = here.bytes >= there.bytes && here.files >= there.files;
+ if (!vouched) {
+ // NOT AN ERROR, AND DELIBERATELY NOT: the move back has succeeded as far
+ // as anything reversible goes — the store is a real directory and the
+ // record is cleared. What is refused is the DELETE, and leaving a second
+ // copy on the platter is the safe half of that decision. The marker is
+ // cleared so the store is not stuck in transition over it.
+ await reclaimParked(paths, log);
+ await clearDirMarker(markerFile);
+ log(
+ `The store is in place, but ${target} was NOT deleted: it holds ` +
+ `${there.files} file(s)/${formatBytes(there.bytes)} against ` +
+ `${here.files}/${formatBytes(here.bytes)} here, so this run cannot ` +
+ `vouch that the local copy is complete. Compare them and remove it ` +
+ `by hand.`,
+ );
+ return {
+ locationId: "",
+ target,
+ bytes: here.bytes,
+ files: here.files,
+ resumed: Boolean(a.resume),
+ retried,
+ };
+ }
+ }
+ await rm(target, { recursive: true, force: true });
+ await reclaimParked(paths, log);
+ await clearDirMarker(markerFile);
+ log(`Done. ${formatBytes(measured.bytes)} back in place.`);
+ return {
+ locationId: "",
+ target,
+ bytes: measured.bytes,
+ files: measured.files,
+ resumed: Boolean(a.resume),
+ retried,
+ };
+}
+
+// Every leftover a crashed run can have parked beside the store, swept on every
+// path — the channel mover's `sweepParked`, for the reason it exists there: a
+// crash between the settings write and the reclaim marker otherwise orphans a
+// full second copy of the store on the volume the move exists to free.
+async function reclaimParked(
+ paths: Paths,
+ log: (m: string) => void,
+): Promise<void> {
+ const parent = path.dirname(paths.savedVideosDir);
+ const names = await readdir(parent).catch(() => [] as string[]);
+ for (const n of names) {
+ if (!n.startsWith(PARKED_PREFIX) && n !== INCOMING_NAME) continue;
+ const p = path.join(parent, n);
+ log(`Reclaiming ${p}`);
+ await rm(p, { recursive: true, force: true });
+ }
+}
+
+// Resolved through every symlink when the path exists, lexically when it does
+// not — `relocateChannelMedia.ts`'s helper, same name, same reason.
+async function realOrResolved(p: string): Promise<string> {
+ return await realpath(p).catch(() => path.resolve(p));
+}
+
+function isWithin(parent: string, child: string): boolean {
+ const rel = path.relative(parent, child);
+ return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
+}
diff --git a/common/controller/storageLocations.ts b/common/controller/storageLocations.ts
@@ -1,6 +1,7 @@
import path from "node:path";
import { readlink, stat, symlink, unlink } from "node:fs/promises";
import { getPaths, type Paths } from "../lib/paths";
+import { getFreeBytes } from "../lib/diskSpace";
import {
getSettings,
writeSettings,
@@ -82,6 +83,14 @@ export type LocationRollup = {
// Every channel whose `config.dataDir` is under this location's root, sorted.
slugs: string[];
total: number;
+ // Sum of `snapshot.totalMediaBytes` over the channels on this location, and
+ // how many of them could not contribute one (no snapshot, or one written
+ // before the field existed). THE SECOND NUMBER IS WHY THE FIRST IS HONEST: a
+ // location whose channels have never had a report reads `0 bytes` otherwise,
+ // which on a storage page is a claim that a 500 GB drive is empty. Every
+ // surface renders "+ n unknown" beside the total.
+ bytes: number;
+ unknownBytes: number;
// `inspectChannelMedia` status, bucketed into the three numbers the page
// shows. `unreachable` DELIBERATELY ABSORBS `inconsistent`: both mean "this
// channel's media is not readable through its link right now", which is the
@@ -95,9 +104,37 @@ export type LocationRollup = {
};
function emptyRollup(locationId: string): LocationRollup {
- return { locationId, slugs: [], total: 0, ok: 0, unreachable: 0, moving: 0 };
+ return {
+ locationId,
+ slugs: [],
+ total: 0,
+ bytes: 0,
+ unknownBytes: 0,
+ ok: 0,
+ unreachable: 0,
+ moving: 0,
+ };
}
+// THE SYNTHETIC LOCATION: the corpus volume itself.
+//
+// `paths.channelsDir` is where a channel's media lives when it has not been
+// moved anywhere, and that is the row the operator actually acts on — "what is
+// still on the internal disk, largest first" is the whole question a storage
+// page is asked. It is NOT stored in settings and never will be: there is
+// nothing to configure (the root is wherever the corpus is), nothing to
+// re-point (re-pointing the corpus is moving the corpus), and writing it into
+// `settings.storage.locations` would make it deletable and would make
+// `locationOfDataDir` match every unrelocated channel — which would break the
+// one rule the whole design rests on, that a channel is on a location iff its
+// `dataDir` is under that location's root, and an in-place channel HAS no
+// `dataDir`.
+//
+// So it is assembled where it is rendered, out of the same three facts every
+// other row carries, and it is the one row with no actions.
+export const INTERNAL_LOCATION_ID = "internal";
+export const INTERNAL_LOCATION_LABEL = "Internal (in place)";
+
// One pass over the corpus for EVERY location, not one pass per location: the
// page draws a row per location and the channel list is the same list for all
// of them. Cost is `listChannelConfigs` (one readdir + one config read per
@@ -110,22 +147,48 @@ export async function channelsOnLocation(opts: {
// Already-read configs, when the caller has them (the /storage shell does
// not, the channels page does).
configs?: ReadonlyArray<{ slug: string; config: ChannelConfig }>;
+ // `snapshot.totalMediaBytes` by slug, INJECTED rather than read here. A
+ // snapshot is up to a megabyte of JSON per channel and the two callers
+ // already hold theirs (`listChannelBriefs`); a controller that went and read
+ // 71 of them to add one integer each would be the third read of the same
+ // file in one render. A slug that is absent (or maps to undefined) counts
+ // towards `unknownBytes`, never towards `bytes`.
+ mediaBytes?: Readonly<Record<string, number | undefined>>;
+ // The in-place row (see INTERNAL_LOCATION_ID). When true, every channel with
+ // NO `dataDir` is rolled up under that id alongside the configured ones.
+ includeInternal?: boolean;
}): Promise<Record<string, LocationRollup>> {
const out: Record<string, LocationRollup> = {};
for (const loc of opts.locations) out[loc.id] = emptyRollup(loc.id);
- if (opts.locations.length === 0) return out;
+ if (opts.includeInternal) {
+ out[INTERNAL_LOCATION_ID] = emptyRollup(INTERNAL_LOCATION_ID);
+ }
+ if (opts.locations.length === 0 && !opts.includeInternal) return out;
const configs = opts.configs ?? (await listChannelConfigs(opts.paths));
for (const { slug, config } of configs) {
const dataDir = config.dataDir?.trim();
- if (!dataDir) continue;
- const loc = locationOfDataDir(dataDir, opts.locations as StorageLocation[]);
- if (!loc) continue;
- const roll = out[loc.id];
+ const loc = dataDir
+ ? locationOfDataDir(dataDir, opts.locations as StorageLocation[])
+ : null;
+ // In place: no recorded dataDir at all. A dataDir under a root NOBODY named
+ // is neither in place nor on a location, and it is deliberately counted in
+ // neither — /storage says so by the totals not adding up to the corpus, and
+ // the remedy is to name that root as a location.
+ const id = loc
+ ? loc.id
+ : !dataDir && opts.includeInternal
+ ? INTERNAL_LOCATION_ID
+ : null;
+ if (!id) continue;
+ const roll = out[id];
roll.slugs.push(slug);
roll.total += 1;
+ const bytes = opts.mediaBytes?.[slug];
+ if (typeof bytes === "number") roll.bytes += bytes;
+ else roll.unknownBytes += 1;
const media = await inspectChannelMedia(opts.paths, slug, config);
- if (media.status === "ok") roll.ok += 1;
+ if (media.status === "ok" || media.status === "in-place") roll.ok += 1;
else if (media.status === "in-transition") roll.moving += 1;
else roll.unreachable += 1;
}
@@ -133,6 +196,45 @@ export async function channelsOnLocation(opts: {
return out;
}
+// HOW MUCH ROOM EACH VOLUME HAS, WITHOUT PROBING.
+//
+// `probeLocation` is up to three subprocesses and is right for /storage, which
+// renders once per navigation. It is wrong for /channels, which renders on
+// every auto-refresh with 71 rows on it — the rule the badge already follows is
+// that TABLES NEVER PROBE.
+//
+// So this is two syscalls per location: one `stat` to find out whether the root
+// is there at all, and `getFreeBytes` only when it is. The stat is what makes
+// the answer honest — `getFreeBytes` walks up to the nearest existing ancestor
+// on ENOENT, so an unmounted root would otherwise report the free space of
+// whatever is mounted over its parent, which on this machine is the disk the
+// operator is trying to empty. Absent → undefined, rendered as "—".
+//
+// `internal` is always present: it is the corpus volume, and the process is
+// reading the corpus out of it.
+export async function volumeFreeBytes(opts: {
+ paths: Paths;
+ locations: readonly StorageLocation[];
+}): Promise<Record<string, number | undefined>> {
+ const out: Record<string, number | undefined> = {};
+ const corpus = await getFreeBytes(opts.paths.channelsDir);
+ out[INTERNAL_LOCATION_ID] = Number.isFinite(corpus) ? corpus : undefined;
+ await Promise.all(
+ opts.locations.map(async (loc) => {
+ const there = await stat(loc.root)
+ .then((st) => st.isDirectory())
+ .catch(() => false);
+ if (!there) {
+ out[loc.id] = undefined;
+ return;
+ }
+ const free = await getFreeBytes(loc.root);
+ out[loc.id] = Number.isFinite(free) ? free : undefined;
+ }),
+ );
+ return out;
+}
+
// ---------------------------------------------------------------------------
// The probe memo
// ---------------------------------------------------------------------------
diff --git a/common/controller/storageWatch.test.ts b/common/controller/storageWatch.test.ts
@@ -0,0 +1,377 @@
+import { beforeEach, test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import type { SiteSettings } from "../lib/settings";
+import {
+ compileLanes,
+ sanitizeChannelPriority,
+} from "../lib/channelPriority";
+import { LANES } from "../lib/autoQueueTypes";
+import {
+ resetStorageWatchSuspicion,
+ runStorageWatchPass,
+} from "./storageWatch";
+
+// THE CONFIRMATION COUNT IS MODULE STATE (see storageWatch.ts rule 3), so each
+// case starts from a clean one — otherwise the second test inherits the first
+// test's suspicions and pauses on what should be its first pass.
+beforeEach(() => resetStorageWatchSuspicion());
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/storageWatch.test.ts
+//
+// NO SUBPROCESS. `bins` is passed as a Paths-shaped object whose findmnt points
+// at a binary that does not exist, so `probeLocation` fails open to "identity
+// unknown" — which is the container case and exactly what this pass must keep
+// working under. What decides here is the ROOT's existence and the channel's
+// own link, which is what the pass actually reads.
+
+// Lane policies whose roots the compiler already wrote. See the seed comment in
+// storageWatch.ts: a corpus whose trees were NEVER compiled keeps its hand-made
+// order, and the watch must not be the thing that switches it over.
+const COMPILED_LANES = (() => {
+ const roots = compileLanes(sanitizeChannelPriority({ channels: {} }), [], []);
+ return Object.fromEntries(
+ LANES.map((lane) => [lane, { enabled: false, root: roots[lane] }]),
+ );
+})();
+
+type H = {
+ paths: Paths;
+ root: string;
+ io: { read: () => SiteSettings; write: (n: SiteSettings) => Promise<void> };
+ writes: number;
+};
+
+async function withTmp(fn: (h: H) => Promise<void>): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-storagewatch-"));
+ const transcriptsDir = path.join(dir, "corpus");
+ const paths = {
+ transcriptsDir,
+ channelsDir: path.join(transcriptsDir, "channels"),
+ findmntBin: path.join(dir, "no-such-findmnt"),
+ udisksctlBin: path.join(dir, "no-such-udisksctl"),
+ } as Paths;
+ const root = path.join(dir, "platter");
+ await mkdir(paths.channelsDir, { recursive: true });
+ await mkdir(root, { recursive: true });
+ const h: H = {
+ paths,
+ root,
+ writes: 0,
+ io: {
+ read: () => settings,
+ write: async (n) => {
+ settings = n;
+ h.writes += 1;
+ },
+ },
+ };
+ let settings = {
+ storage: {
+ locations: [{ id: "cold", label: "Cold", root, autoRepoint: false }],
+ defaultLocationId: "cold",
+ },
+ channelPriority: sanitizeChannelPriority({ channels: {} }),
+ // The compiled-lane flag the writer's seed condition reads. Compiled here,
+ // so the seed path is not taken and the test is about the watch and not
+ // about the migration — `hasCompiledLaneRoots` is exercised by
+ // channelPriorityCompile.test.ts.
+ autoQueue: COMPILED_LANES,
+ } as unknown as SiteSettings;
+ try {
+ await fn(h);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+// A channel whose `data/` is a link to `<root>/<slug>/data`, with the target
+// present or not.
+async function seedRelocated(
+ h: H,
+ slug: string,
+ opts: { targetExists: boolean },
+): Promise<void> {
+ const channelDir = path.join(h.paths.channelsDir, slug);
+ await mkdir(channelDir, { recursive: true });
+ const target = path.join(h.root, slug, "data");
+ if (opts.targetExists) await mkdir(target, { recursive: true });
+ await writeFile(
+ path.join(channelDir, "config.json"),
+ JSON.stringify({ handling: "youtube", dataDir: target }),
+ );
+ await symlink(target, path.join(channelDir, "data"));
+}
+
+function tierOf(h: H, slug: string): string | undefined {
+ return h.io.read().channelPriority.channels[slug]?.tier;
+}
+
+// ONE BAD READ IS A SUSPICION, TWO IN A ROW IS A FACT — availability is a bare
+// stat with a blanket catch, so an EIO or a spun-down disk reads exactly like
+// "not mounted". Most cases here are about what happens once a drive really is
+// gone, so they run the confirming pair and assert on the second.
+async function twoPasses(h: H) {
+ const first = await runStorageWatchPass({
+ paths: h.paths,
+ io: h.io,
+ bins: h.paths,
+ });
+ const second = await runStorageWatchPass({
+ paths: h.paths,
+ io: h.io,
+ bins: h.paths,
+ });
+ return { first, second };
+}
+
+test("a channel whose target is gone is auto-paused, once, in one write", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "gone-a", { targetExists: false });
+ await seedRelocated(h, "gone-b", { targetExists: false });
+ await seedRelocated(h, "fine", { targetExists: true });
+
+ // RUN THE PASSES ONE AT A TIME HERE, not through twoPasses(): the write
+ // count between them is exactly what this case is about.
+ const pass = () =>
+ runStorageWatchPass({ paths: h.paths, io: h.io, bins: h.paths });
+ const first = await pass();
+ // The first pass only suspects — and writes NOTHING, which is the point:
+ // one flaky stat must not rewrite the corpus's priority document.
+ assert.deepEqual(first.suspected.sort(), ["gone-a", "gone-b"]);
+ assert.deepEqual(first.paused, []);
+ assert.equal(first.wrote, false);
+ assert.equal(h.writes, 0);
+ // The second confirms.
+ const second = await pass();
+ assert.deepEqual(second.paused.sort(), ["gone-a", "gone-b"]);
+ assert.deepEqual(second.restored, []);
+ assert.equal(second.wrote, true);
+ // TWO CHANNELS, ONE WRITE. Ten on a drive that vanished must be one pulse
+ // bump, not ten.
+ assert.equal(h.writes, 1);
+ assert.equal(tierOf(h, "gone-a"), "paused");
+ assert.equal(tierOf(h, "fine"), undefined);
+ assert.equal(
+ h.io.read().channelPriority.channels["gone-a"].autoPaused?.reason,
+ "storage",
+ );
+
+ // A QUIET PASS WRITES NOTHING. The document already describes the world.
+ const third = await runStorageWatchPass({ paths: h.paths, io: h.io, bins: h.paths });
+ assert.deepEqual(third.paused, []);
+ assert.equal(third.wrote, false);
+ assert.equal(h.writes, 1);
+ });
+});
+
+test("the drive coming back restores the tier it overwrote", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "away", { targetExists: false });
+ const settings = h.io.read();
+ await h.io.write({
+ ...settings,
+ channelPriority: sanitizeChannelPriority({
+ channels: { away: { tier: "low", rank: 4 } },
+ }),
+ });
+ h.writes = 0;
+
+ await twoPasses(h);
+ assert.equal(tierOf(h, "away"), "paused");
+
+ await mkdir(path.join(h.root, "away", "data"), { recursive: true });
+ const back = await runStorageWatchPass({ paths: h.paths, io: h.io, bins: h.paths });
+ assert.deepEqual(back.restored, ["away"]);
+ assert.equal(tierOf(h, "away"), "low");
+ assert.equal(h.io.read().channelPriority.channels.away.rank, 4);
+ assert.equal(h.io.read().channelPriority.channels.away.autoPaused, undefined);
+ });
+});
+
+// THE OPERATOR'S WORD WINS. A channel they paused is not the machine's to
+// claim, and one they resumed while the drive was still away must not be
+// re-paused... by a restore. (Re-pausing it on a LATER pass is correct: the
+// drive is still gone. What must not happen is the restore un-pausing a
+// deliberate pause, which is what the no-record no-op guarantees.)
+test("a manually paused channel is never claimed by the watch", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "off", { targetExists: false });
+ const settings = h.io.read();
+ await h.io.write({
+ ...settings,
+ channelPriority: sanitizeChannelPriority({
+ channels: { off: { tier: "paused" } },
+ }),
+ });
+ h.writes = 0;
+ const { second } = await twoPasses(h);
+ assert.deepEqual(second.paused, []);
+ assert.equal(second.wrote, false);
+ assert.equal(h.writes, 0);
+ assert.equal(
+ h.io.read().channelPriority.channels.off.autoPaused,
+ undefined,
+ );
+ });
+});
+
+// A MARKER MEANS A MOVE IS RUNNING OR WAS INTERRUPTED, and the relocate job is
+// precisely the thing an auto-pause would then be refusing.
+test("a channel mid-relocation is not auto-paused", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "moving", { targetExists: false });
+ await writeFile(
+ path.join(h.paths.channelsDir, "moving", ".relocating.json"),
+ JSON.stringify({
+ target: path.join(h.root, "moving", "data"),
+ direction: "out",
+ startedAt: "",
+ phase: "copy",
+ }),
+ );
+ const { second } = await twoPasses(h);
+ assert.deepEqual(second.paused, []);
+ assert.equal(second.wrote, false);
+ });
+});
+
+// A CHANNEL MOVED BACK IN PLACE CARRIES NO DRIVE TO BE AWAY, but it can carry a
+// record from before — which has to come off or it stays paused for ever.
+test("a record on an in-place channel is restored", async () => {
+ await withTmp(async (h) => {
+ const channelDir = path.join(h.paths.channelsDir, "home");
+ await mkdir(path.join(channelDir, "data"), { recursive: true });
+ await writeFile(
+ path.join(channelDir, "config.json"),
+ JSON.stringify({ handling: "youtube" }),
+ );
+ const settings = h.io.read();
+ await h.io.write({
+ ...settings,
+ channelPriority: sanitizeChannelPriority({
+ channels: {
+ home: {
+ tier: "paused",
+ autoPaused: {
+ reason: "storage",
+ since: "2026-09-20T00:00:00.000Z",
+ previousTier: "normal",
+ },
+ },
+ },
+ }),
+ });
+ h.writes = 0;
+ const r = await runStorageWatchPass({ paths: h.paths, io: h.io, bins: h.paths });
+ assert.deepEqual(r.restored, ["home"]);
+ assert.equal(h.io.read().channelPriority.channels.home, undefined);
+ });
+});
+
+// `write: false` IS IDLE BOOT. It observes and reports; the write is the work,
+// and idle boot refuses work.
+test("write: false reports the transition and changes nothing", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "gone", { targetExists: false });
+ // The first pass only suspects, whatever `write` says.
+ const opts = { paths: h.paths, io: h.io, bins: h.paths, write: false };
+ assert.deepEqual((await runStorageWatchPass(opts)).suspected, ["gone"]);
+ const r = await runStorageWatchPass(opts);
+ assert.deepEqual(r.paused, ["gone"]);
+ assert.equal(r.wrote, false);
+ assert.equal(h.writes, 0);
+ assert.equal(tierOf(h, "gone"), undefined);
+ });
+});
+
+test("no locations and nothing auto-paused is a free pass", async () => {
+ await withTmp(async (h) => {
+ const settings = h.io.read();
+ await h.io.write({
+ ...settings,
+ storage: { locations: [], defaultLocationId: "" },
+ });
+ h.writes = 0;
+ const r = await runStorageWatchPass({ paths: h.paths, io: h.io, bins: h.paths });
+ assert.deepEqual(r, {
+ probed: 0,
+ suspected: [],
+ repointed: [],
+ paused: [],
+ restored: [],
+ wrote: false,
+ });
+ });
+});
+
+// ONE BAD READ MUST NOT PAUSE A TIER. Availability is a bare `stat` with a
+// blanket catch (storageVolumes.ts), so an EIO on a flaky cable or a disk that
+// has spun down and needs a beat to answer is indistinguishable from "not
+// mounted" — and pausing on it rewrites the corpus's priority document for a
+// drive that is fine.
+test("a drive that blips for one pass is never paused", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "blip", { targetExists: false });
+ const first = await runStorageWatchPass({
+ paths: h.paths,
+ io: h.io,
+ bins: h.paths,
+ });
+ assert.deepEqual(first.suspected, ["blip"]);
+ assert.deepEqual(first.paused, []);
+ assert.equal(h.writes, 0);
+
+ // It answers on the next pass. Nothing was ever paused, and the suspicion
+ // is dropped — so a LATER real outage starts its own two-pass count rather
+ // than pausing immediately on the strength of a blip an hour ago.
+ await mkdir(path.join(h.root, "blip", "data"), { recursive: true });
+ const second = await runStorageWatchPass({
+ paths: h.paths,
+ io: h.io,
+ bins: h.paths,
+ });
+ assert.deepEqual(second.paused, []);
+ assert.deepEqual(second.restored, []);
+ assert.equal(second.wrote, false);
+ assert.equal(h.writes, 0);
+
+ // Prove the suspicion really was dropped: the drive going away again takes
+ // two fresh passes.
+ await rm(path.join(h.root, "blip"), { recursive: true, force: true });
+ assert.deepEqual(
+ (await runStorageWatchPass({ paths: h.paths, io: h.io, bins: h.paths }))
+ .paused,
+ [],
+ );
+ assert.deepEqual(
+ (await runStorageWatchPass({ paths: h.paths, io: h.io, bins: h.paths }))
+ .paused,
+ ["blip"],
+ );
+ });
+});
+
+// RESTORE STAYS SINGLE-PASS, and the asymmetry is the point: being slow to
+// pause costs a few refused units (the start-of-work guards catch those), while
+// being slow to restore leaves a lane off after the operator fixed the cable.
+test("the restore needs only one good pass", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "back", { targetExists: false });
+ await twoPasses(h);
+ assert.equal(tierOf(h, "back"), "paused");
+ h.writes = 0;
+ await mkdir(path.join(h.root, "back", "data"), { recursive: true });
+ const r = await runStorageWatchPass({
+ paths: h.paths,
+ io: h.io,
+ bins: h.paths,
+ });
+ assert.deepEqual(r.restored, ["back"]);
+ assert.equal(h.writes, 1);
+ });
+});
diff --git a/common/controller/storageWatch.ts b/common/controller/storageWatch.ts
@@ -0,0 +1,345 @@
+import { getPaths, type Paths } from "../lib/paths";
+import {
+ getSettings,
+ writeSettings,
+ type SiteSettings,
+} from "../lib/settings";
+import {
+ autoPausedSlugs,
+ autoPauseForMedia,
+ channelPriorityFromLegacy,
+ compileLanes,
+ hasCompiledLaneRoots,
+ isDefaultChannelPriority,
+ resolveFocusSlugs,
+ restoreAfterMedia,
+ sanitizeChannelPriority,
+ type ChannelPriority,
+} from "../lib/channelPriority";
+import { LANES } from "../lib/autoQueueTypes";
+import { siteChannelIndex } from "../lib/site";
+import { inspectChannelMedia } from "../lib/channelMedia";
+import { locationOfDataDir } from "../lib/storageLocations";
+import type { VolumeBins } from "../lib/storageVolumes";
+import { listChannelConfigs } from "./channels";
+import { maybeAutoRepoint, probeAllLocations } from "./storageLocations";
+
+// THE DRIVE WENT AWAY WHILE THE CHANNEL WAS ON — now what.
+//
+// Operator ask, 2026-09-18: "Can that gracefully handle the case if the drive
+// disconnects while the channel is on? Maybe an automatic disable and flag?"
+//
+// Every guard the corpus has for an unreachable channel runs at the START of a
+// piece of work (`runManagedFunction`'s first statement, `buildChannelWork`'s
+// per-tick inspect, the snapshot regen, the operation batch). None of them is a
+// detector: they refuse work that is already being asked for, which means a
+// channel on a drive that vanished sits there being refused, over and over,
+// with nothing anywhere saying why. The flag the operator asked for is a state,
+// and a state needs something that looks.
+//
+// SO THIS LOOKS, ON A CADENCE, AND WRITES AT MOST ONCE PER PASS.
+//
+// Two rules, and they are what keep it from being a settings-churn machine:
+//
+// 1. IT ONLY EVER WRITES ON A TRANSITION. A pass that finds the world exactly
+// as the document already describes it writes nothing — no settings write,
+// so no pulse revision bump, so no reader re-polls. A flapping drive costs
+// two writes per flap, not one per tick.
+// 2. IT RESTORES ONLY WHAT IT PAUSED. `restoreAfterMedia` is a no-op without
+// an `autoPaused` record, and the one priority writer clears that record on
+// any manual tier change — so a drive coming back can never un-pause a
+// channel the operator paused on purpose in the meantime.
+// 3. IT TAKES TWO CONSECUTIVE DOWN PASSES TO PAUSE, AND ONE UP PASS TO
+// RESTORE. Availability is a bare `stat` with a blanket catch
+// (`storageVolumes.ts`) — an EIO on a flaky cable, or a disk that has spun
+// down and needs a beat to answer, reads exactly like "not mounted". One
+// such read would otherwise pause every channel on the drive and rewrite
+// the corpus's priority document. The confirmation is held IN MEMORY, not
+// in settings: a pending suspicion is not a fact worth persisting, and a
+// process restart starting the count again is the safe direction. The
+// asymmetry is deliberate — being slow to pause costs a few refused units
+// (the start-of-work guards catch those), while being slow to RESTORE costs
+// the operator a lane that stays off after they fixed the cable.
+//
+// IT IS A RUNNER, so `ARCHILYZER_IDLE_BOOT` must not arm it: a container
+// pointed at somebody else's corpus for the first time has no business
+// rewriting that corpus's priority document seconds after `docker compose up`.
+// The probe is read-only and cheap; the WRITE is the work, and idle boot
+// refuses work. `runStorageWatchPass({ write: false })` is the observation
+// without the consequence, which is what the boot pass and a test want.
+
+export type StorageWatchResult = {
+ // Locations probed this pass.
+ probed: number;
+ // Channels this pass saw as unreachable for the FIRST time. They are not
+ // paused yet; the next pass decides. Reported so a caller (and the test) can
+ // see the confirmation working rather than infer it from silence.
+ suspected: string[];
+ // Locations whose volume was found at a different mountpoint and for which an
+ // auto re-point was queued. Empty unless a location has `autoRepoint` on.
+ repointed: string[];
+ // Channels newly auto-paused, and channels restored. Both empty on a quiet
+ // pass, which is the overwhelming majority.
+ paused: string[];
+ restored: string[];
+ // Whether settings were written. False whenever both lists are empty.
+ wrote: boolean;
+};
+
+export type StorageWatchOpts = {
+ paths?: Paths;
+ bins?: VolumeBins;
+ io?: { read: () => SiteSettings; write: (next: SiteSettings) => Promise<void> };
+ // False observes and reports without touching settings — idle boot, and the
+ // unit tests.
+ write?: boolean;
+ log?: (line: string) => void;
+};
+
+const DEFAULT_IO = { read: getSettings, write: writeSettings };
+
+// Channels seen down on the LAST pass and not yet paused. Module state, and
+// deliberately not settings: see rule 3 in the header.
+const suspected = new Set<string>();
+
+// Test seam, and the escape hatch for a process that wants a clean count.
+export function resetStorageWatchSuspicion(): void {
+ suspected.clear();
+}
+
+export async function runStorageWatchPass(
+ opts: StorageWatchOpts = {},
+): Promise<StorageWatchResult> {
+ const io = opts.io ?? DEFAULT_IO;
+ const log = opts.log ?? (() => {});
+ const paths = opts.paths ?? getPaths();
+ const settings = io.read();
+ const locations = settings.storage.locations;
+ const out: StorageWatchResult = {
+ probed: 0,
+ suspected: [],
+ repointed: [],
+ paused: [],
+ restored: [],
+ wrote: false,
+ };
+
+ // A channel can only be auto-paused for a location's sake, so with no
+ // locations configured there is nothing to watch — EXCEPT the records a
+ // previous configuration left behind, which must still be restorable. Hence
+ // the early return is on "no locations AND nothing auto-paused".
+ const already = autoPausedSlugs(settings.channelPriority);
+ if (locations.length === 0 && already.length === 0) return out;
+
+ // REFRESH, NOT THE MEMO. The memo exists so a page render does not fork
+ // eighteen subprocesses per click; this pass runs on a cadence measured in
+ // minutes and its whole job is to notice a change, so an answer taken up to
+ // ten seconds ago is not what it is asking for.
+ const probes = await probeAllLocations(locations, opts.bins ?? paths, {
+ refresh: true,
+ });
+ out.probed = Object.keys(probes).length;
+
+ // THE DRIVE CAME UP SOMEWHERE ELSE — FOLLOW IT, IF THE OPERATOR ARMED THAT.
+ //
+ // `mounted-elsewhere` is the one status with a remedy that moves no bytes:
+ // the volume IS here, under a different mountpoint, and a re-point rewrites
+ // the links. Until now only the BOOT pass took it, so a disk that came back
+ // at a new mountpoint while the editor was up sat there while this pass
+ // dutifully paused every channel on it — and the fix was a restart. Same
+ // opt-in (`autoRepoint`), same preflight, same refusal-with-a-reason; what
+ // changes is that the cadence can reach it.
+ //
+ // It runs BEFORE the per-channel loop and the enqueued job runs after this
+ // pass returns, so this pass still sees (and may still suspect) the channels
+ // on that location — which is correct: nothing has moved yet, and the
+ // two-pass confirmation gives the re-point a whole interval to land before
+ // anything is paused.
+ if (opts.write !== false) {
+ for (const loc of locations) {
+ const probe = probes[loc.id];
+ if (!probe || probe.status !== "mounted-elsewhere" || !loc.autoRepoint) {
+ continue;
+ }
+ const outcome = await maybeAutoRepoint({
+ paths,
+ location: loc,
+ probe,
+ bins: opts.bins ?? paths,
+ io,
+ }).catch((err) => ({
+ started: false as const,
+ reason: (err as Error).message,
+ }));
+ if (outcome.started) {
+ out.repointed.push(loc.id);
+ log(
+ `[storage] "${loc.id}": the volume came up at a different mountpoint ` +
+ `— auto re-point queued (${outcome.newRoot})`,
+ );
+ } else {
+ log(`[storage] "${loc.id}": auto re-point declined — ${outcome.reason}`);
+ }
+ }
+ }
+
+ const configs = await listChannelConfigs(paths);
+ let model: ChannelPriority = settings.channelPriority;
+
+ for (const { slug, config } of configs) {
+ const wasAutoPaused = Boolean(model.channels[slug]?.autoPaused);
+ const dataDir = config.dataDir?.trim();
+ if (!dataDir) {
+ // In place. It cannot be on a drive that went away — but it CAN carry a
+ // record from before it was moved back, and that record has to come off
+ // or the channel stays paused for ever.
+ suspected.delete(slug);
+ if (wasAutoPaused) {
+ model = restoreAfterMedia(model, slug);
+ out.restored.push(slug);
+ }
+ continue;
+ }
+ // THE LOCATION'S PROBE FIRST, THE CHANNEL'S OWN STAT SECOND. The probe is
+ // the cheap corpus-wide answer (one findmnt per location, not per channel);
+ // `inspectChannelMedia` is what decides, because a location can be
+ // available while one channel's target under it is missing — a half-done
+ // move, a directory deleted by hand.
+ const loc = locationOfDataDir(dataDir, locations);
+ const probe = loc ? probes[loc.id] : undefined;
+ const locationDown = Boolean(loc) && probe?.status !== "available";
+ const media = await inspectChannelMedia(paths, slug, config);
+ // `in-transition` is NEVER a reason to pause: a marker means a move is
+ // running or was interrupted, and the relocate job is precisely the thing
+ // that would then be refused by the state it created.
+ const down =
+ media.status === "in-transition"
+ ? false
+ : locationDown || media.status === "unreachable";
+
+ if (down && !wasAutoPaused) {
+ // ONE BAD READ IS A SUSPICION, TWO IN A ROW IS A FACT. See rule 3.
+ if (!suspected.has(slug)) {
+ suspected.add(slug);
+ out.suspected.push(slug);
+ log(
+ `[storage] ${slug}: media unreachable (${
+ loc ? `location "${loc.id}" is ${probe?.status ?? "unprobed"}` : media.status
+ }) — waiting for a second pass to confirm before pausing`,
+ );
+ continue;
+ }
+ const before = model;
+ model = autoPauseForMedia(model, slug);
+ // autoPauseForMedia no-ops on a channel the OPERATOR already paused —
+ // which is right, and means "nothing changed" is a normal outcome here.
+ if (model !== before) {
+ out.paused.push(slug);
+ log(
+ `[storage] ${slug}: media unreachable (${
+ loc ? `location "${loc.id}" is ${probe?.status ?? "unprobed"}` : media.status
+ }) — auto-paused`,
+ );
+ }
+ continue;
+ }
+ if (!down) suspected.delete(slug);
+ if (!down && wasAutoPaused) {
+ const restoredTo = model.channels[slug]?.autoPaused?.previousTier;
+ model = restoreAfterMedia(model, slug);
+ out.restored.push(slug);
+ log(`[storage] ${slug}: media reachable again, tier restored to ${restoredTo}`);
+ }
+ }
+
+ if (out.paused.length === 0 && out.restored.length === 0) return out;
+ if (opts.write === false) return out;
+ // ONE WRITE PER PASS, whatever the pass found. Ten channels on one drive that
+ // vanished is one settings write and one pulse bump, not ten.
+ //
+ // FROM A FRESH READ, because a probe of several locations is seconds of wall
+ // clock during which an operator may have changed a tier in another tab —
+ // and clobbering that with a snapshot taken before the pass started would
+ // silently undo it. The edits are re-applied to whatever is current.
+ const latest = io.read();
+ // THE LEGACY SEED, AND WHY A BACKGROUND PASS MUST PAY IT TOO.
+ //
+ // `laneDispatchRoot` is all-or-nothing on `isDefaultChannelPriority`: the
+ // moment the document says ANYTHING, the stored lane trees stop being
+ // dispatched from and compiled ones take over. So a pass that auto-paused one
+ // channel on a corpus whose document was empty would, as a side effect,
+ // replace the operator's hand-made lane order with an all-normal alphabetical
+ // one — silently, at 3am, because a USB cable came loose. That is the same
+ // trap `saveChannelPriorityAction` documents (the S2/S3 review, finding 3),
+ // and it is worse here because nobody clicked anything.
+ //
+ // Same remedy, same condition: when the stored document says nothing AND the
+ // stored trees were never compiled, derive the document the migration would
+ // have produced FIRST and apply the pause on top of that. The corpus's
+ // existing order survives.
+ const stored = latest.channelPriority;
+ const base =
+ isDefaultChannelPriority(stored) && !hasCompiledLaneRoots(latest.autoQueue)
+ ? channelPriorityFromLegacy(
+ configs.map((c) => ({ slug: c.slug, config: {} })),
+ latest.autoQueue,
+ )
+ : stored;
+ let merged: ChannelPriority = base;
+ for (const slug of out.paused) merged = autoPauseForMedia(merged, slug);
+ for (const slug of out.restored) merged = restoreAfterMedia(merged, slug);
+ merged = sanitizeChannelPriority(merged);
+ // AND THE TREES, in the same write. Two writes to one settings file race each
+ // other, and a document that has changed tiers with trees that have not is a
+ // corpus dispatching from an order nobody holds any more. The condition is
+ // the writer's: a still-default document with never-compiled trees keeps its
+ // hand-made ones.
+ const autoQueue = { ...latest.autoQueue };
+ if (!isDefaultChannelPriority(merged) || hasCompiledLaneRoots(latest.autoQueue)) {
+ const slugs = configs.map((c) => c.slug);
+ const focusSlugs = resolveFocusSlugs(merged, siteChannelIndex(paths), slugs);
+ const roots = compileLanes(merged, slugs, focusSlugs);
+ for (const lane of LANES) {
+ autoQueue[lane] = { ...latest.autoQueue[lane], root: roots[lane] };
+ }
+ }
+ await io.write({ ...latest, channelPriority: merged, autoQueue });
+ out.wrote = true;
+ return out;
+}
+
+// ---------------------------------------------------------------------------
+// The cadence
+// ---------------------------------------------------------------------------
+
+// Five minutes. A drive does not come and go on a timescale a person would
+// notice faster than that, and every pass is one findmnt per location plus two
+// stats per relocated channel — cheap, but not free, and this runs for the life
+// of the process.
+export const STORAGE_WATCH_INTERVAL_MS = 5 * 60_000;
+
+let timer: ReturnType<typeof setInterval> | null = null;
+
+// ARMED ONCE PER PROCESS. `unref()` so it never holds the event loop open — a
+// CLI that imports a controller must still exit.
+export function startStorageWatch(
+ opts: StorageWatchOpts & { intervalMs?: number } = {},
+): boolean {
+ if (timer) return false;
+ const every = opts.intervalMs ?? STORAGE_WATCH_INTERVAL_MS;
+ timer = setInterval(() => {
+ void runStorageWatchPass(opts).catch((err) => {
+ (opts.log ?? console.warn)(
+ `[storage] watch pass failed: ${(err as Error).message}`,
+ );
+ });
+ }, every);
+ timer.unref?.();
+ return true;
+}
+
+export function stopStorageWatch(): void {
+ if (!timer) return;
+ clearInterval(timer);
+ timer = null;
+}
diff --git a/common/jobs/jobKinds.ts b/common/jobs/jobKinds.ts
@@ -399,6 +399,24 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
queueKeyStrategy: "custom",
needsMedia: false,
},
+ // THE SAVED-VIDEO STORE, ONTO A LOCATION AND BACK. Same mechanism as the
+ // channel move (relocateDir.ts is literally the same code) over one directory
+ // that belongs to no channel — so no `channelSlug`, and `needsMedia: false`
+ // for the relocation's reason: this kind is what FIXES a store on a drive
+ // that is not there.
+ //
+ // ON THE RELOCATION QUEUE KEY, with the channel move and the re-point. All
+ // three rewrite symlinks under the same roots and all three run their space
+ // check when they START; the registry caps a key at concurrency 1, so one at
+ // a time across the three kinds is the whole point.
+ "relocate-saved-videos": {
+ kind: "relocate-saved-videos",
+ label: "Move the saved-video store",
+ drainable: false,
+ replayable: false,
+ queueKeyStrategy: "custom",
+ needsMedia: false,
+ },
// THE SAME LINKS, WITHOUT THE BYTES. A re-point rewrites every channel
// symlink on one storage location plus the location's root, for the case the
// relocation above cannot help with: the media never moved, the DISK did, and
diff --git a/common/jobs/jobSpec.ts b/common/jobs/jobSpec.ts
@@ -53,6 +53,13 @@ const REPLAY_BUCKETS: ReadonlySet<string> = new Set<ReplayBucket>([
"supersededAutoSubs",
]);
+// Is this bucket name one a replayed job can be re-derived from? Exported so a
+// caller naming a bucket (the ops API's retry-bucket route) can decide whether
+// the job is replayable without re-spelling the set.
+export function isReplayBucket(value: unknown): value is ReplayBucket {
+ return typeof value === "string" && REPLAY_BUCKETS.has(value);
+}
+
// Defensive parse for a spec read back from JSON (the <id>.meta.json sidecar).
// Returns null on anything malformed so a hand-edited or stale file can't
// crash a reader. Mirrors the tolerance of readJobMeta / readWorkerDefaults.
diff --git a/common/jobs/progressParsers.test.ts b/common/jobs/progressParsers.test.ts
@@ -3,7 +3,12 @@ import assert from "node:assert/strict";
import {
createDiarizeProgressParser,
createDownloadProgressParser,
+ formatRsyncProgressDetail,
+ isRsyncProgressLine,
+ parseRsyncProgress,
+ rsyncProgressFraction,
} from "./progressParsers";
+import { formatBytes } from "../lib/format";
// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test jobs/progressParsers.test.ts
@@ -138,3 +143,119 @@ test("the diarize parser ignores everything that is not its own output", () => {
);
assert.equal(p.feed("[download] 50.0% of 1.00GiB at 2.31MiB/s ETA 00:30"), null);
});
+
+// --- rsync --info=progress2 -------------------------------------------------
+//
+// The exact bytes rsync 3.x writes, captured off a real copy (two files, 35 MB)
+// and pasted verbatim: one \r-separated buffer, the early frame with no xfr#
+// suffix, a mid frame, and the two identical final frames rsync always prints.
+const RSYNC_CHUNK =
+ "\r 32,768 0% 0.00kB/s 0:00:00 " +
+ "\r 30,000,000 85% 1.03GB/s 0:00:03 (xfr#1, to-chk=1/3)" +
+ "\r 35,000,000 100% 1.05GB/s 0:00:00 (xfr#2, to-chk=0/3)";
+
+test("rsync progress: bytes, percent, rate and ETA off one frame", () => {
+ const p = parseRsyncProgress(
+ " 30,000,000 85% 1.03GB/s 0:01:03 (xfr#1, to-chk=1/3)",
+ );
+ assert.deepEqual(p, {
+ bytes: 30_000_000,
+ percent: 85,
+ rate: "1.03GB/s",
+ etaSeconds: 63,
+ });
+});
+
+test("rsync progress: a whole \\r buffer splits into frames", () => {
+ const frames = RSYNC_CHUNK.split(/[\r\n]+/)
+ .filter((l) => l.trim() !== "")
+ .map(parseRsyncProgress);
+ assert.equal(frames.length, 3);
+ assert.deepEqual(
+ frames.map((f) => f?.bytes),
+ [32_768, 30_000_000, 35_000_000],
+ );
+ assert.deepEqual(
+ frames.map((f) => f?.percent),
+ [0, 85, 100],
+ );
+});
+
+test("rsync progress: an hours-long ETA is seconds, not a string", () => {
+ assert.equal(
+ parseRsyncProgress(" 1,000 10% 1.00MB/s 2:03:04")?.etaSeconds,
+ 2 * 3600 + 3 * 60 + 4,
+ );
+});
+
+test("rsync progress: everything that is not a frame is not a frame", () => {
+ for (const line of [
+ "sending incremental file list",
+ "20240101_test1234567/",
+ "sent 35,012,345 bytes received 4,321 bytes 70,033,332.00 bytes/sec",
+ "total size is 35,000,000 speedup is 1.00",
+ ".d..t...... 20240101_test1234567/",
+ "",
+ "rsync: [sender] change_dir failed: No such file or directory (2)",
+ ]) {
+ assert.equal(parseRsyncProgress(line), null, line);
+ assert.equal(isRsyncProgressLine(line), false, line);
+ }
+});
+
+// THE FRACTION IS AGAINST THE MEASURED TREE, not rsync's percentage — which
+// under incremental recursion is a percentage of what it has enumerated so far
+// and walks backwards. The controller has already measured the whole tree for
+// its space check, so it has a denominator that only moves one way.
+test("rsync progress: the fraction divides by the measured tree", () => {
+ const p = parseRsyncProgress(" 30,000,000 85% 1.03GB/s 0:00:03")!;
+ assert.equal(rsyncProgressFraction(p, 60_000_000), 0.5);
+ // No measurement (a caller that never walked the tree) falls back to rsync's
+ // own percentage rather than reporting zero.
+ assert.equal(rsyncProgressFraction(p, 0), 0.85);
+ // Clamped: a tree that grew under the copy must not report 140 %.
+ assert.equal(rsyncProgressFraction(p, 10_000_000), 1);
+});
+
+test("rsync progress: the detail line is the one wording", () => {
+ const p = parseRsyncProgress(" 30,000,000 85% 1.03GB/s 0:05:32")!;
+ assert.equal(
+ formatRsyncProgressDetail(p, 60_000_000, formatBytes),
+ "28.6 MB of 57.2 MB · 50 % · 1.03GB/s · ETA 5:32",
+ );
+ // A finished frame prints 0:00:00, and an ETA of zero is noise.
+ const done = parseRsyncProgress(" 60,000,000 100% 1.05GB/s 0:00:00")!;
+ assert.equal(
+ formatRsyncProgressDetail(done, 60_000_000, formatBytes),
+ "57.2 MB of 57.2 MB · 100 % · 1.05GB/s",
+ );
+});
+
+// A TORN FRAME MUST NEVER PARSE AS A FRAME. A chunk boundary falls wherever the
+// pipe decides, and it lands inside a progress line often enough to matter on a
+// long copy. The halves are the hazard, not the loss: ` 30,000,` still
+// matches nothing, but the NEXT chunk's leading ` 30` can parse as a frame
+// reporting thirty bytes — the bar jumps back to 0 % and the decile latch has
+// already spent its line. relocateDir.ts holds the trailing segment back; this
+// pins what each half does on its own so that buffering is provably necessary.
+test("rsync progress: half a frame is not a frame, and two halves are one", () => {
+ const whole = " 30,000,000 85% 1.03GB/s 0:00:03 (xfr#1, to-chk=1/3)";
+ const cut = 12;
+ const head = whole.slice(0, cut);
+ const tail = whole.slice(cut);
+ // The head alone has no percentage, so it cannot parse.
+ assert.equal(parseRsyncProgress(head), null);
+ // The tail alone is the dangerous one: rejoined wrongly it would be read as a
+ // frame about a handful of bytes.
+ const strayTail = parseRsyncProgress(tail);
+ if (strayTail !== null) {
+ assert.notEqual(strayTail.bytes, 30_000_000);
+ }
+ // Rejoined, it is the frame it always was.
+ assert.deepEqual(parseRsyncProgress(head + tail), {
+ bytes: 30_000_000,
+ percent: 85,
+ rate: "1.03GB/s",
+ etaSeconds: 3,
+ });
+});
diff --git a/common/jobs/progressParsers.ts b/common/jobs/progressParsers.ts
@@ -455,3 +455,98 @@ export function createParakeetProgressParser(): {
},
};
}
+
+// ---------------------------------------------------------------------------
+// rsync --info=progress2, for the relocate job
+// ---------------------------------------------------------------------------
+
+// ONE LINE OF `rsync --info=progress2`, parsed.
+//
+// rsync rewrites this line in place with a carriage return, so a chunk read off
+// the child's stdout holds several of them; the caller splits on /[\r\n]/ and
+// feeds each part. The shape (measured, not assumed — rsync 3.x, no
+// --human-readable):
+//
+// 30,000,000 85% 1.03GB/s 0:00:00 (xfr#1, to-chk=1/3)
+//
+// Four fields: bytes transferred so far (grouped with commas), rsync's own
+// percentage, a rate, and an ETA as h:mm:ss. The trailing `(xfr#…)` appears only
+// once a file completes and is deliberately not parsed — `to-chk` counts FILES,
+// and a relocate is priced in bytes.
+//
+// WHY THE PERCENTAGE IS NOT THE FRACTION WE REPORT. rsync's own percentage is
+// against the total it has scanned SO FAR: with incremental recursion (the
+// default) an early line reads "85%" of a tree it has only half enumerated, and
+// the bar then walks backwards. The relocate controller has already measured the
+// whole tree for its space check, so it divides by that instead and the fraction
+// only ever moves forward. This parser returns both and lets the caller choose.
+export type RsyncProgress = {
+ // Bytes rsync says it has transferred so far.
+ bytes: number;
+ // rsync's own percentage, 0..100. See above for why it is not the fraction.
+ percent: number;
+ // Verbatim, e.g. "1.03GB/s". Not re-formatted: it is already the unit an
+ // operator watching a copy reads in.
+ rate: string;
+ // Seconds, from rsync's h:mm:ss ETA.
+ etaSeconds: number;
+};
+
+const RSYNC_PROGRESS_RE =
+ /^\s*([\d,._ ]*\d)\s+(\d{1,3})%\s+(\S+)\s+(\d+):([0-5]?\d):([0-5]?\d)/;
+
+export function parseRsyncProgress(line: string): RsyncProgress | null {
+ const m = line.match(RSYNC_PROGRESS_RE);
+ if (!m) return null;
+ // Grouping separators vary with the locale rsync was built against; strip
+ // everything that is not a digit rather than assuming a comma.
+ const bytes = Number.parseInt(m[1].replace(/\D/g, ""), 10);
+ const percent = Number.parseInt(m[2], 10);
+ if (!Number.isFinite(bytes) || !Number.isFinite(percent)) return null;
+ const etaSeconds =
+ Number.parseInt(m[4], 10) * 3600 +
+ Number.parseInt(m[5], 10) * 60 +
+ Number.parseInt(m[6], 10);
+ return { bytes, percent, rate: m[3], etaSeconds };
+}
+
+// Is this line ONLY a progress redraw? Used by the relocate controller to keep
+// several thousand carriage-return redraws out of a job log that an operator
+// reads afterwards — the parsed decile lines say the same thing in eleven lines.
+export function isRsyncProgressLine(line: string): boolean {
+ return parseRsyncProgress(line) !== null;
+}
+
+// The one wording of a copy's progress, shared by the job task's detail string
+// and the decile line in the log — so the /jobs row and the log cannot word the
+// same instant differently.
+//
+// "12.3 GB of 45.6 GB · 27 % · 110.50MB/s · ETA 5:32"
+//
+// `totalBytes` is the measured source tree, not rsync's running total.
+export function formatRsyncProgressDetail(
+ p: RsyncProgress,
+ totalBytes: number,
+ formatBytes: (n: number) => string,
+): string {
+ const pct =
+ totalBytes > 0
+ ? Math.round(clamp01(p.bytes / totalBytes) * 100)
+ : p.percent;
+ const bits = [
+ `${formatBytes(p.bytes)} of ${totalBytes > 0 ? formatBytes(totalBytes) : "?"}`,
+ `${pct} %`,
+ p.rate,
+ ];
+ if (p.etaSeconds > 0) bits.push(`ETA ${formatClock(p.etaSeconds)}`);
+ return bits.join(" · ");
+}
+
+// 0..1 against the MEASURED tree, with rsync's own percentage as the fallback
+// for a caller that never measured one.
+export function rsyncProgressFraction(
+ p: RsyncProgress,
+ totalBytes: number,
+): number {
+ return totalBytes > 0 ? clamp01(p.bytes / totalBytes) : clamp01(p.percent / 100);
+}
diff --git a/common/jobs/registry.ts b/common/jobs/registry.ts
@@ -49,7 +49,16 @@ export type JobProgress = {
remainingAudioSeconds?: number;
};
-export type JobTaskKind = "download" | "transcribe" | "digest" | "backfill";
+// "relocate" is one CHANNEL'S media move, not one video: the relocate job has
+// exactly one sub-operation and its progress is bytes copied by rsync. It is a
+// task rather than a JobProgress metric because JobProgress counts artifacts
+// re-countable from disk, and a copy in flight is neither.
+export type JobTaskKind =
+ | "download"
+ | "transcribe"
+ | "digest"
+ | "backfill"
+ | "relocate";
// A single in-flight sub-operation within a job (one video download or one
// transcription). Only currently-running tasks are kept on the record — they
diff --git a/common/jobs/snapshotScheduler.ts b/common/jobs/snapshotScheduler.ts
@@ -60,6 +60,9 @@ const NO_REGEN_KINDS = new Set<string>([
// has one — so "move out, then move back" would be blocked by a report
// nobody needed.
"relocate-channel-media",
+ // Moving the saved-video store changes where bytes are, not what any channel
+ // has — every count in every report is identical afterwards.
+ "relocate-saved-videos",
// A RE-POINT DOES NOT EVEN MOVE THE BYTES. It rewrites n symlinks and n
// `dataDir` fields so the media that came back on a different mountpoint is
// reachable again; every count in every report is what it was before. It
diff --git a/common/lib/channelPriority.test.ts b/common/lib/channelPriority.test.ts
@@ -3,6 +3,11 @@ import assert from "node:assert/strict";
import {
CHANNEL_TIERS,
PRIORITY_OPERATIONS,
+ autoPauseForMedia,
+ autoPauseReasonOf,
+ autoPausedSlugs,
+ clearAutoPause,
+ restoreAfterMedia,
PRIO_CATCH_ALL_ID,
STORED_CHANNEL_TIERS,
channelPriorityFromLegacy,
@@ -1121,3 +1126,253 @@ test("a rename onto an existing entry keeps the destination's", () => {
const next = renameChannelInPriority(model, "old", "taken");
assert.deepEqual(next.channels, { taken: { tier: "paused" } });
});
+
+// --- Auto-pause: the machine's own pause, and its undo ----------------------
+
+test("auto-pause records the tier it overwrote and forces paused", () => {
+ const model = sanitizeChannelPriority({
+ focus: { kind: "none" },
+ channels: { a: { tier: "low", rank: 3 } },
+ });
+ const now = new Date("2026-09-20T12:00:00.000Z");
+ const next = autoPauseForMedia(model, "a", now);
+ assert.equal(next.channels.a.tier, "paused");
+ assert.deepEqual(next.channels.a.autoPaused, {
+ reason: "storage",
+ since: "2026-09-20T12:00:00.000Z",
+ previousTier: "low",
+ });
+ // Everything else on the entry survives — the pause is a tier change, not a
+ // reset.
+ assert.equal(next.channels.a.rank, 3);
+ // And a channel with no entry at all gets one recording the default.
+ assert.equal(
+ autoPauseForMedia(model, "fresh", now).channels.fresh.autoPaused
+ ?.previousTier,
+ "normal",
+ );
+});
+
+// A FLAPPING DRIVE MUST NOT OVERWRITE `previousTier` WITH `paused` — that would
+// make the restore restore nothing, which is the failure this no-op prevents.
+test("auto-pause is a no-op on a channel it already auto-paused", () => {
+ const once = autoPauseForMedia(
+ sanitizeChannelPriority({ channels: { a: { tier: "normal" } } }),
+ "a",
+ new Date("2026-09-20T12:00:00.000Z"),
+ );
+ const twice = autoPauseForMedia(once, "a", new Date("2026-09-21T12:00:00Z"));
+ assert.equal(twice, once);
+ assert.equal(twice.channels.a.autoPaused?.previousTier, "normal");
+});
+
+// A CHANNEL THE OPERATOR PAUSED IS NOT THE MACHINE'S TO CLAIM: recording it
+// would hand the next restore permission to turn it back on.
+test("auto-pause is a no-op on a channel the operator already paused", () => {
+ const model = sanitizeChannelPriority({ channels: { a: { tier: "paused" } } });
+ assert.equal(autoPauseForMedia(model, "a"), model);
+});
+
+test("restore puts the tier back and clears the record", () => {
+ const paused = autoPauseForMedia(
+ sanitizeChannelPriority({ channels: { a: { tier: "low", rank: 2 } } }),
+ "a",
+ );
+ const back = restoreAfterMedia(paused, "a");
+ assert.equal(back.channels.a.tier, "low");
+ assert.equal(back.channels.a.autoPaused, undefined);
+ assert.equal(back.channels.a.rank, 2);
+});
+
+test("restore only undoes what auto-pause made", () => {
+ const manual = sanitizeChannelPriority({ channels: { a: { tier: "paused" } } });
+ assert.equal(restoreAfterMedia(manual, "a"), manual);
+ assert.equal(restoreAfterMedia(manual, "nobody"), manual);
+});
+
+test("clearAutoPause is how a manual tier change wins", () => {
+ const paused = autoPauseForMedia(
+ sanitizeChannelPriority({ channels: { a: { tier: "normal" } } }),
+ "a",
+ );
+ const manual = clearAutoPause({ ...paused.channels.a, tier: "low" });
+ assert.equal(manual.autoPaused, undefined);
+ assert.equal(manual.tier, "low");
+ // And restore then has nothing to say about it.
+ const model = { ...paused, channels: { a: manual } };
+ assert.equal(restoreAfterMedia(model, "a"), model);
+});
+
+// AN AUTO-PAUSE ONLY MEANS ANYTHING ON A PAUSED CHANNEL. A record beside a
+// non-paused tier is a leftover, and restoring from it would put back a tier
+// from an earlier era.
+test("the sanitizer drops a record whose tier is not paused, and round-trips one that is", () => {
+ const stray = sanitizeChannelPriority({
+ channels: {
+ a: {
+ tier: "normal",
+ autoPaused: { reason: "storage", since: "x", previousTier: "low" },
+ },
+ },
+ });
+ assert.deepEqual(stray.channels, {});
+ const kept = sanitizeChannelPriority({
+ channels: {
+ a: {
+ tier: "paused",
+ autoPaused: {
+ reason: "storage",
+ since: "2026-09-20T00:00:00.000Z",
+ previousTier: "low",
+ },
+ },
+ },
+ });
+ assert.deepEqual(kept.channels.a.autoPaused, {
+ reason: "storage",
+ since: "2026-09-20T00:00:00.000Z",
+ previousTier: "low",
+ });
+ // Idempotent.
+ assert.deepEqual(sanitizeChannelPriority(kept), kept);
+ // An unknown reason is not a reason.
+ assert.equal(
+ sanitizeChannelPriority({
+ channels: {
+ a: { tier: "paused", autoPaused: { reason: "weather", previousTier: "low" } },
+ },
+ }).channels.a?.autoPaused,
+ undefined,
+ );
+ // `previousTier: "paused"` would restore to paused — a no-op dressed as a
+ // restore. It reads as the default.
+ assert.equal(
+ sanitizeChannelPriority({
+ channels: {
+ a: {
+ tier: "paused",
+ autoPaused: { reason: "storage", since: "", previousTier: "paused" },
+ },
+ },
+ }).channels.a.autoPaused?.previousTier,
+ "normal",
+ );
+});
+
+// AN OLDER SHAPE PARSES, which is the whole lane-migration discipline: a binary
+// that drops the field leaves the channel Paused with nothing lost but the
+// automatic restore.
+test("an entry with no record parses exactly as it did before", () => {
+ const model = sanitizeChannelPriority({
+ channels: { a: { tier: "paused", rank: 1 } },
+ });
+ assert.equal(model.channels.a.autoPaused, undefined);
+ assert.equal(autoPauseReasonOf(model, "a"), null);
+ assert.deepEqual(autoPausedSlugs(model), []);
+});
+
+test("the reason is one sentence, and names the tier it will restore", () => {
+ const model = autoPauseForMedia(
+ sanitizeChannelPriority({ channels: { a: { tier: "low" } } }),
+ "a",
+ new Date("2026-09-20T12:00:00.000Z"),
+ );
+ const reason = autoPauseReasonOf(model, "a") ?? "";
+ assert.match(reason, /drive that is not there since 2026-09-20/);
+ assert.match(reason, /returns to low/);
+ assert.deepEqual(autoPausedSlugs(model), ["a"]);
+});
+
+// AN UNPLUGGED USB CABLE MUST NOT DESTROY THE QUEUE ORDER. The sanitizer drops
+// a rank from a channel that is paused everywhere — right for a pause the
+// operator meant, catastrophic for a temporary one, because the restore would
+// put the channel back unranked at the bottom of its tier.
+test("an auto-paused channel keeps its rank across a sanitize", () => {
+ const paused = sanitizeChannelPriority(
+ autoPauseForMedia(
+ sanitizeChannelPriority({ channels: { a: { tier: "normal", rank: 4 } } }),
+ "a",
+ ),
+ );
+ assert.equal(paused.channels.a.tier, "paused");
+ assert.equal(paused.channels.a.rank, 4);
+ const back = sanitizeChannelPriority(restoreAfterMedia(paused, "a"));
+ assert.equal(back.channels.a.tier, "normal");
+ assert.equal(back.channels.a.rank, 4);
+ // A pause the operator MEANT still drops its rank, unchanged.
+ assert.equal(
+ sanitizeChannelPriority({ channels: { a: { tier: "paused", rank: 4 } } })
+ .channels.a.rank,
+ undefined,
+ );
+});
+
+// THE FULL ROUND TRIP: pause → sanitize → restore → sanitize gives back the
+// entry the operator wrote, in every field.
+//
+// The overrides half is the one that bit. `sanitizeOverrides` normalises away
+// any override equal to the BASE TIER — which is what keeps the document a
+// list of exceptions — and while the machine's pause stands the base on the
+// entry is the forced `paused`. Measured against that, every `{op:"paused"}`
+// fence the operator set stops being an exception and is deleted, so the
+// restore hands back a channel with no fence at all. On this corpus that is
+// legal-mindset losing both of its, and cornbreadman losing its sync fence to
+// one hiccup of a USB cable.
+test("an auto-pause round trip preserves tier, rank AND per-operation overrides", () => {
+ const written = sanitizeChannelPriority({
+ channels: {
+ "legal-mindset": {
+ tier: "normal",
+ rank: 7,
+ overrides: { sync: "paused", download: "paused" },
+ },
+ },
+ });
+ assert.deepEqual(written.channels["legal-mindset"], {
+ tier: "normal",
+ rank: 7,
+ overrides: { sync: "paused", download: "paused" },
+ });
+
+ // The drive goes away. The tier is forced, the fences are NOT touched.
+ const paused = sanitizeChannelPriority(
+ autoPauseForMedia(written, "legal-mindset", new Date("2026-09-20T00:00:00Z")),
+ );
+ assert.equal(paused.channels["legal-mindset"].tier, "paused");
+ assert.equal(paused.channels["legal-mindset"].rank, 7);
+ assert.deepEqual(paused.channels["legal-mindset"].overrides, {
+ sync: "paused",
+ download: "paused",
+ });
+ // Sanitizing twice must not erode it either — the document is written back
+ // to disk on every settings write.
+ assert.deepEqual(sanitizeChannelPriority(paused), paused);
+
+ // And it comes back exactly as it went in.
+ const restored = sanitizeChannelPriority(
+ restoreAfterMedia(paused, "legal-mindset"),
+ );
+ assert.deepEqual(restored.channels["legal-mindset"], {
+ tier: "normal",
+ rank: 7,
+ overrides: { sync: "paused", download: "paused" },
+ });
+});
+
+// The mirror case: an override equal to the RESTORED base is still normalised
+// away while auto-paused, because that is what it will be on restore.
+test("an override equal to the previous tier is still dropped while auto-paused", () => {
+ const paused = sanitizeChannelPriority(
+ autoPauseForMedia(
+ sanitizeChannelPriority({
+ channels: { a: { tier: "low", overrides: { sync: "low" } } },
+ }),
+ "a",
+ ),
+ );
+ assert.equal(paused.channels.a.overrides, undefined);
+ assert.equal(
+ sanitizeChannelPriority(restoreAfterMedia(paused, "a")).channels.a.tier,
+ "low",
+ );
+});
diff --git a/common/lib/channelPriority.ts b/common/lib/channelPriority.ts
@@ -135,6 +135,30 @@ export type ChannelPriorityEntry = {
// `{tier:"paused", overrides:{sync:"normal"}}`, is "sync only": keep the
// playlist and metadata current, dispatch nothing.
overrides?: Partial<Record<PriorityOperation, StoredChannelTier>>;
+ // PAUSED BY THE MACHINE, NOT BY THE OPERATOR, and what to put back.
+ //
+ // Set when the drive a channel's media is on stops being there: the watch
+ // pass records the tier the channel HAD and forces `paused`, so nothing in
+ // any lane dispatches against a `data/` nobody can read. Cleared — and the
+ // tier restored — when the drive comes back.
+ //
+ // WHY IT IS A FIELD AND NOT A DERIVED STATE. The lanes read `tier`; making
+ // them all ask a second question would be four more places to forget. And
+ // the tier the channel is to be RESTORED to is not derivable from anything
+ // once it has been overwritten — that is the whole content of this field.
+ //
+ // OPTIONAL, and an older binary that drops it leaves the channel Paused with
+ // nothing lost but the automatic restore. The operator's own word always
+ // wins: a MANUAL tier change clears it (see clearAutoPause), so a drive
+ // coming back can never un-pause a channel somebody paused on purpose.
+ autoPaused?: {
+ // One reason today. A union so a second one has somewhere to go, and so a
+ // surface can say WHICH machine decided rather than "automatic".
+ reason: "storage";
+ // ISO, for "auto-paused — media unreachable since <date>".
+ since: string;
+ previousTier: StoredChannelTier;
+ };
};
export type ChannelPriority = {
@@ -242,8 +266,30 @@ export function sanitizeChannelPriority(value: unknown): ChannelPriority {
? raw.tier
: DEFAULT_CHANNEL_TIER;
const entry: ChannelPriorityEntry = { tier };
- const overrides = sanitizeOverrides(raw.overrides, tier);
+ // AUTO-PAUSE FIRST, BECAUSE THE OVERRIDES ARE NORMALISED AGAINST THE BASE
+ // TIER — AND WHILE THE MACHINE'S PAUSE STANDS, `tier` IS NOT IT.
+ //
+ // `sanitizeOverrides` drops any override equal to the base, which is what
+ // keeps the document a list of exceptions. Measure that against the FORCED
+ // `paused` and every `{op: "paused"}` fence the operator set is an
+ // "exception" that is no longer an exception, so it is deleted — and the
+ // restore then hands back a channel with the fence gone. Reproduced:
+ // `{tier:"normal", overrides:{sync:"paused"}}` auto-paused and restored
+ // came back as a bare `{tier:"normal"}`. On this corpus that is
+ // legal-mindset losing both its fences, and cornbreadman — which lives on
+ // the platter with `overrides:{sync:"paused"}` — losing its sync fence to
+ // one hiccup of a USB cable.
+ //
+ // `previousTier` is the base the operator actually set, so that is what the
+ // exceptions are exceptions to.
+ const autoPaused = sanitizeAutoPause(raw.autoPaused, tier);
+ if (autoPaused) entry.autoPaused = autoPaused;
+ const overrides = sanitizeOverrides(
+ raw.overrides,
+ autoPaused ? autoPaused.previousTier : tier,
+ );
if (overrides) entry.overrides = overrides;
+
if (typeof raw.rank === "number" && Number.isFinite(raw.rank)) {
// RANK IS ONLY MEANINGFUL WHERE SOMETHING IS ORDERED. A channel that is
// paused for EVERY operation is in no group anywhere, so its rank is dead
@@ -253,14 +299,22 @@ export function sanitizeChannelPriority(value: unknown): ChannelPriority {
const orderedSomewhere = PRIORITY_OPERATIONS.some(
(op) => (overrides?.[op] ?? tier) !== "paused",
);
- if (orderedSomewhere) entry.rank = Math.floor(raw.rank);
+ // AN AUTO-PAUSED CHANNEL KEEPS ITS RANK. The rule above is about a pause
+ // the operator MEANT — a channel that is off everywhere is in no group
+ // and its rank is dead weight in the file. An auto-pause is temporary by
+ // construction (the record exists precisely to undo it), so dropping the
+ // rank here would mean an unplugged USB cable silently destroyed the
+ // operator's queue order, and the restore would put the channel back
+ // unranked at the bottom of its tier.
+ if (orderedSomewhere || autoPaused) entry.rank = Math.floor(raw.rank);
}
// An entry that says nothing the default does not say is DROPPED, so the
// document stays a list of exceptions and re-sanitizing is identity.
if (
entry.tier === DEFAULT_CHANNEL_TIER &&
entry.rank === undefined &&
- entry.overrides === undefined
+ entry.overrides === undefined &&
+ entry.autoPaused === undefined
) {
continue;
}
@@ -269,6 +323,121 @@ export function sanitizeChannelPriority(value: unknown): ChannelPriority {
return { focus: sanitizeFocus(r.focus), channels };
}
+// AN AUTO-PAUSE ONLY MEANS ANYTHING ON A PAUSED CHANNEL. A record whose tier
+// is not `paused` is a leftover — the operator changed the tier through some
+// path that did not clear it, or the file was hand-edited — and keeping it
+// would make the next restore put back a tier from an earlier era. Dropped.
+function sanitizeAutoPause(
+ value: unknown,
+ tier: StoredChannelTier,
+): ChannelPriorityEntry["autoPaused"] | null {
+ if (tier !== "paused") return null;
+ if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+ const r = value as Record<string, unknown>;
+ if (r.reason !== "storage") return null;
+ const previousTier: StoredChannelTier = isStoredChannelTier(r.previousTier)
+ ? r.previousTier
+ : DEFAULT_CHANNEL_TIER;
+ // A `previousTier` of `paused` would restore to paused, which is a no-op
+ // dressed as a restore. It reads as the default instead.
+ return {
+ reason: "storage",
+ since: typeof r.since === "string" ? r.since : "",
+ previousTier: previousTier === "paused" ? DEFAULT_CHANNEL_TIER : previousTier,
+ };
+}
+
+// --- Auto-pause: the machine's own pause, and its undo ----------------------
+
+// PAUSE A CHANNEL BECAUSE ITS DRIVE IS NOT THERE, recording what to put back.
+//
+// A NO-OP IN TWO CASES, and both matter. Already auto-paused: a flapping drive
+// must not overwrite `previousTier` with the `paused` it wrote last time, which
+// would make the restore restore nothing. Already paused by the OPERATOR: the
+// channel is off because somebody said so, and claiming the machine did it
+// would hand the next restore permission to turn it back on.
+export function autoPauseForMedia(
+ model: ChannelPriority,
+ slug: string,
+ now: Date = new Date(),
+): ChannelPriority {
+ const key = slug.trim();
+ if (!key) return model;
+ const entry = model.channels[key];
+ if (entry?.autoPaused) return model;
+ const previousTier = entry?.tier ?? DEFAULT_CHANNEL_TIER;
+ if (previousTier === "paused") return model;
+ return {
+ ...model,
+ channels: {
+ ...model.channels,
+ [key]: {
+ ...(entry ?? { tier: DEFAULT_CHANNEL_TIER }),
+ tier: "paused",
+ autoPaused: {
+ reason: "storage",
+ since: now.toISOString(),
+ previousTier,
+ },
+ },
+ },
+ };
+}
+
+// THE UNDO, and only of what this made. No record → nothing to restore, and in
+// particular a channel the operator paused by hand stays paused.
+export function restoreAfterMedia(
+ model: ChannelPriority,
+ slug: string,
+): ChannelPriority {
+ const key = slug.trim();
+ const entry = model.channels[key];
+ if (!entry?.autoPaused) return model;
+ const { autoPaused, ...rest } = entry;
+ return {
+ ...model,
+ channels: {
+ ...model.channels,
+ [key]: { ...rest, tier: autoPaused.previousTier },
+ },
+ };
+}
+
+// THE OPERATOR'S WORD WINS. Called by the one priority writer whenever a tier
+// is set by hand: the record goes, so a drive coming back later cannot undo a
+// decision a person made in the meantime.
+export function clearAutoPause(
+ entry: ChannelPriorityEntry,
+): ChannelPriorityEntry {
+ if (!entry.autoPaused) return entry;
+ const { autoPaused: _dropped, ...rest } = entry;
+ return rest;
+}
+
+// The sentence a surface says beside a Paused badge, or null. ONE wording, so
+// the rack, the channel page and /review cannot word it three ways.
+export function autoPauseReasonOf(
+ model: ChannelPriority,
+ slug: string,
+): string | null {
+ const auto = model.channels[slug]?.autoPaused;
+ if (!auto) return null;
+ const since = auto.since ? ` since ${auto.since.slice(0, 10)}` : "";
+ return (
+ `Auto-paused — its media is on a drive that is not there${since}. ` +
+ `It returns to ${auto.previousTier} on its own when the drive is back.`
+ );
+}
+
+// Every channel this document has auto-paused, sorted. What the watch pass asks
+// so it can restore the ones whose drives came back.
+export function autoPausedSlugs(model: ChannelPriority): string[] {
+ return Object.entries(model.channels)
+ .filter(([, e]) => e.autoPaused)
+ .map(([slug]) => slug)
+ .sort();
+}
+
// --- Reading the document ---------------------------------------------------
// The channel's BASE tier — what `/channels` shows in the tier column, and the
diff --git a/common/lib/savedVideo-server.ts b/common/lib/savedVideo-server.ts
@@ -16,6 +16,8 @@ import {
type SavedVideoOrigin,
type SavedVideoPointer,
} from "./savedVideo";
+import { assertSavedVideosStoreWritable } from "./savedVideoStore";
+import type { Paths } from "./paths";
// Filesystem side of the saved-video store. See savedVideo.ts for the layout.
@@ -110,7 +112,18 @@ export async function persistSourceVideo(opts: {
// SavedVideoOrigin — it records the requester WITHOUT changing keepReason,
// which is what keeps the container out of the retention prune.
origin?: SavedVideoOrigin;
+ // WHEN THE STORE IS BEING MOVED, THIS DOES NOT RUN. Passed by the download
+ // path, which has them; a caller that omits them opts out, which is right for
+ // a store that is not the corpus one. See lib/savedVideoStore.ts for the
+ // three ways a write landing mid-move loses a container silently — the worst
+ // of them is `moveFileCrossDevice`'s unconditional `mkdir -p` recreating
+ // `saved-videos` as a real directory in the instant between the mover's
+ // rename and its symlink.
+ paths?: Pick<Paths, "transcriptsDir" | "savedVideosDir">;
}): Promise<SavedVideoPointer> {
+ if (opts.paths) {
+ await assertSavedVideosStoreWritable(opts.paths, opts.storeDir);
+ }
const src = path.join(opts.videoDir, opts.sourceFilename);
const dest = path.join(opts.storeDir, opts.sourceFilename);
const st = await stat(src);
@@ -149,9 +162,16 @@ async function pruneEmptyStoreDirs(storeDir: string): Promise<void> {
// Reverse persistSourceVideo: move the stored container back into the data dir
// and remove the pointer. Returns false when there was no pointer to reverse.
-export async function unpersistSavedVideo(videoDir: string): Promise<boolean> {
+export async function unpersistSavedVideo(
+ videoDir: string,
+ // Same opt-in guard as persistSourceVideo, and for the same window: this
+ // READS out of the store and then prunes its empty dirs, both of which race a
+ // move in flight.
+ paths?: Pick<Paths, "transcriptsDir" | "savedVideosDir">,
+): Promise<boolean> {
const pointer = await loadSavedVideo(videoDir);
if (!pointer) return false;
+ if (paths) await assertSavedVideosStoreWritable(paths, pointer.dir);
const src = savedVideoPath(pointer);
const dest = path.join(videoDir, pointer.file);
try {
diff --git a/common/lib/savedVideoStore.test.ts b/common/lib/savedVideoStore.test.ts
@@ -0,0 +1,168 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "./paths";
+import {
+ assertSavedVideosStoreWritable,
+ isInCorpusSavedVideoStore,
+ readSavedVideosMarker,
+ savedVideosMarkerPath,
+ SavedVideosStoreInTransitionError,
+} from "./savedVideoStore";
+import { persistSourceVideo } from "./savedVideo-server";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test lib/savedVideoStore.test.ts
+//
+// THE GUARD AT THE MOMENT OF THE WRITE. The relocation queue key serialises
+// relocations against each other and says nothing about a download; this is
+// what actually stops a container landing in a store that is being moved — and
+// it has to live in lib/, because `savedVideo-server.ts` is what does the
+// landing and lib may not import controller.
+
+async function withTmp(
+ fn: (paths: Paths, videoDir: string) => Promise<void>,
+): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-store-guard-"));
+ const transcriptsDir = path.join(dir, "corpus");
+ const paths = {
+ transcriptsDir,
+ channelsDir: path.join(transcriptsDir, "channels"),
+ savedVideosDir: path.join(transcriptsDir, "saved-videos"),
+ } as Paths;
+ const videoDir = path.join(paths.channelsDir, "chan", "data", "vid1");
+ await mkdir(videoDir, { recursive: true });
+ await mkdir(paths.savedVideosDir, { recursive: true });
+ try {
+ await fn(paths, videoDir);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+async function writeMarker(paths: Paths, phase = "copy"): Promise<void> {
+ await writeFile(
+ savedVideosMarkerPath(paths),
+ JSON.stringify({
+ target: "/mnt/platter/saved-videos",
+ direction: "out",
+ startedAt: new Date().toISOString(),
+ phase,
+ }),
+ );
+}
+
+test("no marker, no refusal", async () => {
+ await withTmp(async (paths) => {
+ assert.equal(await readSavedVideosMarker(paths), null);
+ await assertSavedVideosStoreWritable(paths, paths.savedVideosDir);
+ });
+});
+
+test("a marker refuses a write into the corpus store, by name", async () => {
+ await withTmp(async (paths) => {
+ await writeMarker(paths, "swap");
+ await assert.rejects(
+ assertSavedVideosStoreWritable(
+ paths,
+ path.join(paths.savedVideosDir, "chan", "vid1"),
+ ),
+ (err: Error) => {
+ assert.ok(err instanceof SavedVideosStoreInTransitionError);
+ assert.match(err.message, /being moved/);
+ assert.match(err.message, /phase "swap"/);
+ return true;
+ },
+ );
+ });
+});
+
+// A CHANNEL MAY POINT ITS OWN STORE SOMEWHERE ELSE ENTIRELY
+// (`ChannelConfig.savedVideosDir`), and that store is not the one being moved.
+// Refusing a write to it would decline a persist for a move that has nothing to
+// do with it.
+test("a store outside the corpus is not this move's business", async () => {
+ await withTmp(async (paths) => {
+ await writeMarker(paths);
+ assert.equal(
+ isInCorpusSavedVideoStore(paths, "/somewhere/else/chan/vid1"),
+ false,
+ );
+ await assertSavedVideosStoreWritable(paths, "/somewhere/else/chan/vid1");
+ });
+});
+
+// THE WHOLE POINT, end to end: a persist that fires while the store is being
+// moved does not move the container. The three ways it ends badly are in
+// savedVideoStore.ts's header; the worst is the container landing in the
+// directory the swap is about to park, which `reclaimParked` then rm -rf's
+// while `saved-video.json` still points at it.
+test("persistSourceVideo refuses while the store is in transition, and moves nothing", async () => {
+ await withTmp(async (paths, videoDir) => {
+ const source = "source-media.mp4";
+ await writeFile(path.join(videoDir, source), "the only copy");
+ const storeDir = path.join(paths.savedVideosDir, "chan", "vid1");
+ await writeMarker(paths);
+
+ await assert.rejects(
+ persistSourceVideo({
+ videoDir,
+ sourceFilename: source,
+ storeDir,
+ paths,
+ }),
+ /being moved/,
+ );
+ // The container is still where it was, with no pointer beside it and
+ // nothing created in the store.
+ assert.equal(
+ await readFile(path.join(videoDir, source), "utf8"),
+ "the only copy",
+ );
+ assert.equal(await exists(path.join(videoDir, "saved-video.json")), false);
+ assert.equal(await exists(storeDir), false);
+
+ // Clear the marker and the same call succeeds — the guard is the marker
+ // and nothing else.
+ await rm(savedVideosMarkerPath(paths), { force: true });
+ const pointer = await persistSourceVideo({
+ videoDir,
+ sourceFilename: source,
+ storeDir,
+ paths,
+ });
+ assert.equal(pointer.dir, storeDir);
+ assert.equal(
+ await readFile(path.join(storeDir, source), "utf8"),
+ "the only copy",
+ );
+ });
+});
+
+// A caller that passes no paths opts out. That is right for a store that is not
+// the corpus one, and it is what keeps this change from touching every existing
+// call site.
+test("persistSourceVideo without paths does not consult the marker", async () => {
+ await withTmp(async (paths, videoDir) => {
+ await writeMarker(paths);
+ await writeFile(path.join(videoDir, "source-media.mp4"), "x");
+ const storeDir = path.join(paths.savedVideosDir, "chan", "vid1");
+ const pointer = await persistSourceVideo({
+ videoDir,
+ sourceFilename: "source-media.mp4",
+ storeDir,
+ });
+ assert.equal(pointer.file, "source-media.mp4");
+ });
+});
+
+async function exists(p: string): Promise<boolean> {
+ try {
+ await readFile(p);
+ return true;
+ } catch (err) {
+ return (err as NodeJS.ErrnoException).code === "EISDIR";
+ }
+}
diff --git a/common/lib/savedVideoStore.ts b/common/lib/savedVideoStore.ts
@@ -0,0 +1,115 @@
+import path from "node:path";
+import { readFile } from "node:fs/promises";
+import type { Paths } from "./paths";
+import type { RelocationMarker } from "./channelMedia";
+
+// IS THE SAVED-VIDEO STORE SAFE TO WRITE INTO RIGHT NOW?
+//
+// The store moves to another drive the same way a channel's `data/` does, and
+// it inherits the same hazard: for the length of a multi-hour copy the bytes at
+// `transcripts/saved-videos` are being read, verified and then swapped, and
+// anything that writes into them in the meantime is one of three failures, all
+// of them silent until much later.
+//
+// 1. A container landing mid-copy makes `verifyCopy` refuse AT THE END of the
+// whole transfer — the omnimirror incident, exactly, one directory down.
+// 2. A container landing between the verify and the rename lands in the dir
+// the swap is about to PARK, and `reclaimParked` then `rm -rf`s it while
+// the video's `saved-video.json` still points at it. That is the only copy
+// of a source container, gone, with a pointer that outlives it.
+// 3. `moveFileCrossDevice` does an unconditional `mkdir -p` of the
+// destination's parent, so a persist firing between the `rename` and the
+// `symlink` RECREATES `saved-videos` as a real directory — the symlink is
+// then never made, and the move would have recorded the store as being on
+// a location it cannot be reached at.
+//
+// THIS MODULE IS lib/, AND IT HAS TO BE. `common/lib/savedVideo-server.ts` is
+// what persists a container, it is lib, and lib may not import controller
+// (architecture.test.ts) — so the marker's name and its reader live here, next
+// to `channelMedia.ts`, which is the same module for the same reason about a
+// channel. The controller that WRITES the marker imports these; it does not own
+// them.
+//
+// The marker's shape is deliberately a channel relocation marker
+// (`{target, direction, startedAt, phase}`) — one mover writes both, and a
+// second shape would be a second thing to keep in step.
+
+export const SAVED_VIDEOS_DIRNAME = "saved-videos";
+export const SAVED_VIDEOS_MARKER_FILENAME = ".relocating-saved-videos.json";
+
+export function savedVideosMarkerPath(
+ paths: Pick<Paths, "transcriptsDir">,
+): string {
+ return path.join(paths.transcriptsDir, SAVED_VIDEOS_MARKER_FILENAME);
+}
+
+// The store's home on a location: `<root>/saved-videos`. Flat, beside the
+// channels' `<slug>/data` dirs, and not configurable for the same reason
+// `relocatedDataDir` is not — a mover recognises a target by its shape.
+export function relocatedSavedVideosDir(root: string): string {
+ return path.join(root.trim(), SAVED_VIDEOS_DIRNAME);
+}
+
+export async function readSavedVideosMarker(
+ paths: Pick<Paths, "transcriptsDir">,
+): Promise<RelocationMarker | null> {
+ try {
+ const raw = JSON.parse(
+ await readFile(savedVideosMarkerPath(paths), "utf8"),
+ ) as Partial<RelocationMarker>;
+ if (typeof raw.target !== "string" || raw.target.trim() === "") return null;
+ return {
+ target: raw.target,
+ direction: raw.direction === "back" ? "back" : "out",
+ startedAt: typeof raw.startedAt === "string" ? raw.startedAt : "",
+ phase:
+ raw.phase === "swap" || raw.phase === "reclaim" ? raw.phase : "copy",
+ };
+ } catch {
+ return null;
+ }
+}
+
+// Thrown by assertSavedVideosStoreWritable. A distinct class so a caller can
+// tell "the store is being moved" from any other I/O failure and decline the
+// persist rather than failing the whole download.
+export class SavedVideosStoreInTransitionError extends Error {
+ readonly marker: RelocationMarker;
+ constructor(marker: RelocationMarker) {
+ super(
+ `The saved-video store is being moved (${marker.direction} to ` +
+ `${marker.target}, phase "${marker.phase}") — nothing may be written ` +
+ `into it until that finishes or its marker is cleared on /storage.`,
+ );
+ this.name = "SavedVideosStoreInTransitionError";
+ this.marker = marker;
+ }
+}
+
+// Is `dir` the corpus store, or inside it? A channel may point its own store
+// somewhere else entirely (`ChannelConfig.savedVideosDir`), and that store is
+// not the one being moved — guarding a write to it would refuse a persist for a
+// move that has nothing to do with it.
+export function isInCorpusSavedVideoStore(
+ paths: Pick<Paths, "savedVideosDir">,
+ dir: string,
+): boolean {
+ const rel = path.relative(paths.savedVideosDir, dir);
+ return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
+}
+
+// THE GUARD, at the moment of the write. One `readFile` of a small JSON file
+// per persisted container — the same cost `inspectChannelMedia` pays per lane
+// tick, and a persist happens once per kept video, not once per tick.
+//
+// It is checked AT THE WRITE and not only at the start of the download because
+// the window is the whole copy: a download that began before the move started
+// is exactly the one that would land a container in the parked directory.
+export async function assertSavedVideosStoreWritable(
+ paths: Pick<Paths, "transcriptsDir" | "savedVideosDir">,
+ storeDir: string,
+): Promise<void> {
+ if (!isInCorpusSavedVideoStore(paths, storeDir)) return;
+ const marker = await readSavedVideosMarker(paths);
+ if (marker) throw new SavedVideosStoreInTransitionError(marker);
+}
diff --git a/common/lib/settings.ts b/common/lib/settings.ts
@@ -959,7 +959,26 @@ export function sanitizeStorage(value: unknown): StorageSettings {
const defaultLocationId = locations.some((l) => l.id === wanted)
? wanted
: (locations[0]?.id ?? "");
- return { locations, defaultLocationId };
+ // THE SAVED-VIDEO STORE'S LOCATION IS NOT FALLEN BACK, and the asymmetry
+ // with `defaultLocationId` above is deliberate. That one is a PREFERENCE, so
+ // picking another location when the named one is gone is helpful. This one is
+ // a RECORD OF WHERE BYTES ARE: pointing it at a different location because
+ // the recorded one was deleted would claim the store had moved when nothing
+ // had. A dangling id sanitizes to "" — "in place" — which is what the disk
+ // says as soon as anybody looks, and the symlink (if any) keeps working
+ // regardless, because the store is reached through it and not through this.
+ const savedWanted =
+ typeof r.savedVideosLocationId === "string"
+ ? r.savedVideosLocationId.trim()
+ : "";
+ const savedVideosLocationId = locations.some((l) => l.id === savedWanted)
+ ? savedWanted
+ : "";
+ return {
+ locations,
+ defaultLocationId,
+ ...(savedVideosLocationId ? { savedVideosLocationId } : {}),
+ };
}
// 4 hours. Measured: videos over this are 8.2% of the corpus by count but hold
diff --git a/common/lib/storageLocations.ts b/common/lib/storageLocations.ts
@@ -57,6 +57,16 @@ export type StorageSettings = {
locations: StorageLocation[];
// The location prefilled as the destination of a move. "" = no default.
defaultLocationId: string;
+ // WHERE THE SAVED-VIDEO STORE IS, by location id. "" = in place, under the
+ // corpus at `paths.savedVideosDir`.
+ //
+ // A RECORD OF WHAT IS ON DISK, never an intention — the same contract as a
+ // channel's `config.dataDir`. It is written by the move, on success, after
+ // the copy has verified and the symlink is in place; nothing else writes it,
+ // and a reader that disagrees with the disk trusts the disk. Optional so an
+ // older settings.json parses (and an older binary that drops it leaves a
+ // store that still works, because the symlink is what every reader follows).
+ savedVideosLocationId?: string;
};
// Strip trailing slashes so "/mnt/platter/" and "/mnt/platter" are one root.
diff --git a/common/views/freeUpSelection.test.ts b/common/views/freeUpSelection.test.ts
@@ -0,0 +1,79 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { selectToFreeBytes, type FreeUpCandidate } from "./freeUpSelection";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test views/freeUpSelection.test.ts
+
+const GB = 1024 ** 3;
+
+function c(
+ slug: string,
+ gb: number | null,
+ inPlace = true,
+): FreeUpCandidate {
+ return { slug, bytes: gb === null ? null : gb * GB, inPlace };
+}
+
+test("largest first, stopping at the first pick that clears the target", () => {
+ const r = selectToFreeBytes(
+ [c("small", 10), c("huge", 300), c("mid", 120), c("tiny", 1)],
+ 200 * GB,
+ );
+ assert.deepEqual(r.slugs, ["huge"]);
+ assert.equal(r.bytes, 300 * GB);
+ assert.equal(r.shortfall, 0);
+});
+
+test("it keeps taking until the target is met", () => {
+ const r = selectToFreeBytes([c("a", 80), c("b", 70), c("c", 60)], 200 * GB);
+ assert.deepEqual(r.slugs, ["a", "b", "c"]);
+ assert.equal(r.bytes, 210 * GB);
+ assert.equal(r.shortfall, 0);
+});
+
+// MOVING A CHANNEL THAT IS ALREADY ON THE PLATTER FREES NOTHING on the disk
+// being emptied — counting it would report progress the operator would not get.
+test("channels already on a location are never picked", () => {
+ const r = selectToFreeBytes(
+ [c("moved", 500, false), c("here", 50)],
+ 200 * GB,
+ );
+ assert.deepEqual(r.slugs, ["here"]);
+ assert.equal(r.shortfall, 150 * GB);
+ assert.match(r.note, /150\.0 GB short/);
+});
+
+// A ZERO NOBODY MEASURED IS NOT A ZERO: ranking an unmeasured channel as empty
+// would leave the biggest thing on the disk at the bottom of the list.
+test("an unmeasured channel is excluded and counted, never ranked as empty", () => {
+ const r = selectToFreeBytes([c("unknown", null), c("known", 30)], 200 * GB);
+ assert.deepEqual(r.slugs, ["known"]);
+ assert.equal(r.unmeasured, 1);
+ assert.match(r.note, /1 channel\(s\) have no size/);
+ // An unmeasured channel that is not even in place is not our problem.
+ assert.equal(
+ selectToFreeBytes([c("elsewhere", null, false)], 10 * GB).unmeasured,
+ 0,
+ );
+});
+
+test("a zero or negative target selects nothing", () => {
+ for (const target of [0, -1]) {
+ assert.deepEqual(selectToFreeBytes([c("a", 80)], target).slugs, []);
+ }
+});
+
+test("ties break by slug so the same target always proposes the same list", () => {
+ const rows = [c("beta", 50), c("alpha", 50), c("gamma", 50)];
+ const first = selectToFreeBytes(rows, 100 * GB).slugs;
+ const again = selectToFreeBytes([...rows].reverse(), 100 * GB).slugs;
+ assert.deepEqual(first, ["alpha", "beta"]);
+ assert.deepEqual(again, first);
+});
+
+test("nothing measurable says so rather than proposing an empty selection", () => {
+ const r = selectToFreeBytes([c("x", null), c("y", null)], 100 * GB);
+ assert.deepEqual(r.slugs, []);
+ assert.match(r.note, /Nothing in place has a measured size/);
+});
diff --git a/common/views/freeUpSelection.ts b/common/views/freeUpSelection.ts
@@ -0,0 +1,104 @@
+// "FREE UP N GB" — which channels to move, given a number of bytes to reclaim.
+//
+// The operator's actual question on a disk that is 96 % full is not "which
+// channel is biggest" but "what is the shortest list I can move to get 200 GB
+// back". Answering it by hand means sorting the table by size, adding figures
+// in your head and ticking rows until the sum clears the target — which is
+// exactly the kind of arithmetic a computer should not be making a person do at
+// 3am on a nearly-full disk.
+//
+// PURE, like everything in views/: it takes rows and a target and returns
+// slugs. The page holds the rows, the deck ticks the boxes, and the existing
+// bulk Move action is what actually moves anything — this selects, it does not
+// act.
+//
+// THREE RULES, each of which is a refusal to guess:
+//
+// 1. ONLY CHANNELS THAT ARE IN PLACE. Moving a channel that is already on the
+// platter frees nothing on the disk the operator is trying to empty; it
+// would be counted as progress and deliver none.
+// 2. A CHANNEL WITH NO MEASUREMENT IS NOT A CHANNEL WITH ZERO BYTES. A
+// snapshot written before `totalMediaBytes` existed carries no figure, and
+// ranking it as empty would leave the largest channel on the disk at the
+// bottom of the list. They are excluded and COUNTED, so the deck can say
+// "3 channels have no size yet — refresh their reports" instead of
+// silently proposing a worse answer.
+// 3. LARGEST FIRST, AND THE LAST ONE OVERSHOOTS. The point is the shortest
+// list, so the greedy order is the right one; the final pick will usually
+// carry the total past the target, and that is the honest outcome rather
+// than a subset-sum search for an exact fit nobody asked for.
+
+export type FreeUpCandidate = {
+ slug: string;
+ // `snapshot.totalMediaBytes`, or null when the report predates the field.
+ bytes: number | null;
+ // Its media is on the corpus volume — the disk being freed.
+ inPlace: boolean;
+};
+
+export type FreeUpSelection = {
+ // Largest first, which is also the order they were picked in.
+ slugs: string[];
+ // What the selection would move.
+ bytes: number;
+ // Bytes still wanted after every eligible channel was taken. 0 when the
+ // target was met.
+ shortfall: number;
+ // Channels skipped for want of a measurement (rule 2).
+ unmeasured: number;
+ // A sentence for the deck, built here so the two surfaces that could show it
+ // cannot word it differently.
+ note: string;
+};
+
+export function selectToFreeBytes(
+ candidates: ReadonlyArray<FreeUpCandidate>,
+ targetBytes: number,
+): FreeUpSelection {
+ const unmeasured = candidates.filter(
+ (c) => c.inPlace && c.bytes === null,
+ ).length;
+ const eligible = candidates
+ .filter(
+ (c): c is FreeUpCandidate & { bytes: number } =>
+ c.inPlace && typeof c.bytes === "number" && c.bytes > 0,
+ )
+ .sort((a, b) => b.bytes - a.bytes || a.slug.localeCompare(b.slug));
+
+ const slugs: string[] = [];
+ let bytes = 0;
+ if (targetBytes > 0) {
+ for (const c of eligible) {
+ if (bytes >= targetBytes) break;
+ slugs.push(c.slug);
+ bytes += c.bytes;
+ }
+ }
+ const shortfall = Math.max(0, targetBytes - bytes);
+ return { slugs, bytes, shortfall, unmeasured, note: noteFor(slugs.length, shortfall, unmeasured) };
+}
+
+function gb(n: number): string {
+ return `${(n / 1024 ** 3).toFixed(1)} GB`;
+}
+
+function noteFor(
+ picked: number,
+ shortfall: number,
+ unmeasured: number,
+): string {
+ const tail =
+ unmeasured > 0
+ ? ` ${unmeasured} channel(s) have no size in their report yet and were not considered — refresh them for a better answer.`
+ : "";
+ if (picked === 0 && shortfall > 0) {
+ return `Nothing in place has a measured size to move.${tail}`;
+ }
+ if (shortfall > 0) {
+ return (
+ `Selected all ${picked} measured in-place channel(s) — still ` +
+ `${gb(shortfall)} short of the target.${tail}`
+ );
+ }
+ return `Selected ${picked} channel(s), largest first.${tail}`;
+}
diff --git a/common/views/storage.test.ts b/common/views/storage.test.ts
@@ -23,6 +23,8 @@ function rollup(partial: Partial<LocationRollup>): LocationRollup {
locationId: partial.locationId ?? "x",
slugs: partial.slugs ?? [],
total: partial.total ?? 0,
+ bytes: partial.bytes ?? 0,
+ unknownBytes: partial.unknownBytes ?? 0,
ok: partial.ok ?? 0,
unreachable: partial.unreachable ?? 0,
moving: partial.moving ?? 0,
@@ -256,3 +258,104 @@ test("a location that has never been probed reads as missing, not as a gap", ()
assert.equal(rows[0].lastProbeAgeMs, 0);
assert.equal(rows[0].channelsText, "0 ok / 0 unreachable / 0 moving");
});
+
+const GB = 1024 ** 3;
+
+test("bytes travel with the count of channels that could not be measured", () => {
+ const { rows, bytesOnLocation, bytesInPlace } = buildStorageRows({
+ locations: [loc("cold", "/mnt/cold")],
+ internal: { root: "/corpus/channels", freeBytes: 67 * GB },
+ defaultLocationId: "cold",
+ probes: {},
+ rollups: {
+ cold: rollup({ locationId: "cold", total: 3, ok: 3, bytes: 1500 * GB }),
+ internal: rollup({
+ locationId: "internal",
+ total: 5,
+ ok: 5,
+ bytes: 523 * GB,
+ unknownBytes: 2,
+ }),
+ },
+ registry: NO_JOBS,
+ now: NOW,
+ });
+ // The internal row is FIRST — it is the row with the problem.
+ assert.equal(rows[0].id, "internal");
+ assert.equal(rows[0].kind, "internal");
+ assert.equal(rows[0].root, "/corpus/channels");
+ assert.equal(rows[0].freeBytes, 67 * GB);
+ assert.equal(rows[0].bytes, 523 * GB);
+ assert.equal(rows[0].bytesText, "523.00 GB + 2 unmeasured");
+ assert.equal(rows[1].bytesText, "1.46 TB");
+ assert.equal(bytesInPlace, 523 * GB);
+ assert.deepEqual(bytesOnLocation, {
+ internal: 523 * GB,
+ cold: 1500 * GB,
+ });
+});
+
+// A ZERO THAT NOBODY MEASURED IS NOT A ZERO. A fresh corpus whose channels have
+// never had a report would otherwise say a 2 TB drive holds 0 B.
+test("nothing measured says so instead of claiming an empty drive", () => {
+ const { rows } = buildStorageRows({
+ locations: [loc("cold", "/mnt/cold")],
+ defaultLocationId: "",
+ probes: {},
+ rollups: {
+ cold: rollup({ locationId: "cold", total: 4, ok: 4, unknownBytes: 4 }),
+ },
+ registry: NO_JOBS,
+ now: NOW,
+ });
+ assert.equal(rows[0].bytesText, "size unknown until Refresh report");
+});
+
+test("the internal row offers nothing, and says why for each", () => {
+ const { rows } = buildStorageRows({
+ locations: [],
+ internal: { root: "/corpus/channels" },
+ defaultLocationId: "",
+ probes: {},
+ rollups: {},
+ registry: NO_JOBS,
+ now: NOW,
+ });
+ assert.equal(rows.length, 1);
+ assert.equal(rows[0].status, "available");
+ assert.equal(rows[0].freeBytes, undefined);
+ for (const a of rows[0].actions) {
+ assert.equal(a.offered, false, a.kind);
+ assert.match(a.withheld ?? "", /not a configured location/);
+ }
+});
+
+test("every row links to its own channel list, biggest first", () => {
+ const { rows } = buildStorageRows({
+ locations: [loc("cold", "/mnt/cold")],
+ internal: { root: "/corpus/channels" },
+ defaultLocationId: "",
+ probes: {},
+ rollups: {},
+ registry: NO_JOBS,
+ now: NOW,
+ });
+ assert.equal(rows[0].channelsHref, "/channels?location=internal&sort=size");
+ assert.equal(rows[1].channelsHref, "/channels?location=cold&sort=size");
+});
+
+test("no internal input means no internal row (a caller that wants only the locations)", () => {
+ const { rows, bytesInPlace } = buildStorageRows({
+ locations: [loc("cold", "/mnt/cold")],
+ defaultLocationId: "",
+ probes: {},
+ rollups: {},
+ registry: NO_JOBS,
+ now: NOW,
+ });
+ assert.deepEqual(
+ rows.map((r) => r.id),
+ ["cold"],
+ );
+ assert.equal(bytesInPlace, 0);
+});
diff --git a/common/views/storage.ts b/common/views/storage.ts
@@ -7,6 +7,7 @@ import type {
LocationRollup,
MemoizedProbe,
} from "../controller/storageLocations";
+import type { SavedVideosStoreStatus } from "../controller/relocateSavedVideos";
import type { RegistryReader } from "./inputs";
// THE /storage PAYLOAD — one row per storage location, and for each row the
@@ -55,6 +56,12 @@ export type StorageRow = {
id: string;
label: string;
root: string;
+ // "internal" is the SYNTHETIC row — the corpus volume, where an unrelocated
+ // channel's media is. It is not in settings (see INTERNAL_LOCATION_ID) and it
+ // is the one row with nothing to refresh, re-point, mount, edit or delete:
+ // every action on it is present and withheld, because a greyed button with no
+ // reason is what this page exists not to be.
+ kind: "location" | "internal";
isDefault: boolean;
autoRepoint: boolean;
status: StorageLocationStatus;
@@ -69,6 +76,18 @@ export type StorageRow = {
// `n ok / n unreachable / n moving` — the cell's text, built here so the page
// and any future poll cannot word it differently.
channelsText: string;
+ // `/channels?location=<id>` — the list this row summarises, largest first.
+ // Built here rather than in the page so the param name has ONE spelling
+ // across the two surfaces that use it.
+ channelsHref: string;
+ // Media bytes on this location, summed from each channel's last report, and
+ // how many channels on it could not contribute a figure.
+ bytes: number;
+ unknownBytes: number;
+ // "1.42 TB (2 channels unmeasured)" / "size unknown until Refresh report".
+ // ONE wording, and never a bare "0 B" for a location whose channels have
+ // simply never had a report — that reads as an empty drive.
+ bytesText: string;
// Absent when the location is not available, and ALSO when it is available
// but unmeasurable (getFreeBytes fails open to Infinity, which the probe
// drops rather than carry). Render "—" for both.
@@ -84,16 +103,79 @@ export type StorageRow = {
actions: StorageActionView[];
};
+// THE SAVED-VIDEO STORE, AS A ROW OF ITS OWN.
+//
+// It is the one large thing in the corpus that is not a channel's `data/`, so
+// no channel move can ever reach it — and until now nothing could move it at
+// all. It is NOT a location (nothing lives "on" it) and it is not a channel, so
+// it gets its own block under the locations rather than being forced into
+// either table.
+export type SavedVideosView = {
+ // `paths.savedVideosDir` — the path every reader uses, moved or not.
+ dir: string;
+ // Where the bytes actually are: the link target, or `dir` when in place.
+ at: string;
+ // The location it is on, or "" for the corpus volume.
+ locationId: string;
+ locationLabel: string;
+ status: SavedVideosStoreStatus;
+ statusLabel: string;
+ detail?: string;
+ bytes: number;
+ files: number;
+ // Destinations the Move control offers: every configured location it is not
+ // already on, plus "" (in place) when it is somewhere else.
+ destinations: Array<{ id: string; label: string }>;
+ // Why every control is off, or null. A move of the store shares the relocate
+ // queue key with channel moves and the re-point, so one running anywhere
+ // freezes this too.
+ busy: string | null;
+ // A marker is present AND nothing is running — the state Resume and Clear
+ // exist for. Separate from `busy`, which the marker itself sets: reading the
+ // hatch off `busy === null` would hide it in exactly the state it is for.
+ canResume: boolean;
+};
+
export type StorageRowsPayload = {
rows: StorageRow[];
+ // Absent when the caller did not ask for it (a unit test of the rows alone).
+ savedVideos?: SavedVideosView;
+ // Media bytes per row id, INCLUDING "internal". The same numbers the rows
+ // carry, lifted out so a caller that wants the totals (the /channels meter
+ // bridge) does not have to re-fold the rows.
+ bytesOnLocation: Record<string, number>;
+ // The corpus volume's share — `bytesOnLocation.internal`, named because it is
+ // the number the whole exercise is about: what is still on the disk that is
+ // 96 % full.
+ bytesInPlace: number;
defaultLocationId: string;
// Whether `udisksctl` resolved in this process. False in a container, and the
// reason the Mount button is withheld there.
udisksctlAvailable: boolean;
};
+// What the shell measured about the store. Every fact arrives as an argument,
+// as everywhere in views/: the walk, the lstat and the marker read are the
+// shell's.
+export type SavedVideosInputs = {
+ dir: string;
+ at: string;
+ locationId: string;
+ status: SavedVideosStoreStatus;
+ detail?: string;
+ bytes: number;
+ files: number;
+ hasMarker: boolean;
+};
+
export type StorageRowsInputs = {
locations: readonly StorageLocation[];
+ // Omit to leave `savedVideos` off the payload entirely.
+ savedVideos?: SavedVideosInputs;
+ // The corpus volume as a row. Absent → no internal row (a caller that only
+ // wants the configured locations). `freeBytes` is a statfs of the root, taken
+ // by the shell, because nothing in views/ may touch a disk.
+ internal?: { root: string; freeBytes?: number };
defaultLocationId: string;
// By location id. A location with no entry has never been probed in this
// process — treated as `missing` with an unknown identity rather than
@@ -119,6 +201,16 @@ export const STORAGE_STATUS_LABEL: Record<StorageLocationStatus, string> = {
};
export const REPOINT_JOB_KIND = "repoint-storage-location";
+export const SAVED_VIDEOS_JOB_KIND = "relocate-saved-videos";
+
+export const SAVED_VIDEOS_STATUS_LABEL: Record<SavedVideosStoreStatus, string> =
+ {
+ "in-place": "In place",
+ ok: "Relocated · reachable",
+ unreachable: "Relocated · unreachable",
+ "in-transition": "Move in flight",
+ inconsistent: "Inconsistent",
+ };
function identityLine(identity: StorageIdentity): string | null {
if (!identity.known) return null;
@@ -139,29 +231,80 @@ function countsOf(rollup: LocationRollup | undefined): StorageChannelCounts {
};
}
+// Bytes → GB with two decimals, or TB past a terabyte. Local rather than
+// `lib/format`'s formatBytes because this module is reachable from the client
+// bundle and is deliberately import-free.
+function bytesLabel(n: number): string {
+ const GB = 1024 ** 3;
+ if (n >= 1024 * GB) return `${(n / (1024 * GB)).toFixed(2)} TB`;
+ if (n >= GB) return `${(n / GB).toFixed(2)} GB`;
+ if (n >= 1024 * 1024) return `${(n / (1024 * 1024)).toFixed(1)} MB`;
+ return `${n} B`;
+}
+
+// THE ONE WORDING OF A SIZE THAT MAY BE PARTLY UNKNOWN.
+//
+// A channel whose snapshot predates `totalMediaBytes` contributes nothing to
+// the sum, and a total that silently omits it is worse than no total: it ranks
+// a 400 GB channel as empty. So the count of unmeasured channels travels with
+// the figure everywhere it goes, and a row where NOTHING could be measured says
+// so instead of printing "0 B".
+export function storageBytesText(bytes: number, unknown: number): string {
+ if (unknown > 0 && bytes === 0) {
+ return "size unknown until Refresh report";
+ }
+ if (unknown > 0) {
+ return `${bytesLabel(bytes)} + ${unknown} unmeasured`;
+ }
+ return bytesLabel(bytes);
+}
+
// The one running re-point, or null. Reported for the whole page rather than
// per row because the job record carries no location id: it is enqueued on the
// shared `relocate` queue key, which the registry caps at concurrency 1, so
// "one is running" is a fact about the machine and not about a row.
function runningRepoint(registry: RegistryReader): string | null {
+ // TWO KINDS, ONE QUEUE. A re-point and a saved-video store move both rewrite
+ // symlinks under the same roots and both share `relocationQueueKey()`, which
+ // the registry caps at concurrency 1 — so either one running is a fact about
+ // the machine and freezes every row on this page, not just its own. (A
+ // CHANNEL move is on that key too and deliberately does NOT freeze this page:
+ // it touches one channel's `data/`, never a location's root or the store, and
+ // /storage has been usable during one since locations shipped.)
const job = registry
.list()
.find(
(j) =>
- j.kind === REPOINT_JOB_KIND &&
+ (j.kind === REPOINT_JOB_KIND || j.kind === SAVED_VIDEOS_JOB_KIND) &&
(j.status === "running" || j.status === "queued"),
);
if (!job) return null;
+ const what =
+ job.kind === REPOINT_JOB_KIND
+ ? "A storage re-point"
+ : "A saved-video store move";
return (
- `A storage re-point is ${job.status} (job ${job.id}). One runs at a ` +
+ `${what} is ${job.status} (job ${job.id}). One runs at a ` +
`time — wait for it to finish, or cancel it on /jobs.`
);
}
+// The one spelling of the /channels filter param, so the two surfaces that
+// write it cannot disagree about whether it is `location` or `loc`.
+export const LOCATION_FILTER_PARAM = "location";
+export const INTERNAL_ROW_ID = "internal";
+
+export function channelsHrefForLocation(id: string): string {
+ // `size` descending, because the only reason to open this list is to find the
+ // channels worth moving, and that is the biggest ones.
+ return `/channels?${LOCATION_FILTER_PARAM}=${encodeURIComponent(id)}&sort=size`;
+}
+
export function buildStorageRows(i: StorageRowsInputs): StorageRowsPayload {
const busy = runningRepoint(i.registry);
const udisksctlAvailable = i.udisksctlAvailable ?? false;
- const rows = i.locations.map((loc): StorageRow => {
+ const bytesOnLocation: Record<string, number> = {};
+ const configured = i.locations.map((loc): StorageRow => {
const probe = i.probes[loc.id];
const status: StorageLocationStatus = probe?.status ?? "missing";
const counts = countsOf(i.rollups[loc.id]);
@@ -182,13 +325,18 @@ export function buildStorageRows(i: StorageRowsInputs): StorageRowsPayload {
offered: !busy,
...(busy ? { withheld: busy } : {}),
},
- deleteAction(loc, counts, busy),
+ deleteAction(loc, counts, busy, i.savedVideos?.locationId === loc.id),
];
+ const roll = i.rollups[loc.id];
+ const bytes = roll?.bytes ?? 0;
+ const unknownBytes = roll?.unknownBytes ?? 0;
+ bytesOnLocation[loc.id] = bytes;
return {
id: loc.id,
label: loc.label || loc.id,
root: loc.root,
+ kind: "location",
isDefault: loc.id === i.defaultLocationId,
autoRepoint: loc.autoRepoint,
status,
@@ -196,6 +344,10 @@ export function buildStorageRows(i: StorageRowsInputs): StorageRowsPayload {
identity: probe ? identityLine(probe.identity) : null,
channels: counts,
channelsText: `${counts.ok} ok / ${counts.unreachable} unreachable / ${counts.moving} moving`,
+ channelsHref: channelsHrefForLocation(loc.id),
+ bytes,
+ unknownBytes,
+ bytesText: storageBytesText(bytes, unknownBytes),
...(probe?.freeBytes !== undefined ? { freeBytes: probe.freeBytes } : {}),
lastProbeAgeMs: probe ? Math.max(0, i.now - probe.probedAt) : 0,
...(probe?.warning ? { warning: probe.warning } : {}),
@@ -204,7 +356,114 @@ export function buildStorageRows(i: StorageRowsInputs): StorageRowsPayload {
actions,
};
});
- return { rows, defaultLocationId: i.defaultLocationId, udisksctlAvailable };
+ const rows = i.internal
+ ? [internalRow(i, bytesOnLocation), ...configured]
+ : configured;
+ return {
+ rows,
+ ...(i.savedVideos
+ ? { savedVideos: savedVideosView(i, i.savedVideos, busy) }
+ : {}),
+ bytesOnLocation,
+ bytesInPlace: bytesOnLocation[INTERNAL_ROW_ID] ?? 0,
+ defaultLocationId: i.defaultLocationId,
+ udisksctlAvailable,
+ };
+}
+
+// THE STORE'S OWN ROW. `busy` is the page's — one relocation runs at a time
+// across all three kinds on the shared queue key, so a channel move in flight
+// freezes this too, and saying so is better than a button that refuses.
+function savedVideosView(
+ i: StorageRowsInputs,
+ sv: SavedVideosInputs,
+ pageBusy: string | null,
+): SavedVideosView {
+ const on = i.locations.find((l) => l.id === sv.locationId);
+ const busy =
+ pageBusy ??
+ (sv.status === "in-transition"
+ ? (sv.detail ?? "A move of the store is in flight or was interrupted.")
+ : null);
+ // "In place" is offered only when it is NOT in place, and a location is
+ // offered only when it is not already the one it is on. An option that would
+ // be a no-op is an option that refuses.
+ const destinations: Array<{ id: string; label: string }> = [];
+ if (sv.locationId !== "") {
+ destinations.push({ id: "", label: "Internal (in place)" });
+ }
+ for (const l of i.locations) {
+ if (l.id !== sv.locationId) {
+ destinations.push({ id: l.id, label: l.label || l.id });
+ }
+ }
+ return {
+ dir: sv.dir,
+ at: sv.at,
+ locationId: sv.locationId,
+ locationLabel: on ? on.label || on.id : "Internal (in place)",
+ status: sv.status,
+ statusLabel: SAVED_VIDEOS_STATUS_LABEL[sv.status],
+ ...(sv.detail ? { detail: sv.detail } : {}),
+ bytes: sv.bytes,
+ files: sv.files,
+ destinations,
+ busy,
+ canResume: sv.hasMarker && pageBusy === null,
+ };
+}
+
+// THE CORPUS VOLUME AS A ROW, AND IT IS FIRST.
+//
+// It is first because it is the row with the problem: 523 GB on a disk with 67
+// left is not a footnote under the locations that were added to fix it. It is
+// `available` by construction — the process is reading the corpus out of it —
+// and every action is withheld with the reason, which for all five is the same
+// fact said five ways: there is nothing here to configure, because this is not
+// a configured place. Moving media ONTO it is "move back in place", and that
+// lives on the channel's own Storage panel where the media is.
+function internalRow(
+ i: StorageRowsInputs,
+ bytesOnLocation: Record<string, number>,
+): StorageRow {
+ const internal = i.internal as NonNullable<StorageRowsInputs["internal"]>;
+ const roll = i.rollups[INTERNAL_ROW_ID];
+ const counts = countsOf(roll);
+ const bytes = roll?.bytes ?? 0;
+ const unknownBytes = roll?.unknownBytes ?? 0;
+ bytesOnLocation[INTERNAL_ROW_ID] = bytes;
+ const withheld =
+ "The corpus volume is where media lives when it has not been moved " +
+ "anywhere. It is not a configured location: there is nothing to re-point, " +
+ "and moving media back onto it is the channel's own Storage panel.";
+ return {
+ id: INTERNAL_ROW_ID,
+ label: "Internal (in place)",
+ root: internal.root,
+ kind: "internal",
+ isDefault: false,
+ autoRepoint: false,
+ status: "available",
+ statusLabel: STORAGE_STATUS_LABEL.available,
+ identity: null,
+ channels: counts,
+ channelsText: `${counts.ok} ok / ${counts.unreachable} unreachable / ${counts.moving} moving`,
+ channelsHref: channelsHrefForLocation(INTERNAL_ROW_ID),
+ bytes,
+ unknownBytes,
+ bytesText: storageBytesText(bytes, unknownBytes),
+ ...(internal.freeBytes !== undefined ? { freeBytes: internal.freeBytes } : {}),
+ lastProbeAgeMs: 0,
+ busy: null,
+ actions: (
+ ["refresh", "repoint", "mount", "edit", "delete"] as StorageActionKind[]
+ ).map((kind) => ({
+ kind,
+ label: kind[0].toUpperCase() + kind.slice(1),
+ offered: false,
+ withheld,
+ })),
+ };
}
// RE-POINT IS OFFERED FOR EXACTLY ONE STATUS. `mounted-elsewhere` means the
@@ -302,10 +561,23 @@ function deleteAction(
loc: StorageLocation,
counts: StorageChannelCounts,
busy: string | null,
+ // The saved-video store lives here. A resident like any other, and orphaned
+ // by exactly the same delete — see the action's own refusal.
+ hasStore: boolean,
): StorageActionView {
if (busy) {
return { kind: "delete", label: "Delete", offered: false, withheld: busy };
}
+ if (hasStore) {
+ return {
+ kind: "delete",
+ label: "Delete",
+ offered: false,
+ withheld:
+ `The saved-video store is under ${loc.root}. Move it back in place, ` +
+ `or onto another location, first.`,
+ };
+ }
if (counts.total > 0) {
return {
kind: "delete",
diff --git a/common/ytdlp/downloadOneManaged.ts b/common/ytdlp/downloadOneManaged.ts
@@ -278,6 +278,12 @@ async function finalizeAppExtraction(opts: {
storeDir,
keepReason: opts.category === "none" ? undefined : opts.category,
origin: opts.origin,
+ // The store-in-transition guard. A move of the saved-video store is a
+ // multi-hour copy, and a container landing in the middle of it is lost
+ // three different ways — see lib/savedVideoStore.ts. The catch below is
+ // already the right handling: the container stays in the data dir with a
+ // line in the log, and the next persist (after the move) picks it up.
+ paths: opts.paths,
});
opts.onLog(
`Persisted source video to ${path.join(pointer.dir, pointer.file)} (${pointer.bytes} bytes).\n`,
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,12 @@
# Changelog
## [Unreleased]
+- **A relocate job says how far it has got.** `rsync` has been printing its progress the whole time (`--info=progress2`) and every frame of it went into the job log as a carriage-return redraw of one line — so a 131 GB move and a 3 MB one looked identical from `/jobs`: a spinner. Now each frame is parsed into the **task bar** every other long job on that page already draws, reading `12.3 GB of 45.6 GB · 27 % · 110.50MB/s · ETA 5:32`, and the log gets **one line per 10 %** instead of several thousand frames of one. The percentage is against the tree the job already measured for its space check, not rsync's own — under incremental recursion that one is a percentage of what it has enumerated so far and walks backwards.
+- **`/channels` is where storage is managed now.** Two new columns: **Location** (which volume this channel's media is on — *Internal* when it has not moved) and **Size** (every byte under its `data/`, from its last report, sortable biggest-first). Free space is *not* a column, because it is a fact about a disk and not about a channel: there is one read-out per **volume** in a new bar above the rack, and each chip is also a **filter** — `?location=platter` lists exactly the channels on that drive, and `/storage` links straight here with the biggest first. Beside them, **Free up N GB**: type a number, press *Select largest*, and the largest channels still on the internal disk are ticked until the target is met, ready for the Move button that was already there. Channels already on another volume are never picked (moving one frees nothing on the disk you are emptying) and channels whose report carries no size are **skipped and counted** rather than ranked as empty — which would have put the biggest thing on the disk at the bottom of the list.
+- **The corpus volume is a row on `/storage`, and it is the first one.** 523 GB on a disk with 67 GB left is not a footnote under the locations that were added to fix it. It shows the channels in place, what they hold and the free space, and links to its own list. It is the one row with nothing to refresh, re-point, mount, edit or delete — and it says so, once, rather than as five greyed buttons. Every location row grows the same size read-out, and a location whose channels have never had a report says **"size unknown until Refresh report"** rather than claiming a 2 TB drive holds nothing.
+- **A channel's Storage panel has one verb.** *Move media to…* and *Move back in place* were two sections with two buttons whose availability was the inverse of each other — one decision split across two controls. The corpus volume is now a destination in the same select; picking it moves the media back. While the media is on a location it is the only destination offered, because a move straight from one location to another is refused by the mover itself.
+- **The saved-video store can be moved to another drive.** It is the one large thing in the corpus that belongs to no channel, so no channel move could ever reach it. `/storage` now shows it with its size and where it is, and moves it onto a location — and back — by exactly the mechanism a channel uses: the copy is verified before the source is touched, a symlink is left behind, and every reader keeps working unchanged. An interrupted move can be **resumed** rather than restarted, and its marker cleared if it cannot.
+- **A drive that disappears now disables and flags the channels on it, and un-does that when it comes back.** Every guard in the app refuses work on unreachable media at the moment the work starts; none of them was a *detector*, so a channel whose drive fell off a cable sat there being refused with nothing anywhere saying why. A five-minute pass now notices, **pauses those channels and marks them** — the tier column shows a *storage* badge with the reason — and restores each one to the tier it had when the drive returns. It never claims a channel you paused yourself, and changing a tier by hand permanently takes it out of the machine's hands. It writes only when something has actually changed, and never arms at all under `ARCHILYZER_IDLE_BOOT`.
- **The drives a channel's media lives on are named places now, and one click re-points them.** The cold root used to be a single string typed into Settings, and a relocated channel's `config.dataDir` an absolute path — so when the platter was automounted at `/run/media/user/<uuid>` and came back somewhere else, every channel on it read *unreachable* and the only remedy was SSH and hand edits. **`/storage`** (twelfth entry, under Machine) lists each media root as a **location** with a name, a status and the channels on it: `Available`, `Not mounted`, `Not attached`, or **`Mounted elsewhere`** — which is the one that matters, because it means the disk is here under a different mountpoint, and the row then offers **Re-point**, which rewrites every channel's `data/` symlink and `config.dataDir` and **moves no bytes at all**. There is a **Refresh** per row (a probe is `findmnt`, memoised for ten seconds, and it never writes availability to `settings.json`), a **Mount** for an attached-but-unmounted volume, a per-location **auto re-point** opt-in for operators who would rather it just happened, and a boot pass that checks every location as the editor starts. Identity is the volume's filesystem **UUID**, learned at the last successful probe — in a container there are no block devices to learn it from, every probe **fails open to "unknown", and re-point by path is the whole story** (see RUNNING_IN_DOCKER.md). The old `settings.storage.mediaRoot` **migrates on read** into a one-entry list called *Default*; the Settings field is now a link to the page. Everywhere a move starts, the destination is a **name picked from a list** rather than a path retyped per channel: the channel's Storage panel has a **destination select** showing each drive's current state (with *Another root…* keeping the free-text box), and `/channels`' selection deck has the same select for a whole batch — and what reaches the server is the **id**, never the root, so a page rendered before a re-point cannot aim a batch at a root that has since moved. The badge on every channel row says **`on Platter`** instead of sixty columns of absolute path, or **`on Platter — unreachable`** when the drive is not there. **And an interrupted move can be finished.** The controller has always resumed a half-done copy; nothing in the editor could reach it, so the only offered way out of a killed rsync was *Clear marker* and a full re-copy — which for the incident behind this work meant re-copying 131 GB that was already correctly on the far side. The panel now offers **Resume move** beside it, and the same release closes the three ways that incident happened: the relocate copy raced an auto-queue digest unit that made **no job record**, so "is this channel busy" now asks the lanes as well as the registry, and a lane that finds a relocation marker stops instead of writing into a directory being copied; a sidecar written mid-copy left one directory timestamp differing and `verifyCopy` refused the whole 131 GB, so a drift that is *only* directory mtimes now gets one more `rsync -a` pass and a re-verify (content drift still refuses, and still says the source has not been touched). Also fixed: on `/channels` a dimmed row's **Advanced menu drew underneath the rows below it** — `opacity` on a `<tr>` makes a stacking context, so the row is dimmed cell by cell now, and the cell hosting the popover is left alone.
- **The operations poll reads the auto-queue's state file once instead of four times.** Every payload the editor draws — the jobs head, the workers grid, the operations board, the sync schedule, the widget's tiles, the pulse token — used to be computed by a function that did its own reading, so each of the four lanes on `/operations` opened `.auto-queue/state.json` for itself: four parses of the same document every three seconds, on a page whose four lanes were always reading one document. Those builders are pure functions in the shared core now — they are handed the settings, the registry, the scheduler, the pool, the clock and their readings, and they cannot reach disk or construct a singleton, which a layer test enforces rather than a comment asking nicely. The reading happens once, at the edge, and is shared. **Nothing moved that you can see**: same pages, same URLs, same JSON on every endpoint, same numbers — the difference is that each payload now has unit tests of its own (the console's cooldown filter, the pulse token's sensitivity, the workers grid's task grouping), where previously the only way to test one was to render the page that showed it.
- **`/channels` is a rack now, with one selection deck and a meter bridge.** The page had two selection bars for one selection — a floating one for Tier and Focus, and a second block below sixty-seven rows for Move media, both saying "N selected" and both offering Clear. There is **one deck**: it docks under the table when you tick a row, carries **Tier**, **Focus** and **Media** side by side, and unmounts when you untick. The destination root lives in its own box beside the button (the button used to carry it in its label, where it truncated to *Move media to…* and you could not read where the files were going). **The table stops spilling off the screen.** It lives in one scroll region: the column headers pin to its top, the checkbox and slug cells pin to its left, and a section's name pins under the headers — so the identity column and the meter bridge header stay on screen while sixteen columns scroll sideways. The six pipeline columns read as **one block** rather than six loose dashes: a shared *Pipeline* eyebrow, a surface behind them, a rule at each end. **Rows are 41 px instead of ~90.** The tier cell is one line, and being held by a focus is a small **held** chip rather than the same orange sentence repeated on sixty-one rows — the sentence is stated once, with a count, on the focus line above the table, and each chip still carries the full reason for a screen reader and on hover. Opening a row's *Advanced* overlays the rows below instead of pushing them down. **The page's caveat is at the top.** The note saying every number here is read from each channel's last report, and how old the oldest one is, used to be the last thing on the page in 11 px type under a floating bar; it is the subtitle beside the title now, with the channel count. The band legend and the Names·A / Names·T explainer moved above the table too, beside *Group by section*. In the header, *Sync every channel*, *Full sweep every channel* and *Update all reports* are outlines under an **Every channel** eyebrow that says what they sweep, and **New channel** is the only filled button. Nothing on disk moved and no control changed its name.
diff --git a/editor/app/api/ops/_lib.ts b/editor/app/api/ops/_lib.ts
@@ -0,0 +1,209 @@
+import { NextResponse } from "next/server";
+import { authorizeWorkerRequest } from "yt-dlp-transcript-common/lib/workerToken";
+import { isValidChannelSlug } from "yt-dlp-transcript-common/controller/channels";
+import type { StreamActionResult } from "yt-dlp-transcript-common/jobs/streamCommand";
+import type { QueueOutcome } from "../../channels/lib/queueForSlugs";
+
+// THE OPS API IS ADAPTERS, AND NOTHING ELSE.
+//
+// Every route under /api/ops is ~5 lines that validate a JSON body and call ONE
+// existing server action. No route may contain a rule the UI does not already
+// enforce: the point of the layer is that an agent driving the editor over HTTP
+// and an operator clicking the same button get the same refusal, with the same
+// sentence, from the same code. A check written here would be a second opinion
+// nobody maintains.
+//
+// THE TOKEN IS THE WORKER TOKEN, ON PURPOSE. `WORKER_TOKEN` already gates the
+// LAN worker protocol and already means "this instance accepts instructions
+// from something that is not the browser in front of it". A second secret would
+// be a second thing to distribute, rotate and leave unset; the failure modes are
+// identical, so the gate is. Unset => 503 (the surface is off, you opt in),
+// wrong => 401.
+//
+// A JOB-STARTING ROUTE RETURNS A jobId AND NEVER STREAMS. runManagedFunction
+// hands back a ReadableStream the browser consumes; an HTTP caller wants to
+// disconnect and poll. So every adapter cancels the stream (which stops pushing
+// into the controller and leaves the on-disk log running — see streamCommand's
+// `cancel()` note) and returns the id. Follow it with /api/jobs/<id>/log.
+
+export type OpsBody = Record<string, unknown>;
+
+// Thrown by the field readers below; caught by `ops()` and rendered as a 400.
+export class OpsInputError extends Error {}
+
+export function opsFail(
+ error: string,
+ status = 400,
+ extra?: Record<string, unknown>,
+): NextResponse {
+ return NextResponse.json({ ok: false, error, ...extra }, { status });
+}
+
+// Auth + body parse + unknown-key rejection, wrapped around one handler.
+//
+// UNKNOWN KEYS ARE A 400, not a silent ignore. A caller that misspells
+// `downloadFilterExclude` would otherwise get a cheerful `{ ok: true }` and a
+// channel that still downloads everything. The allow-list IS the route's
+// documented body shape.
+export async function ops(
+ request: Request,
+ allowedKeys: readonly string[],
+ run: (body: OpsBody) => Promise<NextResponse>,
+): Promise<NextResponse> {
+ const auth = authorizeWorkerRequest(request.headers.get("authorization"));
+ if (!auth.ok) return opsFail(auth.error, auth.status);
+ let body: unknown;
+ try {
+ body = await request.json();
+ } catch {
+ return opsFail("malformed JSON body");
+ }
+ if (typeof body !== "object" || body === null || Array.isArray(body)) {
+ return opsFail("body must be a JSON object");
+ }
+ const unknown = Object.keys(body as OpsBody).filter(
+ (k) => !allowedKeys.includes(k),
+ );
+ if (unknown.length) {
+ return opsFail(
+ `unknown key(s): ${unknown.join(", ")} — this route accepts ${
+ allowedKeys.length ? allowedKeys.join(", ") : "no keys"
+ }`,
+ );
+ }
+ try {
+ return await run(body as OpsBody);
+ } catch (e) {
+ if (e instanceof OpsInputError) return opsFail(e.message);
+ return opsFail((e as Error).message, 500);
+ }
+}
+
+// A GET route's gate. Same token, no body.
+export function opsAuth(request: Request): NextResponse | null {
+ const auth = authorizeWorkerRequest(request.headers.get("authorization"));
+ return auth.ok ? null : opsFail(auth.error, auth.status);
+}
+
+// --- field readers ----------------------------------------------------------
+
+export function reqString(body: OpsBody, key: string): string {
+ const v = body[key];
+ if (typeof v !== "string" || !v.trim()) {
+ throw new OpsInputError(`"${key}" is required and must be a non-empty string`);
+ }
+ return v.trim();
+}
+
+// A CHANNEL SLUG, NOT MERELY A STRING. Every slug below reaches a `path.join`
+// under `channelsDir`, and the readers swallow their own errors — so a
+// traversing segment would fail SILENTLY (an empty config, an "empty channel")
+// rather than loudly. `isValidChannelSlug` is CHANNEL_SLUG_RE, which forbids
+// "/" and "..", and is what every other slug-taking surface in the app uses.
+//
+// One reader for every route rather than a check per route: a route added later
+// gets this for free by calling reqSlug instead of reqString, and there is one
+// place to be wrong.
+export function reqSlug(body: OpsBody, key: string): string {
+ const v = reqString(body, key);
+ if (!isValidChannelSlug(v)) {
+ throw new OpsInputError(
+ `"${v}" is not a valid channel slug (letters, digits, ".", "_", "-"; must start with a letter or digit)`,
+ );
+ }
+ return v;
+}
+
+export function reqSlugs(body: OpsBody, key: string): string[] {
+ const values = reqStringArray(body, key);
+ for (const v of values) {
+ if (!isValidChannelSlug(v)) {
+ throw new OpsInputError(
+ `"${v}" is not a valid channel slug (letters, digits, ".", "_", "-"; must start with a letter or digit)`,
+ );
+ }
+ }
+ return values;
+}
+
+export function optString(body: OpsBody, key: string): string | undefined {
+ const v = body[key];
+ if (v === undefined) return undefined;
+ if (typeof v !== "string") {
+ throw new OpsInputError(`"${key}" must be a string`);
+ }
+ return v;
+}
+
+export function optBool(body: OpsBody, key: string): boolean | undefined {
+ const v = body[key];
+ if (v === undefined) return undefined;
+ if (typeof v !== "boolean") {
+ throw new OpsInputError(`"${key}" must be a boolean`);
+ }
+ return v;
+}
+
+export function reqStringArray(body: OpsBody, key: string): string[] {
+ const v = body[key];
+ if (
+ !Array.isArray(v) ||
+ v.length === 0 ||
+ v.some((s) => typeof s !== "string" || !s.trim())
+ ) {
+ throw new OpsInputError(
+ `"${key}" is required and must be a non-empty array of strings`,
+ );
+ }
+ return (v as string[]).map((s) => s.trim());
+}
+
+export function oneOf<T extends string>(
+ body: OpsBody,
+ key: string,
+ values: readonly T[],
+): T {
+ const v = reqString(body, key);
+ if (!(values as readonly string[]).includes(v)) {
+ throw new OpsInputError(`"${key}" must be one of ${values.join(", ")}`);
+ }
+ return v as T;
+}
+
+// --- result mapping ---------------------------------------------------------
+
+// StreamActionResult -> { ok: true, jobId } | { ok: false, error, info? }.
+// The stream is cancelled, never returned: see the header.
+export function jobResponse(result: StreamActionResult): NextResponse {
+ if (!result.ok) {
+ return opsFail(result.error, 400, result.info ? { info: true } : undefined);
+ }
+ void result.stream.cancel();
+ return NextResponse.json({ ok: true, jobId: result.jobId });
+}
+
+// The `{ error } | undefined` shape every /channels action returns.
+export function actionResponse(
+ result: { error: string } | undefined,
+): NextResponse {
+ if (result?.error) return opsFail(result.error);
+ return NextResponse.json({ ok: true });
+}
+
+// The `{ ok } | { ok: false, error }` shape the lane/pause actions return.
+export function okResponse(
+ result: { ok: boolean; error?: string },
+): NextResponse {
+ if (!result.ok) return opsFail(result.error ?? "action failed");
+ return NextResponse.json({ ok: true });
+}
+
+// A bulk fan-out's { queued, skipped }. A skip is not a failure — the caller
+// gets both lists and decides, exactly as the bulk bar in the UI does.
+export function queueResponse(outcome: QueueOutcome): NextResponse {
+ return NextResponse.json({
+ ok: true,
+ queued: outcome.queued,
+ skipped: outcome.skipped,
+ });
+}
diff --git a/editor/app/api/ops/build-deploy/route.ts b/editor/app/api/ops/build-deploy/route.ts
@@ -0,0 +1,33 @@
+import { NextResponse } from "next/server";
+import {
+ buildAndDeployAction,
+ buildAndDeployAllSitesAction,
+} from "../../../sites/lib/buildAction";
+import { jobResponse, OpsInputError, ops, optBool, optString } from "../_lib";
+
+export const dynamic = "force-dynamic";
+
+// POST { siteId: string, skipArchives? } | { all: true, skipArchives? }
+// -> { ok: true, jobId }
+//
+// One managed job either way (build then deploy, one log, one Cancel), so the
+// caller polls /api/jobs/<jobId>/log exactly as for a single site.
+export async function POST(request: Request) {
+ return ops(request, ["siteId", "all", "skipArchives"], async (body) => {
+ const skipArchives = optBool(body, "skipArchives");
+ const all = optBool(body, "all");
+ const siteId = optString(body, "siteId");
+ if (all) {
+ if (siteId) throw new OpsInputError('send either "siteId" or "all", not both');
+ return jobResponse(await buildAndDeployAllSitesAction(skipArchives));
+ }
+ if (!siteId?.trim()) {
+ throw new OpsInputError('"siteId" is required (or send { "all": true })');
+ }
+ return jobResponse(await buildAndDeployAction(siteId.trim(), skipArchives));
+ });
+}
+
+export function GET() {
+ return NextResponse.json({ ok: false, error: "POST only" }, { status: 405 });
+}
diff --git a/editor/app/api/ops/build-index/route.ts b/editor/app/api/ops/build-index/route.ts
@@ -0,0 +1,11 @@
+import { buildIndexAction } from "../../../sites/lib/buildAction";
+import { jobResponse, ops, optString } from "../_lib";
+
+export const dynamic = "force-dynamic";
+
+// POST { queueKey? } -> { ok: true, jobId }. Rebuilds the LMDB corpus index.
+export async function POST(request: Request) {
+ return ops(request, ["queueKey"], async (body) =>
+ jobResponse(await buildIndexAction(optString(body, "queueKey"))),
+ );
+}
diff --git a/editor/app/api/ops/build-site/route.ts b/editor/app/api/ops/build-site/route.ts
@@ -0,0 +1,59 @@
+import { NextResponse } from "next/server";
+import {
+ buildAllSitesAction,
+ buildExportAction,
+} from "../../../sites/lib/buildAction";
+import { jobResponse, OpsInputError, ops, optBool } from "../_lib";
+
+export const dynamic = "force-dynamic";
+
+// POST { siteIds: string[], skipData?, skipArchives? } | { all: true, skipArchives? }
+//
+// BUILD WITHOUT DEPLOYING. `siteIds` queues one build-export job per site on the
+// shared build queue (they run one at a time, as they do from /sites) and
+// returns `{ jobs: [{ siteId, jobId }] }` — a list, because there is a job per
+// site and a caller waiting on them needs all the ids. `{ all: true }` is the
+// single build-all job instead, which is the docker fan-out.
+export async function POST(request: Request) {
+ return ops(
+ request,
+ ["siteIds", "all", "skipData", "skipArchives"],
+ async (body) => {
+ const skipArchives = optBool(body, "skipArchives");
+ if (optBool(body, "all")) {
+ if (body.siteIds !== undefined) {
+ throw new OpsInputError('send either "siteIds" or "all", not both');
+ }
+ return jobResponse(await buildAllSitesAction(skipArchives));
+ }
+ const raw = body.siteIds;
+ if (
+ !Array.isArray(raw) ||
+ raw.length === 0 ||
+ raw.some((s) => typeof s !== "string" || !s.trim())
+ ) {
+ throw new OpsInputError(
+ '"siteIds" is required and must be a non-empty array of strings (or send { "all": true })',
+ );
+ }
+ const skipData = optBool(body, "skipData");
+ const jobs: { siteId: string; jobId: string }[] = [];
+ const skipped: { siteId: string; reason: string }[] = [];
+ for (const siteId of (raw as string[]).map((s) => s.trim())) {
+ const result = await buildExportAction(
+ siteId,
+ undefined,
+ skipData,
+ skipArchives,
+ );
+ if (!result.ok) {
+ skipped.push({ siteId, reason: result.error });
+ continue;
+ }
+ void result.stream.cancel();
+ jobs.push({ siteId, jobId: result.jobId });
+ }
+ return NextResponse.json({ ok: true, jobs, skipped });
+ },
+ );
+}
diff --git a/editor/app/api/ops/channel-config/route.ts b/editor/app/api/ops/channel-config/route.ts
@@ -0,0 +1,47 @@
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels";
+import {
+ applyChannelFormPatch,
+ channelConfigToFormData,
+ validateChannelFormPatch,
+} from "../../../channels/components/channelConfigToForm";
+import { updateChannelAction } from "../../../channels/actions";
+import { actionResponse, OpsInputError, ops, reqSlug } from "../_lib";
+
+export const dynamic = "force-dynamic";
+
+// POST { slug: string, patch: { <Configure-form field>: string|number|boolean|null } }
+//
+// The patch keys are the FORM's field names, not ChannelConfig's, because the
+// form is what validates them: `downloadFilterInclude` / `downloadFilterExclude`
+// rather than a `downloadFilter` object, `ytdlpExtraArgs` as a string or an
+// array of lines. See channelConfigToForm.ts for why the patch is laid over the
+// channel's current form representation instead of being written directly.
+//
+// `""` (or null) clears a field, exactly as clearing the input does.
+export async function POST(request: Request) {
+ return ops(request, ["slug", "patch"], async (body) => {
+ const slug = reqSlug(body, "slug");
+ const patch = body.patch;
+ if (
+ typeof patch !== "object" ||
+ patch === null ||
+ Array.isArray(patch)
+ ) {
+ throw new OpsInputError('"patch" must be a JSON object');
+ }
+ // KEYS FIRST, CHANNEL SECOND. A misspelled field is a fact about the
+ // request; reporting "channel not found" for it would hide the real error.
+ try {
+ validateChannelFormPatch(patch as Record<string, unknown>);
+ } catch (e) {
+ throw new OpsInputError((e as Error).message);
+ }
+ const paths = getPaths();
+ const existing = await readChannelConfig(paths, slug);
+ if (!existing) throw new OpsInputError(`Channel "${slug}" not found`);
+ const fd = channelConfigToFormData(existing);
+ applyChannelFormPatch(fd, patch as Record<string, unknown>);
+ return actionResponse(await updateChannelAction(slug, undefined, fd));
+ });
+}
diff --git a/editor/app/api/ops/channel-priority/route.ts b/editor/app/api/ops/channel-priority/route.ts
@@ -0,0 +1,91 @@
+import { NextResponse } from "next/server";
+import {
+ STORED_CHANNEL_TIERS,
+ PRIORITY_OPERATIONS,
+ type PriorityOperation,
+ type StoredChannelTier,
+} from "yt-dlp-transcript-common/lib/channelPriority";
+import {
+ applyChannelPriorityPresetAction,
+ setChannelOperationTierAction,
+ setChannelTierAction,
+} from "../../../channels/actions";
+import {
+ actionResponse,
+ OpsInputError,
+ ops,
+ optString,
+ reqSlugs,
+} from "../_lib";
+
+export const dynamic = "force-dynamic";
+
+// POST { slugs: string[], tier?: "normal"|"low"|"paused"|null, operation?: one
+// of sync|transcription|download|digest|backfill, preset?: "sync-only"|"clear" }
+//
+// Three named gestures, one route, because they are one gesture in the UI: the
+// /channels deck's tier control. `operation` present pins that operation's
+// override (null clears it back to the base tier); absent sets the BASE tier.
+// `preset` is the deck's two shortcuts and takes no tier.
+export async function POST(request: Request) {
+ return ops(
+ request,
+ ["slugs", "tier", "operation", "preset"],
+ async (body) => {
+ const slugs = reqSlugs(body, "slugs");
+ const preset = optString(body, "preset");
+ if (preset !== undefined) {
+ if (preset !== "sync-only" && preset !== "clear") {
+ throw new OpsInputError('"preset" must be "sync-only" or "clear"');
+ }
+ return actionResponse(
+ await applyChannelPriorityPresetAction(slugs, preset),
+ );
+ }
+ const rawTier = body.tier;
+ if (rawTier !== null && typeof rawTier !== "string") {
+ throw new OpsInputError(
+ `"tier" must be one of ${STORED_CHANNEL_TIERS.join(", ")} (or null with an "operation", to clear the override)`,
+ );
+ }
+ if (
+ rawTier !== null &&
+ !(STORED_CHANNEL_TIERS as readonly string[]).includes(rawTier)
+ ) {
+ throw new OpsInputError(
+ `"tier" must be one of ${STORED_CHANNEL_TIERS.join(", ")}`,
+ );
+ }
+ const operation = optString(body, "operation");
+ if (operation !== undefined) {
+ if (!(PRIORITY_OPERATIONS as readonly string[]).includes(operation)) {
+ throw new OpsInputError(
+ `"operation" must be one of ${PRIORITY_OPERATIONS.join(", ")}`,
+ );
+ }
+ return actionResponse(
+ await setChannelOperationTierAction(
+ slugs,
+ operation as PriorityOperation,
+ rawTier as StoredChannelTier | null,
+ ),
+ );
+ }
+ if (rawTier === null) {
+ throw new OpsInputError(
+ 'a null "tier" clears an operation override — name an "operation", or send a stored tier',
+ );
+ }
+ return actionResponse(
+ await setChannelTierAction(slugs, rawTier as StoredChannelTier),
+ );
+ },
+ );
+}
+
+export function GET() {
+ return NextResponse.json(
+ { ok: false, error: "POST only" },
+ { status: 405 },
+ );
+}
diff --git a/editor/app/api/ops/channel/[slug]/route.ts b/editor/app/api/ops/channel/[slug]/route.ts
@@ -0,0 +1,100 @@
+import { NextResponse } from "next/server";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import {
+ isValidChannelSlug,
+ readChannelConfig,
+ readChannelSnapshot,
+ readChannelStat,
+} from "yt-dlp-transcript-common/controller/channels";
+import { inspectChannelMedia } from "yt-dlp-transcript-common/lib/channelMedia";
+import {
+ overridesOf,
+ rankOf,
+ tierOf,
+} from "yt-dlp-transcript-common/lib/channelPriority";
+import { getSettings } from "yt-dlp-transcript-common/lib/settings";
+import { opsAuth } from "../../_lib";
+
+export const dynamic = "force-dynamic";
+
+// GET /api/ops/channel/<slug>
+//
+// THE READ SIDE, so a caller never opens transcripts/ itself. Everything a
+// script needs to decide what to do next about one channel: its stored config,
+// its report totals and bucket SIZES (not the id lists — a bucket can hold tens
+// of thousands of ids and a caller deciding "is there work" only needs the
+// count; /api/ops/retry-bucket resolves the ids server-side anyway), its
+// priority tier, and — the one that cannot be inferred from the filesystem —
+// whether its media is reachable.
+//
+// `media.status` is `inspectChannelMedia`'s, which is the ONE module that can
+// tell an unmounted drive from an empty channel. Every enumerator else swallows
+// ENOENT on data/ as "no videos", so a script reading counts alone would read an
+// unmounted platter as a channel that has downloaded nothing.
+//
+// `?counts=1` ADDS THE LIVE ON-DISK COUNTS, and it is opt-in because
+// readChannelStat walks every video directory — eleven thousand readdirs on the
+// largest channel here. The report's totals answer the same question from a
+// file, and a poll loop asking for them every few seconds must not be the thing
+// that hammers the platter.
+export async function GET(
+ request: Request,
+ { params }: { params: Promise<{ slug: string }> },
+) {
+ const denied = opsAuth(request);
+ if (denied) return denied;
+ const { slug } = await params;
+ // The same shape check every POST route applies through reqSlug (_lib.ts);
+ // spelled out here only because the slug arrives as a route param, not in a
+ // body. readChannelConfig swallows its own errors, so a traversing segment
+ // would fail silently rather than loudly.
+ if (!isValidChannelSlug(slug)) {
+ return NextResponse.json(
+ { ok: false, error: `"${slug}" is not a valid channel slug` },
+ { status: 400 },
+ );
+ }
+ const paths = getPaths();
+ const config = await readChannelConfig(paths, slug);
+ if (!config) {
+ return NextResponse.json(
+ { ok: false, error: `Channel "${slug}" not found` },
+ { status: 404 },
+ );
+ }
+ const priority = getSettings().channelPriority;
+ const wantCounts = new URL(request.url).searchParams.get("counts") === "1";
+ const [snapshot, stat, media] = await Promise.all([
+ readChannelSnapshot(paths, slug),
+ wantCounts ? readChannelStat(paths, slug) : Promise.resolve(null),
+ inspectChannelMedia(paths, slug, config),
+ ]);
+ const buckets = snapshot
+ ? Object.fromEntries(
+ Object.entries(snapshot.buckets).map(([k, v]) => [
+ k,
+ Array.isArray(v) ? v.length : v,
+ ]),
+ )
+ : null;
+ return NextResponse.json({
+ ok: true,
+ slug,
+ config,
+ priority: {
+ tier: tierOf(priority, slug),
+ overrides: overridesOf(priority, slug),
+ rank: rankOf(priority, slug),
+ },
+ // null unless ?counts=1 — see the header.
+ counts: stat,
+ media,
+ report: snapshot
+ ? {
+ generatedAt: snapshot.generatedAt,
+ totals: snapshot.totals,
+ buckets,
+ }
+ : null,
+ });
+}
diff --git a/editor/app/api/ops/download-missing/route.ts b/editor/app/api/ops/download-missing/route.ts
@@ -0,0 +1,21 @@
+import { downloadMissingAction } from "../../../channels/[slug]/pipelineActions";
+import { jobResponse, ops, optBool, optString, reqSlug } from "../_lib";
+
+export const dynamic = "force-dynamic";
+
+// POST { slug, queueKey?, ignoreArchive?, abortOnError? } -> { ok: true, jobId }
+export async function POST(request: Request) {
+ return ops(
+ request,
+ ["slug", "queueKey", "ignoreArchive", "abortOnError"],
+ async (body) =>
+ jobResponse(
+ await downloadMissingAction(
+ reqSlug(body, "slug"),
+ optString(body, "queueKey"),
+ optBool(body, "ignoreArchive"),
+ optBool(body, "abortOnError"),
+ ),
+ ),
+ );
+}
diff --git a/editor/app/api/ops/import-video/route.ts b/editor/app/api/ops/import-video/route.ts
@@ -0,0 +1,17 @@
+import { importVideoAction } from "../../../channels/[slug]/pipelineActions";
+import { jobResponse, ops, optString, reqSlug, reqString } from "../_lib";
+
+export const dynamic = "force-dynamic";
+
+// POST { slug: string, url: string, queueKey?: string } -> { ok: true, jobId }
+export async function POST(request: Request) {
+ return ops(request, ["slug", "url", "queueKey"], async (body) =>
+ jobResponse(
+ await importVideoAction(
+ reqSlug(body, "slug"),
+ reqString(body, "url"),
+ optString(body, "queueKey"),
+ ),
+ ),
+ );
+}
diff --git a/editor/app/api/ops/lane/route.ts b/editor/app/api/ops/lane/route.ts
@@ -0,0 +1,80 @@
+import { NextResponse } from "next/server";
+import { LANES, type AutoQueueKind } from "yt-dlp-transcript-common/lib/autoQueueTypes";
+import { getSettings } from "yt-dlp-transcript-common/lib/settings";
+import { drainAutoRunner } from "yt-dlp-transcript-common/controller/autoRunner";
+import {
+ pauseLaneAction,
+ resumeLaneAction,
+ saveAutoQueueAction,
+ startAutoQueueAction,
+ stopAutoQueueAction,
+} from "../../../operations/actions";
+import { OpsInputError, okResponse, ops, oneOf, optBool } from "../_lib";
+
+export const dynamic = "force-dynamic";
+
+// POST { lane: transcription|download|digest|backfill,
+// held?: boolean, enabled?: boolean, action?: "start"|"stop"|"drain" }
+//
+// One lane, up to three independent changes, applied in the order the operator
+// would: enable the policy, set the gate, then bring the runner up or down.
+// Each is the existing action and nothing more.
+//
+// `held` IS NOT `enabled`. A hold is a dispatch gate (the runner idle-waits and
+// keeps its place); disabling is the policy's master switch. The pair is what
+// /operations draws as two separate controls and this route keeps them separate.
+//
+// `action` mirrors /api/auto-queue/control's body — same three verbs, same
+// meanings — so a caller that already drives that route needs nothing new.
+export async function POST(request: Request) {
+ return ops(request, ["lane", "held", "enabled", "action"], async (body) => {
+ const lane = oneOf(body, "lane", LANES) as AutoQueueKind;
+ const enabled = optBool(body, "enabled");
+ const held = optBool(body, "held");
+ const action = body.action;
+ if (
+ action !== undefined &&
+ action !== "start" &&
+ action !== "stop" &&
+ action !== "drain"
+ ) {
+ throw new OpsInputError('"action" must be "start", "stop" or "drain"');
+ }
+ if (enabled === undefined && held === undefined && action === undefined) {
+ throw new OpsInputError(
+ 'nothing to do — send at least one of "enabled", "held" or "action"',
+ );
+ }
+ if (enabled !== undefined) {
+ // EVERY OTHER POLICY FIELD IS CARRIED FROM THE STORED POLICY, including
+ // `root`: saveAutoQueueAction refuses a tree edit while the priority
+ // model compiles the roots, and handing it back the stored tree is what
+ // makes this a pure enable/disable rather than a tree write.
+ const policy = getSettings().autoQueue[lane];
+ const saved = await saveAutoQueueAction(lane, {
+ enabled,
+ maxWorkers: policy.maxWorkers,
+ replaceAutoSubs: policy.replaceAutoSubs === true,
+ order: policy.order ?? "listed",
+ root: policy.root,
+ });
+ if (!saved.ok) return okResponse(saved);
+ }
+ if (held !== undefined) {
+ const gated = held
+ ? await pauseLaneAction(lane)
+ : await resumeLaneAction(lane);
+ if (!gated.ok) return okResponse(gated);
+ }
+ if (action === "start") {
+ const started = await startAutoQueueAction(lane);
+ if (!started.ok) return okResponse(started);
+ } else if (action === "stop") {
+ const stopped = await stopAutoQueueAction(lane);
+ if (!stopped.ok) return okResponse(stopped);
+ } else if (action === "drain") {
+ drainAutoRunner(lane);
+ }
+ return NextResponse.json({ ok: true, lane });
+ });
+}
diff --git a/editor/app/api/ops/metadata-scan/route.ts b/editor/app/api/ops/metadata-scan/route.ts
@@ -0,0 +1,16 @@
+import { runMetadataScanAction } from "../../../channels/[slug]/pipelineActions";
+import { jobResponse, ops, optString, reqSlug } from "../_lib";
+
+export const dynamic = "force-dynamic";
+
+// POST { slug: string, queueKey?: string } -> { ok: true, jobId }
+export async function POST(request: Request) {
+ return ops(request, ["slug", "queueKey"], async (body) =>
+ jobResponse(
+ await runMetadataScanAction(
+ reqSlug(body, "slug"),
+ optString(body, "queueKey"),
+ ),
+ ),
+ );
+}
diff --git a/editor/app/api/ops/refresh-report/route.ts b/editor/app/api/ops/refresh-report/route.ts
@@ -0,0 +1,43 @@
+import { NextResponse } from "next/server";
+import {
+ refreshAllChannelSnapshotsAction,
+ refreshChannelSnapshotAction,
+} from "../../../channels/actions";
+import {
+ actionResponse,
+ OpsInputError,
+ ops,
+ optBool,
+ optString,
+ reqSlug,
+} from "../_lib";
+
+export const dynamic = "force-dynamic";
+
+// POST { slug: string } | { all: true }
+//
+// The single-channel form REGENERATES SYNCHRONOUSLY (it is a filesystem scan,
+// not a job) and returns `{ ok: true }` once snapshot.json is on disk. The
+// `all` form queues one refresh-report job per channel and returns the bulk
+// { queued, skipped } the /channels header button shows.
+export async function POST(request: Request) {
+ return ops(request, ["slug", "all"], async (body) => {
+ const all = optBool(body, "all");
+ const slug = optString(body, "slug");
+ if (all) {
+ if (slug) {
+ throw new OpsInputError('send either "slug" or "all", not both');
+ }
+ const result = await refreshAllChannelSnapshotsAction();
+ return NextResponse.json({ ok: true, ...result });
+ }
+ if (!slug?.trim()) {
+ throw new OpsInputError('"slug" is required (or send { "all": true })');
+ }
+ // Re-read through reqSlug now that we know it is the single-channel form:
+ // the shape check belongs on the value that reaches a path.join.
+ return actionResponse(
+ await refreshChannelSnapshotAction(reqSlug(body, "slug")),
+ );
+ });
+}
diff --git a/editor/app/api/ops/relocate-back/route.ts b/editor/app/api/ops/relocate-back/route.ts
@@ -0,0 +1,20 @@
+import { moveChannelMediaBackAction } from "../../../channels/[slug]/storageActions";
+import { ops, queueResponse, reqSlugs } from "../_lib";
+import { queueForSlugs } from "../../../channels/lib/queueForSlugs";
+
+export const dynamic = "force-dynamic";
+
+// POST { slugs: string[] } -> { ok: true, queued, skipped }
+//
+// One job per slug on the shared relocation queue, and the per-channel action's
+// own busy guard decides each one — the same fan-out loop the bulk move uses,
+// so a refusal reads identically whether it came from the panel or from here.
+export async function POST(request: Request) {
+ return ops(request, ["slugs"], async (body) =>
+ queueResponse(
+ await queueForSlugs(reqSlugs(body, "slugs"), {
+ run: (slug) => moveChannelMediaBackAction(slug),
+ }),
+ ),
+ );
+}
diff --git a/editor/app/api/ops/relocate/route.ts b/editor/app/api/ops/relocate/route.ts
@@ -0,0 +1,39 @@
+import { bulkRelocateChannelMediaAction } from "../../../channels/bulkStorageActions";
+import {
+ OpsInputError,
+ ops,
+ optString,
+ queueResponse,
+ reqSlugs,
+} from "../_lib";
+
+export const dynamic = "force-dynamic";
+
+// POST { slugs: string[], locationId?: string, root?: string }
+// -> { ok: true, queued, skipped }
+//
+// THE DESTINATION IS A LOCATION ID WHEREVER POSSIBLE — the root is resolved on
+// the server from settings.storage.locations, so a caller holding a stale root
+// cannot aim a batch somewhere a re-point has moved. `root` is the one-off
+// escape hatch the panel also offers. A skip is not a failure: every slug that
+// did not queue comes back with the same sentence the bulk bar shows.
+export async function POST(request: Request) {
+ return ops(request, ["slugs", "locationId", "root"], async (body) => {
+ const slugs = reqSlugs(body, "slugs");
+ const locationId = optString(body, "locationId");
+ const root = optString(body, "root");
+ if ((locationId ? 1 : 0) + (root ? 1 : 0) !== 1) {
+ throw new OpsInputError(
+ 'send exactly one of "locationId" (a location configured on /storage) or "root" (an absolute path)',
+ );
+ }
+ return queueResponse(
+ await bulkRelocateChannelMediaAction(
+ slugs,
+ locationId
+ ? { kind: "location", locationId }
+ : { kind: "custom", root: root as string },
+ ),
+ );
+ });
+}
diff --git a/editor/app/api/ops/retry-bucket/route.ts b/editor/app/api/ops/retry-bucket/route.ts
@@ -0,0 +1,72 @@
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { readChannelSnapshot } from "yt-dlp-transcript-common/controller/channels";
+import { isReplayBucket } from "yt-dlp-transcript-common/jobs/jobSpec";
+import { retryBucketAction } from "../../../channels/[slug]/pipelineActions";
+import {
+ jobResponse,
+ OpsInputError,
+ ops,
+ optBool,
+ optString,
+ reqSlug,
+ reqString,
+} from "../_lib";
+
+export const dynamic = "force-dynamic";
+
+// POST { slug, bucket, queueKey?, abortOnError?, handlingOverride?,
+// forceCookies?, replaceAutoSubs? } -> { ok: true, jobId }
+//
+// THE BUCKET IS RESOLVED FROM THE SNAPSHOT HERE, and that is not new logic: the
+// bucket controls in the UI pass `snapshot.buckets[key]` to the same action, and
+// the action's own argument is a list of ids. An HTTP caller naming a bucket
+// rather than pasting ids is the same gesture — and `bucketKey` travelling with
+// it is what makes the job replayable against the CURRENT bucket, exactly as a
+// clicked one is.
+export async function POST(request: Request) {
+ return ops(
+ request,
+ [
+ "slug",
+ "bucket",
+ "queueKey",
+ "abortOnError",
+ "handlingOverride",
+ "forceCookies",
+ "replaceAutoSubs",
+ ],
+ async (body) => {
+ const slug = reqSlug(body, "slug");
+ const bucket = reqString(body, "bucket");
+ const snapshot = await readChannelSnapshot(getPaths(), slug);
+ if (!snapshot) {
+ throw new OpsInputError(
+ `Channel "${slug}" has no report yet — run /api/ops/refresh-report first.`,
+ );
+ }
+ const ids = (snapshot.buckets as Record<string, unknown>)[bucket];
+ if (!Array.isArray(ids)) {
+ throw new OpsInputError(
+ `"${bucket}" is not a bucket on this channel's report — known buckets: ${Object.keys(
+ snapshot.buckets,
+ ).join(", ")}`,
+ );
+ }
+ return jobResponse(
+ await retryBucketAction(
+ slug,
+ ids as string[],
+ optString(body, "queueKey"),
+ optBool(body, "abortOnError"),
+ optString(body, "handlingOverride"),
+ // Replayable only for the buckets a replay can re-derive; an
+ // ad-hoc bucket name still runs, it just carries no spec — the
+ // same distinction the UI's named vs checkbox controls make.
+ isReplayBucket(bucket) ? bucket : undefined,
+ optBool(body, "forceCookies"),
+ optBool(body, "replaceAutoSubs"),
+ ),
+ );
+ },
+ );
+}
diff --git a/editor/app/api/ops/sync/route.ts b/editor/app/api/ops/sync/route.ts
@@ -0,0 +1,20 @@
+import { syncAction } from "../../../channels/[slug]/pipelineActions";
+import { jobResponse, ops, optBool, optString, reqSlug } from "../_lib";
+
+export const dynamic = "force-dynamic";
+
+// POST { slug: string, full?: boolean, queueKey?: string } -> { ok: true, jobId }
+//
+// `full` forces the periodic whole-listing sweep now; without it the job
+// decides for itself from the configured cadence.
+export async function POST(request: Request) {
+ return ops(request, ["slug", "full", "queueKey"], async (body) =>
+ jobResponse(
+ await syncAction(
+ reqSlug(body, "slug"),
+ optString(body, "queueKey"),
+ optBool(body, "full"),
+ ),
+ ),
+ );
+}
diff --git a/editor/app/channels/[slug]/components/stages/StorageStage.tsx b/editor/app/channels/[slug]/components/stages/StorageStage.tsx
@@ -68,6 +68,13 @@ export type StorageDestination = {
// can never collide with one.
const CUSTOM = "__custom";
+// THE CORPUS VOLUME AS A DESTINATION. Not a location id either (`/storage`
+// never stores one — see INTERNAL_LOCATION_ID in the storage controller), so it
+// cannot collide with a configured one, and picking it runs
+// moveChannelMediaBackAction rather than a relocate.
+const INTERNAL = "__internal";
+const INTERNAL_LABEL = "Internal (in place)";
+
type Props = {
slug: string;
location: ChannelMediaLocation;
@@ -176,20 +183,21 @@ export function StorageStage({
marker={location.marker ?? null}
/>
- <MoveOut
- key="move-out"
+ {/* ONE VERB. "Move media to…" used to be two sections with two buttons —
+ Move media, and Move back in place — which is one decision ("where
+ should this channel's media be") split across two controls whose
+ availability was the inverse of each other. The corpus volume is a
+ destination like any other now; picking it IS moving back, and it
+ runs the same action it always did. */}
+ <MoveMedia
+ key="move-media"
slug={slug}
canMoveOut={!location.relocated}
- blockedReason={blockedReason}
- destinations={destinations}
- defaultLocationId={defaultLocationId}
- />
- <MoveBack
- key="move-back"
- slug={slug}
relocated={location.relocated}
target={location.target}
blockedReason={blockedReason}
+ destinations={destinations}
+ defaultLocationId={defaultLocationId}
// MOVE BACK IS THE ONE DIRECTION THAT ENDS BY DELETING THE TARGET, so
// it is not offered for a location nobody can vouch for. `relocated` is
// true for `inconsistent` and `unreachable` as well as `ok` — config
@@ -211,18 +219,31 @@ export function StorageStage({
);
}
-function MoveOut({
+function MoveMedia({
slug,
canMoveOut,
+ relocated,
+ target,
blockedReason,
destinations,
defaultLocationId,
+ unvouched,
}: {
slug: string;
canMoveOut: boolean;
+ // The channel's media is on a location right now. Then the ONLY destination
+ // offered is the corpus volume: the controller refuses a location-to-location
+ // move by name ("already relocated to X. Move it back in place first."), and
+ // a select full of options that all refuse is worse than a select with one
+ // that works and a sentence saying why.
+ relocated: boolean;
+ target: string | undefined;
blockedReason: string | null;
destinations: StorageDestination[];
defaultLocationId: string;
+ // Why moving back is refused for THIS location, or null. Separate from
+ // blockedReason, which is about jobs and markers.
+ unvouched: string | null;
}) {
// WHICH CONFIGURED LOCATION, or `__custom` for a one-off root. Opens on the
// default location; with none configured there is nothing to pick, so the
@@ -249,21 +270,36 @@ function MoveOut({
const [previewing, setPreviewing] = useState(false);
const [ranHere, setRanHere] = useState(false);
- if (!canMoveOut && !ranHere) return null;
+ if (!canMoveOut && !relocated && !ranHere) return null;
- const chosen = destinations.find((d) => d.id === destId) ?? null;
- const custom = destId === CUSTOM || chosen === null;
+ // DERIVED, never the raw state: a run that just finished flips `relocated`
+ // under this component (the panel stays mounted for its log), and a select
+ // still showing the destination of the move that has already happened would
+ // be offering to do it again.
+ const effectiveId = relocated ? INTERNAL : destId;
+ const movingBack = effectiveId === INTERNAL;
+ const chosen = movingBack
+ ? null
+ : (destinations.find((d) => d.id === effectiveId) ?? null);
+ const custom = !movingBack && (destId === CUSTOM || chosen === null);
const trimmed = customRoot.trim();
// THE ID ALONE GOES TO THE SERVER for a configured location: the root is
// looked up there, from the settings the location list lives in, so the form
// can never send a root a re-point has since moved.
const destination: MoveDestination = custom
? { kind: "custom", root: trimmed }
- : { kind: "location", locationId: chosen.id };
- const key = custom ? `custom:${trimmed}` : `location:${chosen.id}`;
+ : { kind: "location", locationId: chosen?.id ?? "" };
+ const key = custom ? `custom:${trimmed}` : `location:${chosen?.id ?? ""}`;
const named = custom ? trimmed !== "" : true;
- const confirmed = named && key === previewedKey;
- const disabled = !canMoveOut || !confirmed || blockedReason !== null;
+ // NO PREVIEW GATE ON THE WAY BACK, and it is not an oversight: the preview
+ // answers "is there room on the destination", and the controller's move-back
+ // asks the corpus volume that itself, charging only the bytes still missing
+ // from a resumed `data.incoming`. There is no root for the operator to get
+ // wrong either — it is the channel's own directory.
+ const confirmed = movingBack || (named && key === previewedKey);
+ const disabled = movingBack
+ ? !relocated || unvouched !== null || blockedReason !== null
+ : !canMoveOut || !confirmed || blockedReason !== null;
async function runPreview() {
setPreviewing(true);
@@ -285,13 +321,32 @@ function MoveOut({
<div>
<h3 className="text-base font-semibold">Move media to…</h3>
<p className="text-sm text-muted-foreground">
- Copies <code>data/</code> to <code><root>/{slug}/data</code>,
- verifies it, and leaves a symlink behind so every reader, yt-dlp and
- the index keep working unchanged. The source is not touched until the
- copy verifies.
+ {movingBack ? (
+ <>
+ Copies <code>{target ?? "the target"}</code> back into the channel
+ dir, verifies it, replaces the symlink with a real directory and
+ clears the recorded location.
+ </>
+ ) : (
+ <>
+ Copies <code>data/</code> to{" "}
+ <code><root>/{slug}/data</code>, verifies it, and leaves a
+ symlink behind so every reader, yt-dlp and the index keep working
+ unchanged. The source is not touched until the copy verifies.
+ </>
+ )}
</p>
</div>
- {destinations.length > 0 && (
+ {unvouched && (
+ <p
+ role="status"
+ aria-label="move back refused"
+ className="text-sm rounded border border-destructive/50 bg-destructive/5 px-3 py-2"
+ >
+ {unvouched}
+ </p>
+ )}
+ {(destinations.length > 0 || relocated) && (
<label className="flex flex-col gap-1 text-sm">
<span className="font-medium">Destination</span>
<select
@@ -301,8 +356,8 @@ function MoveOut({
// naming an id that is no longer an option, and a <select> whose
// value matches nothing silently shows the first one. `custom` is
// already true in that case — this makes the control agree with it.
- value={custom ? CUSTOM : destId}
- disabled={!canMoveOut}
+ value={movingBack ? INTERNAL : custom ? CUSTOM : destId}
+ disabled={relocated || !canMoveOut}
onChange={(e) => {
setDestId(e.target.value);
// A new destination is a new question. Dropping the preview with
@@ -315,13 +370,29 @@ function MoveOut({
}}
className="rounded border border-border bg-card px-2 py-1 text-sm"
>
- {destinations.map((d) => (
- <option key={d.id} value={d.id}>
- {d.label} — {d.statusLabel}
- </option>
- ))}
- <option value={CUSTOM}>Another root…</option>
+ {/* THE CORPUS VOLUME IS AN OPTION, and while the media is on a
+ location it is the ONLY one — see the prop's comment. */}
+ {relocated ? (
+ <option value={INTERNAL}>{INTERNAL_LABEL}</option>
+ ) : (
+ <>
+ {destinations.map((d) => (
+ <option key={d.id} value={d.id}>
+ {d.label} — {d.statusLabel}
+ </option>
+ ))}
+ <option value={CUSTOM}>Another root…</option>
+ </>
+ )}
</select>
+ {relocated && (
+ <span className="text-xs text-muted-foreground">
+ This channel’s media is on {target ?? "another root"}. Move
+ it back in place first to send it somewhere else — a move from one
+ location straight to another is refused, because the corpus volume
+ is the one place both halves can be verified against.
+ </span>
+ )}
{chosen && (
<span className="text-xs text-muted-foreground font-mono break-all">
{chosen.root}
@@ -342,7 +413,7 @@ function MoveOut({
)}
</label>
)}
- {custom && (
+ {custom && !movingBack && (
<label className="flex flex-col gap-1 text-sm">
<span className="font-medium">Destination root</span>
<input
@@ -366,16 +437,18 @@ function MoveOut({
</span>
</label>
)}
- <div>
- <button
- type="button"
- onClick={runPreview}
- disabled={!canMoveOut || !named || previewing}
- className="px-3 py-1.5 rounded-md border border-border text-sm font-medium disabled:opacity-50"
- >
- {previewing ? "Checking…" : "Preview"}
- </button>
- </div>
+ {!movingBack && (
+ <div>
+ <button
+ type="button"
+ onClick={runPreview}
+ disabled={!canMoveOut || !named || previewing}
+ className="px-3 py-1.5 rounded-md border border-border text-sm font-medium disabled:opacity-50"
+ >
+ {previewing ? "Checking…" : "Preview"}
+ </button>
+ </div>
+ )}
{previewError && (
<p role="alert" className="text-sm text-destructive">
{previewError}
@@ -419,20 +492,26 @@ function MoveOut({
run resumes it rather than starting over.
</p>
)}
- {!confirmed && named && (
+ {!confirmed && named && !movingBack && (
<p className="text-xs text-muted-foreground">
Preview this destination to enable the move.
</p>
)}
+ {/* ONE LOG, ONE ACCESSIBLE NAME, both directions. `moveChannelMediaBackAction`
+ is untouched — it is still the action, still by that name, and the API
+ routes another branch is adding call it unchanged. What moved is which
+ control reaches it. */}
<StreamActionLog
- key="move-out-log"
+ key="move-media-log"
trigger={() => {
setRanHere(true);
- return relocateChannelMediaAction(slug, destination);
+ return movingBack
+ ? moveChannelMediaBackAction(slug)
+ : relocateChannelMediaAction(slug, destination);
}}
cancelAction={cancelJobAction}
- buttonLabel="Move media"
- runningLabel="Moving media…"
+ buttonLabel={movingBack ? "Move back in place" : "Move media"}
+ runningLabel={movingBack ? "Moving back…" : "Moving media…"}
label="Move media"
disabled={disabled}
/>
@@ -440,58 +519,6 @@ function MoveOut({
);
}
-function MoveBack({
- slug,
- relocated,
- target,
- blockedReason,
- unvouched,
-}: {
- slug: string;
- relocated: boolean;
- target: string | undefined;
- blockedReason: string | null;
- // Why moving back is refused for THIS location, or null. Separate from
- // blockedReason, which is about jobs and markers and disables both halves.
- unvouched: string | null;
-}) {
- const [ranHere, setRanHere] = useState(false);
- if (!relocated && !ranHere) return null;
- return (
- <section className="flex flex-col gap-2">
- <div>
- <h3 className="text-base font-semibold">Move back in place</h3>
- <p className="text-sm text-muted-foreground">
- {relocated
- ? `Copies ${target ?? "the target"} back into the channel dir, verifies it, replaces the symlink with a real directory and clears the recorded location.`
- : "This channel's media is in place."}
- </p>
- </div>
- {unvouched && (
- <p
- role="status"
- aria-label="move back refused"
- className="text-sm rounded border border-destructive/50 bg-destructive/5 px-3 py-2"
- >
- {unvouched}
- </p>
- )}
- <StreamActionLog
- key="move-back-log"
- trigger={() => {
- setRanHere(true);
- return moveChannelMediaBackAction(slug);
- }}
- cancelAction={cancelJobAction}
- buttonLabel="Move back in place"
- runningLabel="Moving back…"
- label="Move back in place"
- disabled={!relocated || blockedReason !== null || unvouched !== null}
- />
- </section>
- );
-}
-
// THE TWO WAYS OUT OF A MARKER WHOSE RUN IS GONE — finish it, or throw it away.
//
// Both are offered ONLY when the channel is in-transition AND no job is running:
diff --git a/editor/app/channels/actions.ts b/editor/app/channels/actions.ts
@@ -48,6 +48,7 @@ import {
effectiveTier,
hasCompiledLaneRoots,
isChannelPaused,
+ clearAutoPause,
isDefaultChannelPriority,
rankOf,
renameChannelInPriority,
@@ -756,7 +757,11 @@ function applyPriorityEdit(
? { ...entry, rank: entry.rank + 1 }
: { ...entry };
}
- channels[slug] = { ...entryFor(model, slug), tier: "normal", rank };
+ channels[slug] = clearAutoPause({
+ ...entryFor(model, slug),
+ tier: "normal",
+ rank,
+ });
return { ...model, channels };
}
const channels: Record<string, ChannelPriorityEntry> = { ...model.channels };
@@ -764,7 +769,14 @@ function applyPriorityEdit(
const slug = raw.trim();
if (!slug) continue;
if (edit.kind === "tier") {
- channels[slug] = { ...entryFor(model, slug), tier: edit.tier };
+ // THE OPERATOR'S WORD WINS over the machine's. Setting a tier by hand
+ // clears any auto-pause record, so a drive coming back later cannot
+ // un-pause a channel a person paused in the meantime — nor re-pause one
+ // they deliberately resumed while the drive was still away.
+ channels[slug] = clearAutoPause({
+ ...entryFor(model, slug),
+ tier: edit.tier,
+ });
continue;
}
if (edit.kind === "operation") {
@@ -782,11 +794,11 @@ function applyPriorityEdit(
// "sync-only": paused everywhere, normal for sync. Its rank survives —
// the sync scheduler still orders it.
const entry = entryFor(model, slug);
- channels[slug] = {
+ channels[slug] = clearAutoPause({
...entry,
tier: "paused",
overrides: { sync: "normal" },
- };
+ });
}
return { ...model, channels };
}
diff --git a/editor/app/channels/components/ChannelSelectionDeck.tsx b/editor/app/channels/components/ChannelSelectionDeck.tsx
@@ -75,6 +75,7 @@ export function ChannelSelectionDeck({
onClear,
destinations,
defaultLocationId,
+ freeUpNote = null,
}: {
slugs: string[];
onClear: () => void;
@@ -84,6 +85,11 @@ export function ChannelSelectionDeck({
destinations: BulkDestination[];
// Which one the select opens on — `settings.storage.defaultLocationId`.
defaultLocationId: string;
+ // What "Free up N GB" came to, when that is how this selection was made.
+ // The CONTROL is in the volume bar (it has to work with nothing ticked, and
+ // the deck does not exist then); what belongs here is the answer, beside the
+ // Move button that will act on it.
+ freeUpNote?: string | null;
}) {
// Two runners, because they are two writers: the priority actions clear the
// selection on success, the relocate queues jobs and keeps it.
@@ -251,6 +257,14 @@ export function ChannelSelectionDeck({
</button>
</div>
+ {freeUpNote && (
+ <p
+ aria-label="free up selection note"
+ className="pt-1.5 text-xs text-muted-foreground"
+ >
+ {freeUpNote}
+ </p>
+ )}
{(result || moveError || priorityError) && (
<div className="flex flex-wrap items-center gap-x-4 gap-y-1 pt-1.5 text-xs">
{result && (
diff --git a/editor/app/channels/components/ChannelTierSelect.tsx b/editor/app/channels/components/ChannelTierSelect.tsx
@@ -51,6 +51,11 @@ export type ChannelTierSelectProps = {
focused?: boolean;
// "Held — focus: <name>" for a non-focus row while a focus holds the lanes.
heldReason?: string | null;
+ // Why the machine paused this channel, or null. Distinct from `heldReason`,
+ // which is the focus holding a channel that is otherwise running: this one
+ // says the tier ITSELF was set by something other than the operator, and
+ // will be set back.
+ autoPausedReason?: string | null;
disabled?: boolean;
};
@@ -79,6 +84,7 @@ export default function ChannelTierSelect({
overrides = {},
focused = false,
heldReason = null,
+ autoPausedReason = null,
disabled = false,
}: ChannelTierSelectProps): React.ReactNode {
const [pending, startTransition] = useTransition();
@@ -165,6 +171,21 @@ export default function ChannelTierSelect({
<span className="sr-only"> — {heldReason}</span>
</span>
)}
+ {/* WHY IT IS PAUSED, when it was not the operator who paused it. Without
+ this the rack shows a Paused channel and no way to tell a deliberate
+ pause from a drive that fell off a USB cable — which is exactly the
+ "automatic disable AND FLAG" the operator asked for. */}
+ {autoPausedReason && (
+ <span
+ role="note"
+ aria-label={`auto-paused reason for ${slug}`}
+ title={autoPausedReason}
+ className="rounded-full border border-destructive/30 bg-destructive/5 px-1.5 text-[10px] uppercase tracking-wide text-destructive"
+ >
+ storage
+ <span className="sr-only"> — {autoPausedReason}</span>
+ </span>
+ )}
{error && (
<span
role="alert"
diff --git a/editor/app/channels/components/ChannelVolumeBar.tsx b/editor/app/channels/components/ChannelVolumeBar.tsx
@@ -0,0 +1,186 @@
+"use client";
+
+import Link from "next/link";
+import { useState } from "react";
+import { usePathname, useSearchParams } from "next/navigation";
+import { formatBytes } from "yt-dlp-transcript-common/lib/format";
+// One spelling of the param, shared with the /storage row that writes the link.
+// A value import from views/, which is client-safe: every import in that module
+// is a type import.
+import { LOCATION_FILTER_PARAM } from "yt-dlp-transcript-common/views/storage";
+
+// THE VOLUME BRIDGE — how much room each disk has, said ONCE, and the filter.
+//
+// The operator's ask, verbatim (2026-09-20): "I'd like to see disk space and
+// current storage volume as columns on the channels menu, and let me filter by
+// volume for easy checking of things that may need moving or other processing."
+//
+// The volume of a ROW is a column (ChannelsTable's Location cell). Free space
+// is NOT: it is a fact about a disk, not about a channel, and repeating "67 GB
+// free" down 71 rows would be 71 copies of one number — the same mistake the
+// per-row focus sentence made before the focus bar took it. So it lives here,
+// one chip per volume, above the rack.
+//
+// THE CHIP IS THE FILTER, and it is a LINK. `?location=` is a URL param beside
+// `?site=` for the reason that one is: the page is a server component, the
+// filter changes what the server sends, and a link is shareable — /storage
+// links straight to a filtered list. Client state would also lose the race with
+// the global AutoRefresh's router.refresh(), which is why the sort is the one
+// thing here that stays local.
+
+export type ChannelVolume = {
+ // A location id, "internal" for the corpus volume, or "" for a dataDir under
+ // a root nobody named.
+ id: string;
+ label: string;
+ channels: number;
+ // Media bytes on this volume across the channels on screen, and how many of
+ // them had no figure in their report. The second number is why the first is
+ // honest — see storageBytesText.
+ bytes: number;
+ unmeasured: number;
+ // statfs of the root, or undefined when the root is not there (an unmounted
+ // drive). Never inferred from a parent — see volumeFreeBytes.
+ freeBytes?: number;
+};
+
+export function ChannelVolumeBar({
+ volumes,
+ active,
+ onFreeUp,
+}: {
+ volumes: ChannelVolume[];
+ // The id currently filtered to, or null for "every volume".
+ active: string | null;
+ // "Free up N GB": tick the largest in-place channels until the target is met.
+ // IT LIVES HERE AND NOT ON THE SELECTION DECK because the deck only exists
+ // once something is ticked, and this is the control whose whole job is to do
+ // the ticking. The deck shows what it came to, beside the Move button that
+ // acts on it.
+ onFreeUp?: (targetGB: number) => void;
+}) {
+ const pathname = usePathname();
+ const params = useSearchParams();
+ // THE SITE SCOPE SURVIVES THE VOLUME FILTER. They are two independent
+ // questions ("whose channels" and "which disk") and a chip that silently
+ // dropped ?site= would answer one by discarding the other.
+ const href = (id: string | null): string => {
+ const next = new URLSearchParams(params.toString());
+ if (id === null) next.delete(LOCATION_FILTER_PARAM);
+ else next.set(LOCATION_FILTER_PARAM, id);
+ // A new filter is a new list; the sort /storage asked for belongs to the
+ // link that carried it.
+ next.delete("sort");
+ const q = next.toString();
+ return q ? `${pathname}?${q}` : pathname;
+ };
+
+ if (volumes.length <= 1 && active === null && !onFreeUp) return null;
+
+ return (
+ <div
+ aria-label="storage volumes"
+ className="flex flex-wrap items-center gap-2 px-1 text-xs"
+ >
+ <span className="font-mono text-[10px] uppercase tracking-[0.16em] text-muted-foreground">
+ Volumes
+ </span>
+ <Chip href={href(null)} active={active === null} label="All" />
+ {volumes.map((v) => (
+ <Chip
+ key={v.id || "elsewhere"}
+ href={href(v.id)}
+ active={active === v.id}
+ label={v.label}
+ ariaLabel={`volume ${v.id || "elsewhere"}`}
+ detail={
+ `${v.channels} ch · ${formatBytes(v.bytes)}` +
+ (v.unmeasured > 0 ? ` +${v.unmeasured}?` : "") +
+ ` · ${v.freeBytes === undefined ? "free —" : `${formatBytes(v.freeBytes)} free`}`
+ }
+ title={
+ v.freeBytes === undefined
+ ? `${v.label}: the root is not there — an unmounted drive reports no free space rather than its parent's.`
+ : `${v.label}: ${v.channels} channel(s) hold ${formatBytes(v.bytes)}${
+ v.unmeasured > 0
+ ? `, plus ${v.unmeasured} with no size in their report yet`
+ : ""
+ }. ${formatBytes(v.freeBytes)} free.`
+ }
+ />
+ ))}
+ {onFreeUp && <FreeUpControl onFreeUp={onFreeUp} />}
+ </div>
+ );
+}
+
+// THE ARITHMETIC NOBODY SHOULD BE DOING BY HAND. Sort by size, add the figures
+// up, tick rows until the sum clears the target — on a disk that is 96 % full,
+// at whatever hour it got that way. The rule is in
+// common/views/freeUpSelection.ts; this is a number box and a button.
+function FreeUpControl({ onFreeUp }: { onFreeUp: (targetGB: number) => void }) {
+ const [target, setTarget] = useState("100");
+ const n = Number.parseFloat(target);
+ const valid = Number.isFinite(n) && n > 0;
+ return (
+ <span className="ml-auto inline-flex items-center gap-1.5">
+ <span className="font-mono text-[10px] uppercase tracking-[0.16em] text-muted-foreground">
+ Free up
+ </span>
+ <input
+ type="number"
+ min={1}
+ step={10}
+ aria-label="free up target GB"
+ value={target}
+ onChange={(e) => setTarget(e.target.value)}
+ className="w-20 rounded-md border border-border bg-card px-2 py-0.5 text-xs tabular-nums"
+ />
+ <span className="text-muted-foreground">GB</span>
+ <button
+ type="button"
+ disabled={!valid}
+ aria-label="select largest channels to free up"
+ title="Ticks the largest channels still on the corpus volume until the target is met, largest first. It selects only — moving them is the deck's Move button. Channels already on another volume free nothing here and are never picked; ones with no size in their report are skipped and counted."
+ onClick={() => onFreeUp(n)}
+ className="rounded-md border border-border px-2 py-0.5 text-xs hover:bg-muted disabled:opacity-50"
+ >
+ Select largest
+ </button>
+ </span>
+ );
+}
+
+function Chip({
+ href,
+ active,
+ label,
+ detail,
+ ariaLabel,
+ title,
+}: {
+ href: string;
+ active: boolean;
+ label: string;
+ detail?: string;
+ ariaLabel?: string;
+ title?: string;
+}) {
+ return (
+ <Link
+ href={href}
+ aria-label={ariaLabel}
+ aria-current={active ? "true" : undefined}
+ title={title}
+ className={
+ "inline-flex items-baseline gap-1.5 rounded-full border px-2.5 py-0.5 " +
+ (active
+ ? "border-primary bg-primary/10 text-foreground"
+ : "border-border text-muted-foreground hover:text-foreground")
+ }
+ >
+ <span className="font-medium">{label}</span>
+ {detail && <span className="tabular-nums text-[11px]">{detail}</span>}
+ </Link>
+ );
+}
diff --git a/editor/app/channels/components/ChannelsTable.tsx b/editor/app/channels/components/ChannelsTable.tsx
@@ -24,6 +24,14 @@ import {
ChannelSelectionDeck,
type BulkDestination,
} from "./ChannelSelectionDeck";
+import { ChannelVolumeBar, type ChannelVolume } from "./ChannelVolumeBar";
+import { formatBytes } from "yt-dlp-transcript-common/lib/format";
+import { selectToFreeBytes } from "yt-dlp-transcript-common/views/freeUpSelection";
+// A VALUE import from views/, and it is safe: every import in that module is a
+// type import, so what reaches the client bundle is two string constants. The
+// point is that "internal" and "location" have ONE spelling across /storage
+// (which writes the link) and /channels (which reads it).
+import { INTERNAL_ROW_ID } from "yt-dlp-transcript-common/views/storage";
import {
tierOrder,
type PriorityOperation,
@@ -39,6 +47,8 @@ export type ChannelRowPriority = {
overrides: Partial<Record<PriorityOperation, StoredChannelTier>>;
focused: boolean;
heldReason: string | null;
+ // Why the machine paused it, or null. See autoPauseReasonOf.
+ autoPausedReason: string | null;
};
// A row is a stat plus its pipeline bands, in column order. The bands are
@@ -61,6 +71,16 @@ export type ChannelRow = ChannelStat & {
// badge column is empty for them and the two that matter stand out. See
// components/MediaLocationBadge.tsx.
media: MediaBadgeInput | null;
+ // WHICH VOLUME, as an id the filter can name: a location id, "internal" for
+ // the corpus volume, or "" for a dataDir under a root nobody named. Derived
+ // on the server from `config.dataDir` — a pure prefix match, never a probe.
+ volumeId: string;
+ volumeLabel: string;
+ // `snapshot.totalMediaBytes`. NULL, not 0, for a report written before the
+ // field existed: the Size cell renders that as "—" and the sort puts it
+ // first on ascending, because a 400 GB channel that has not been measured
+ // must not sort as the smallest thing on the disk.
+ mediaBytes: number | null;
};
// A column heading for one pipeline. Comes off the operation registry on the
@@ -93,6 +113,8 @@ type SortKey =
| "playlist"
| "lastSync"
| "report"
+ | "location"
+ | "size"
| `op:${string}`;
type SortDir = "asc" | "desc";
@@ -110,6 +132,12 @@ const DEFAULT_DIR: Record<string, SortDir> = {
// Ascending, and missing dates sort as oldest: the first click puts the
// channels whose numbers cannot be trusted at the top.
report: "asc",
+ location: "asc",
+ // THE ONLY REASON TO SORT BY SIZE is to find what is worth moving, so the
+ // first click is biggest-first. (Descending is the record's default anyway;
+ // it is stated here because this column is the page's whole storage story and
+ // an implicit default is a thing to get wrong later.)
+ size: "desc",
};
// A pipeline column defaults to `reachable` descending: the first click puts the
@@ -205,6 +233,19 @@ function cmp(a: ChannelRow, b: ChannelRow, key: SortKey): number {
a.report.generatedAt ?? undefined,
b.report.generatedAt ?? undefined,
);
+ case "location":
+ // By LABEL, then slug: the operator reads names, and a stable tiebreak
+ // keeps the two halves of a volume from shuffling between renders.
+ return (
+ a.volumeLabel.localeCompare(b.volumeLabel) ||
+ a.slug.localeCompare(b.slug)
+ );
+ case "size":
+ // compareNumbers sorts a missing value FIRST, which on descending (the
+ // default here) puts the unmeasured channels last — behind everything
+ // whose size is known, which is where a row that cannot be ranked
+ // belongs.
+ return compareNumbers(a.mediaBytes, b.mediaBytes);
default:
return 0;
}
@@ -230,6 +271,9 @@ export function ChannelsTable({
defaultLocationId = "",
sites = [],
focusLabel = null,
+ volumes = [],
+ locationFilter = null,
+ initialSort = null,
}: {
channels: ChannelRow[];
// Which pipelines to draw, in group order, resolved on the server from the
@@ -254,8 +298,19 @@ export function ChannelsTable({
sites?: FocusSite[];
// What the focus selector currently names, resolved on the server, or null.
focusLabel?: string | null;
+ // One entry per storage volume the channels on screen live on, with its free
+ // space. Rendered ONCE above the rack, never per row — see ChannelVolumeBar.
+ volumes?: ChannelVolume[];
+ // `?location=`, already validated by the server against what is on screen.
+ locationFilter?: string | null;
+ // The sort the URL asked for (`?sort=size`, which is how /storage links to a
+ // "largest first" list). Only the initial value: sorting is client state
+ // from then on, because a router.replace races the global AutoRefresh.
+ initialSort?: "size" | null;
}) {
- const [sort, setSort] = useState<SortState>(null);
+ const [sort, setSort] = useState<SortState>(
+ initialSort ? { key: initialSort, dir: defaultDirFor(initialSort) } : null,
+ );
// Slugs ticked for a bulk edit. A Set of SLUGS, not indices, so a
// re-render that reorders or drops a row cannot retarget the selection —
// the same reason SyncConsole keys its selection this way.
@@ -274,8 +329,9 @@ export function ChannelsTable({
[channels],
);
const showSections = grouped && !!sections && sections.length > 0 && !!siteId;
- // The select column, eight fixed columns, one per pipeline, then Actions.
- const colSpan = 10 + columns.length;
+ // The select column, ten fixed columns (Location and Size joined the eight),
+ // one per pipeline, then Actions.
+ const colSpan = 12 + columns.length;
// Always the intersection with what is on screen: a slug can leave the table
// between renders (a scope change, a deletion), and a bulk edit must not act
// on a row nobody can see.
@@ -285,6 +341,29 @@ export function ChannelsTable({
const allSelected =
channels.length > 0 && selectedSlugs.length === channels.length;
+ // "FREE UP N GB" — the arithmetic the operator was doing by hand.
+ //
+ // The rule lives in common/views/freeUpSelection.ts (pure, unit-tested); this
+ // is the wire between it and the tick boxes. It SELECTS and does not act:
+ // what moves anything is the deck's existing Move button, with its existing
+ // destination and its existing per-channel skips.
+ //
+ // Always over `channels` — what is on screen — so a volume filter or a site
+ // scope narrows the proposal exactly as the operator expects.
+ const [freeUpNote, setFreeUpNote] = useState<string | null>(null);
+ function freeUp(targetGB: number) {
+ const result = selectToFreeBytes(
+ channels.map((c) => ({
+ slug: c.slug,
+ bytes: c.mediaBytes,
+ inPlace: c.volumeId === INTERNAL_ROW_ID,
+ })),
+ targetGB * 1024 ** 3,
+ );
+ setSelected(new Set(result.slugs));
+ setFreeUpNote(`${formatBytes(result.bytes)} selected. ${result.note}`);
+ }
+
function toggleOne(slug: string) {
setSelected((prev) => {
const next = new Set(prev);
@@ -338,6 +417,11 @@ export function ChannelsTable({
focusLabel={focusLabel}
heldCount={heldCount}
/>
+ <ChannelVolumeBar
+ volumes={volumes}
+ active={locationFilter}
+ onFreeUp={freeUp}
+ />
{/* THE INSTRUMENT BAR: what the rack is showing (grouping) on the left,
how to read it (the band legend, and the two-route explainer behind a
disclosure) on the right. Both used to live in an 11px stack UNDER 67
@@ -453,6 +537,28 @@ export function ChannelsTable({
className="whitespace-nowrap"
title="When this channel's report was last generated. Every count and band on the row is read from it — stale or missing means those numbers may be wrong."
/>
+ {/* THE TWO STORAGE COLUMNS. Which disk the media is on, and how
+ much of it there is — the pair that makes "what should I move"
+ answerable without opening 71 channel pages. Free space is
+ deliberately NOT here: it is a fact about a volume, not about
+ a row, and it is stated once in the volume bar above. */}
+ <SortableTh
+ label="Location"
+ sortKey="location"
+ sort={sort}
+ onClick={onHeaderClick}
+ className="whitespace-nowrap"
+ title="Which storage volume this channel's media is on. Internal is the corpus disk; anything else is a location configured on /storage."
+ />
+ <SortableTh
+ label="Size"
+ sortKey="size"
+ sort={sort}
+ onClick={onHeaderClick}
+ align="right"
+ className="whitespace-nowrap"
+ title="Every byte under this channel's data/ — audio, transcripts, cues, metadata — from its last report. Sorts biggest first."
+ />
{/* THE METER BRIDGE. Six loose grey dashes become one block: the
band columns share an eyebrow naming them, and the rules that
open and close the block run the full height of the rack. */}
@@ -524,9 +630,18 @@ export function ChannelsTable({
the old bar off the bottom of a phone. */}
<ChannelSelectionDeck
slugs={selectedSlugs}
- onClear={() => setSelected(new Set())}
+ onClear={() => {
+ setSelected(new Set());
+ setFreeUpNote(null);
+ }}
destinations={mediaDestinations}
defaultLocationId={defaultLocationId}
+ // THE NOTE, NOT THE CONTROL. The helper itself lives in the volume bar
+ // (it is about volumes and free space, and it has to be reachable with
+ // nothing ticked — the deck does not exist then). What belongs down
+ // here is what the proposal actually came to, beside the Move button
+ // that will act on it.
+ freeUpNote={freeUpNote}
/>
</div>
);
@@ -663,6 +778,7 @@ function ChannelTableRow({
overrides={c.priority.overrides}
focused={c.priority.focused}
heldReason={c.priority.heldReason}
+ autoPausedReason={c.priority.autoPausedReason}
/>
</Td>
<Td
@@ -688,6 +804,32 @@ function ChannelTableRow({
? formatStamp(c.report.generatedAt)
: c.report.state}
</Td>
+ <Td
+ ariaLabel={`media location for ${c.slug}`}
+ className={`whitespace-nowrap text-xs${dim}`}
+ title={c.media?.target ?? undefined}
+ >
+ {/* THE LABEL ONLY — NO SECOND BADGE. The unreachable / in-transition
+ marking is the badge's, and the badge is already on this row, in the
+ Slug cell beside the channel's name; drawing another here put two
+ elements with the same accessible name on every relocated row, which
+ is a strict-mode violation for the suite and a screen reader reading
+ the same sentence twice for a human. The COLUMN answers "which
+ disk"; the badge answers "can it be reached", and one of each per
+ row is the right number. */}
+ {c.volumeLabel}
+ </Td>
+ <Td
+ ariaLabel={`media size for ${c.slug}`}
+ className={`whitespace-nowrap text-right tabular-nums${dim}`}
+ title={
+ c.mediaBytes === null
+ ? "No size in this channel's report yet — refresh it for a figure."
+ : undefined
+ }
+ >
+ {c.mediaBytes === null ? "—" : formatBytes(c.mediaBytes)}
+ </Td>
{columns.map((col, i) => (
<PipelineCell
key={col.id}
diff --git a/editor/app/channels/components/channelConfigToForm.ts b/editor/app/channels/components/channelConfigToForm.ts
@@ -0,0 +1,186 @@
+import type { ChannelConfig } from "yt-dlp-transcript-common/lib/channelConfig";
+
+// THE INVERSE OF parseChannelForm, and it exists for exactly one caller:
+// /api/ops/channel-config.
+//
+// WHY A ROUND TRIP THROUGH FormData rather than a direct config write. The form
+// parser is where a download-filter regex is refused, where a sync cadence is
+// bounded and where a cleared field is turned into an absent key — and
+// `updateChannelAction` DELETES every CHANNEL_FORM_FIELDS key from the stored
+// config before layering the parse result on top, so that a cleared input
+// actually clears. Feed it a FormData carrying only a patch and it would clear
+// everything the patch did not name. So the patch is applied ON TOP of the
+// channel's current form representation, which is what this builds, and the
+// action then behaves EXACTLY as it does for the browser form. One validator,
+// one writer, one set of error sentences.
+//
+// Every key below is a field name `parseChannelForm` reads. Checkbox fields
+// follow the browser: present means checked, absent means unchecked — which is
+// why they are emitted only when true.
+
+// The checkbox-shaped fields (presence, not value).
+export const CHANNEL_FORM_FLAGS = [
+ "keepSourceVideo",
+ "downloadFilterIncludeLivestreams",
+ "audioCheckEnabled",
+ "audioCheckResumeDuringProbe",
+] as const;
+
+// The value-shaped fields. A patch may set any of these to "" to clear it.
+export const CHANNEL_FORM_VALUES = [
+ "name",
+ "handling",
+ "url",
+ "platform",
+ "sourceKind",
+ "postFetcher",
+ "socialHandle",
+ "audioFormat",
+ "downloadFormat",
+ "keepLatest",
+ "extractionMode",
+ "savedVideosDir",
+ "ytdlpExtraArgs",
+ "sleepBetweenDownloadsSeconds",
+ "syncIntervalMinutes",
+ "fullSweepIntervalMinutes",
+ "cookiesFromBrowser",
+ "cookieMode",
+ "downloadFilterInclude",
+ "downloadFilterExclude",
+ "audioCheckIntervalSeconds",
+ "audioCheckMaxRollbacks",
+ "audioCheckCopyTimeoutSeconds",
+] as const;
+
+export type ChannelFormFlag = (typeof CHANNEL_FORM_FLAGS)[number];
+export type ChannelFormValue = (typeof CHANNEL_FORM_VALUES)[number];
+export type ChannelFormField = ChannelFormFlag | ChannelFormValue;
+
+export const CHANNEL_FORM_FIELD_NAMES: readonly ChannelFormField[] = [
+ ...CHANNEL_FORM_VALUES,
+ ...CHANNEL_FORM_FLAGS,
+];
+
+function put(fd: FormData, key: string, value: string | undefined | null): void {
+ if (value === undefined || value === null || value === "") return;
+ fd.set(key, value);
+}
+
+function putNum(fd: FormData, key: string, value: number | undefined): void {
+ if (value === undefined || value === null) return;
+ fd.set(key, String(value));
+}
+
+// The channel's stored config as the Configure form would post it.
+export function channelConfigToFormData(config: ChannelConfig): FormData {
+ const fd = new FormData();
+ // Both are REQUIRED by parseChannelForm and are not CHANNEL_FORM_FIELDS, so
+ // they always travel.
+ fd.set("name", config.name ?? "");
+ fd.set("handling", config.handling ?? "transcribe");
+ put(fd, "url", config.url);
+ put(fd, "platform", config.platform);
+ put(fd, "sourceKind", config.sourceKind);
+ put(fd, "postFetcher", config.postFetcher);
+ put(fd, "socialHandle", config.socialHandle);
+ put(fd, "audioFormat", config.audioFormat);
+ put(fd, "downloadFormat", config.downloadFormat);
+ if (config.keepSourceVideo) fd.set("keepSourceVideo", "on");
+ putNum(fd, "keepLatest", config.keepLatest);
+ put(fd, "extractionMode", config.extractionMode);
+ put(fd, "savedVideosDir", config.savedVideosDir);
+ if (config.ytdlpExtraArgs?.length) {
+ fd.set("ytdlpExtraArgs", config.ytdlpExtraArgs.join("\n"));
+ }
+ putNum(fd, "sleepBetweenDownloadsSeconds", config.sleepBetweenDownloadsSeconds);
+ putNum(fd, "syncIntervalMinutes", config.syncIntervalMinutes);
+ putNum(fd, "fullSweepIntervalMinutes", config.fullSweepIntervalMinutes);
+ put(fd, "cookiesFromBrowser", config.cookiesFromBrowser);
+ put(fd, "cookieMode", config.cookieMode);
+ put(fd, "downloadFilterInclude", config.downloadFilter?.include);
+ put(fd, "downloadFilterExclude", config.downloadFilter?.exclude);
+ if (config.downloadFilter?.includeLivestreams) {
+ fd.set("downloadFilterIncludeLivestreams", "on");
+ }
+ if (config.audioCheck?.enabled) {
+ fd.set("audioCheckEnabled", "on");
+ putNum(fd, "audioCheckIntervalSeconds", config.audioCheck.intervalSeconds);
+ putNum(fd, "audioCheckMaxRollbacks", config.audioCheck.maxRollbacks);
+ putNum(
+ fd,
+ "audioCheckCopyTimeoutSeconds",
+ config.audioCheck.copyTimeoutSeconds,
+ );
+ if (config.audioCheck.resumeDuringProbe) {
+ fd.set("audioCheckResumeDuringProbe", "on");
+ }
+ }
+ return fd;
+}
+
+export type ChannelConfigPatch = Record<string, unknown>;
+
+// Lay a JSON patch over a FormData built above. Values: a string (or number,
+// stringified) sets the field, `""` clears it; a flag takes a boolean.
+// `ytdlpExtraArgs` additionally accepts an array of lines, because that is the
+// shape it has in config.json and a caller should not have to know the textarea
+// is newline-separated.
+//
+// Throws on a key this form has no field for — the allow-list IS the documented
+// body shape, and a misspelled key that silently did nothing would look like a
+// successful save.
+export function applyChannelFormPatch(
+ fd: FormData,
+ patch: ChannelConfigPatch,
+): void {
+ validateChannelFormPatch(patch);
+ for (const [key, value] of Object.entries(patch)) {
+ if ((CHANNEL_FORM_FLAGS as readonly string[]).includes(key)) {
+ if (value) fd.set(key, "on");
+ else fd.delete(key);
+ continue;
+ }
+ if (key === "ytdlpExtraArgs" && Array.isArray(value)) {
+ const joined = (value as string[]).join("\n");
+ if (joined.trim()) fd.set(key, joined);
+ else fd.delete(key);
+ continue;
+ }
+ if (value === null || value === "") {
+ fd.delete(key);
+ continue;
+ }
+ fd.set(key, typeof value === "number" ? String(value) : (value as string));
+ }
+}
+
+// SHAPE ONLY, AND SEPARATE ON PURPOSE: a misspelled field name is a fact about
+// the REQUEST, so the route checks it before it looks the channel up. Otherwise
+// `{ slug: "typo", patch: { downloadFilterExcluded: … } }` reports the channel
+// and never mentions the key that was actually wrong.
+export function validateChannelFormPatch(patch: ChannelConfigPatch): void {
+ for (const [key, value] of Object.entries(patch)) {
+ if ((CHANNEL_FORM_FLAGS as readonly string[]).includes(key)) {
+ if (typeof value !== "boolean") {
+ throw new Error(`"${key}" must be a boolean`);
+ }
+ continue;
+ }
+ if (!(CHANNEL_FORM_VALUES as readonly string[]).includes(key)) {
+ throw new Error(
+ `"${key}" is not a channel config field — accepted: ${CHANNEL_FORM_FIELD_NAMES.join(", ")}`,
+ );
+ }
+ if (key === "ytdlpExtraArgs" && Array.isArray(value)) {
+ if (value.some((v) => typeof v !== "string")) {
+ throw new Error(`"${key}" must be an array of strings`);
+ }
+ continue;
+ }
+ if (value === null || typeof value === "number" || typeof value === "string") {
+ continue;
+ }
+ throw new Error(`"${key}" must be a string, a number, or "" to clear it`);
+ }
+}
diff --git a/editor/app/channels/lib/relocationJob.ts b/editor/app/channels/lib/relocationJob.ts
@@ -44,14 +44,51 @@ export async function enqueueRelocation(opts: {
queueKey: relocationQueueKey(),
paths,
channelSlug: slug,
- fn: async (onLog, signal) => {
+ // THE COPY IS ONE TASK, and it is the only one this job has. rsync already
+ // prints its progress (`--info=progress2`); until now that went nowhere but
+ // the log, as thousands of carriage-return redraws of one line, so a move
+ // of 131 GB looked from /jobs exactly like a move of 3 MB — a spinner. The
+ // controller parses those frames (see makeProgressSink) and hands them back
+ // as `RelocationProgress`; this turns them into the task bar every other
+ // long job on that page already draws.
+ //
+ // ADDED ON THE FIRST FRAME, NOT AT START. A move spends its first seconds in
+ // preflight (space, writability, movable state) and, on a resume, in a
+ // verify pass that transfers nothing — a task bar sitting at 0 % through
+ // that would be claiming a copy had begun.
+ fn: async (onLog, signal, _setProgress, ctx) => {
+ const taskId = `relocate:${slug}`;
+ let started = false;
const result = await relocateChannelMedia({
paths,
slug,
direction,
root,
onLog,
+ onProgress: (p) => {
+ if (!started) {
+ started = true;
+ ctx?.addTask({
+ id: taskId,
+ label:
+ direction === "out"
+ ? `${slug} → ${root ?? "destination"}`
+ : `${slug} → in place`,
+ kind: "relocate",
+ startedAt: Date.now(),
+ });
+ }
+ ctx?.updateTask(taskId, {
+ fraction: p.fraction,
+ detail: p.detail,
+ });
+ },
signal,
+ }).finally(() => {
+ // In a `finally` so a failed or cancelled move does not leave a task on
+ // the record for the life of the process — the same contract
+ // taskHooks.ts's `end()` has.
+ if (started) ctx?.removeTask(taskId);
});
onLog(
`${direction === "out" ? "Moved" : "Moved back"} ${result.files} file(s) / ` +
diff --git a/editor/app/channels/page.tsx b/editor/app/channels/page.tsx
@@ -15,6 +15,7 @@ import {
type Site,
} from "yt-dlp-transcript-common/lib/site";
import {
+ autoPauseReasonOf,
overridesOf,
rankOf,
resolveFocusSlugs,
@@ -30,7 +31,15 @@ import {
getSettings,
type SiteSettings,
} from "yt-dlp-transcript-common/lib/settings";
-import { locationLabelOfDataDir } from "yt-dlp-transcript-common/lib/storageLocations";
+import {
+ locationOfDataDir,
+ type StorageLocation,
+} from "yt-dlp-transcript-common/lib/storageLocations";
+import {
+ INTERNAL_LOCATION_ID,
+ INTERNAL_LOCATION_LABEL,
+ volumeFreeBytes,
+} from "yt-dlp-transcript-common/controller/storageLocations";
import { buildChannelBands } from "yt-dlp-transcript-common/views/pipeline/buildBands";
import { EXTERNAL_BAND_IDS } from "yt-dlp-transcript-common/views/pipeline/buildBands";
import {
@@ -38,6 +47,7 @@ import {
type ChannelRow,
type PipelineColumn,
} from "./components/ChannelsTable";
+import type { ChannelVolume } from "./components/ChannelVolumeBar";
import { buildChannelGroupSections } from "yt-dlp-transcript-common/views/channelGroupSections";
import { SyncAllChannelsButton } from "./components/SyncAllChannelsButton";
import { RefreshAllReportsButton } from "./components/RefreshAllReportsButton";
@@ -145,14 +155,36 @@ function focusLabelOf(
: `${focusSlugs.length} channels`;
}
+// WHICH VOLUME A CHANNEL'S MEDIA IS ON, as a filterable id.
+//
+// Same derivation the badge uses (`config.dataDir` under a location's root,
+// longest match wins) with one addition: no `dataDir` at all means the corpus
+// volume, which is the row the operator is trying to empty and therefore the
+// one they most need to filter to. A `dataDir` under a root NOBODY named is
+// neither — it gets "" and falls out of every volume filter, which is the
+// honest answer and the nudge to name that root on /storage.
+function volumeOf(
+ dataDir: string | undefined,
+ locations: StorageLocation[],
+): { id: string; label: string } {
+ const trimmed = dataDir?.trim();
+ if (!trimmed) {
+ return { id: INTERNAL_LOCATION_ID, label: "Internal" };
+ }
+ const found = locationOfDataDir(trimmed, locations);
+ return found
+ ? { id: found.id, label: found.label || found.id }
+ : { id: "", label: "Elsewhere" };
+}
+
export default async function ChannelsPage({
searchParams,
}: {
- searchParams: Promise<{ site?: string }>;
+ searchParams: Promise<{ site?: string; location?: string; sort?: string }>;
}) {
const paths = getPaths();
const settings = getSettings();
- const { site } = await searchParams;
+ const { site, location: locationParam, sort: sortParam } = await searchParams;
const active = resolveActiveSite(site, listSiteIds(paths));
// Counts come from each channel's last snapshot, not a corpus walk. One read
// serves both the table and the freshness footer below.
@@ -206,11 +238,24 @@ export default async function ChannelsPage({
),
),
);
+ const locations = settings.storage.locations;
+ // TWO SYSCALLS PER VOLUME, NOT A PROBE. See volumeFreeBytes: this table draws
+ // 71 rows on every auto-refresh and the rule is that tables never shell out.
+ const freeByVolume = await volumeFreeBytes({ paths, locations });
const all: ChannelRow[] = stats.map((stat) => {
const brief = briefBySlug.get(stat.slug);
const media = mediaBySlug.get(stat.slug);
+ const volume = volumeOf(brief?.config.dataDir, locations);
return {
...stat,
+ // WHERE THE BYTES ARE, AND HOW MANY. Both off the snapshot already in
+ // hand — no walk, no probe. `mediaBytes` is null for a report written
+ // before the field existed, and the cell renders that as "—" rather than
+ // "0 B": a zero would sort a 400 GB channel to the bottom of the very
+ // list the operator opened to find it.
+ volumeId: volume.id,
+ volumeLabel: volume.label,
+ mediaBytes: brief?.snapshot?.totalMediaBytes ?? null,
pipelines: buildChannelBands(snapshots.get(stat.slug) ?? null, ids),
report: {
generatedAt: brief?.snapshot?.generatedAt ?? null,
@@ -227,10 +272,12 @@ export default async function ChannelsPage({
// settings are here and the table is a client component. NEVER a
// probe: this table draws one badge per row. Undefined for a root
// nobody named, which renders exactly what it rendered before.
- locationLabel: locationLabelOfDataDir(
- media.target,
- settings.storage.locations,
- ),
+ // The row already derived it once for the Location column;
+ // re-deriving it here would be a second answer to the same
+ // question. "" is the root nobody named — the badge takes
+ // undefined for that, which renders exactly what it rendered
+ // before locations existed.
+ locationLabel: volume.id === "" ? undefined : volume.label,
}
: null,
priority: {
@@ -238,6 +285,11 @@ export default async function ChannelsPage({
rank: rankOf(priority, stat.slug),
overrides: overridesOf(priority, stat.slug),
focused: focusSet.has(stat.slug),
+ // WHY IT IS PAUSED, when the machine paused it. Null for every channel
+ // the operator paused (or did not pause) themselves — the record is
+ // cleared by any manual tier change, so this can only ever describe a
+ // pause nobody chose.
+ autoPausedReason: autoPauseReasonOf(priority, stat.slug),
// WHY THE ROW IS HELD, and only while something is actually focused.
// A paused channel is not "held by the focus" — it is off, which its
// own tier already says. The per-lane "and the focus still has pending
@@ -259,12 +311,52 @@ export default async function ChannelsPage({
// single site is the active scope — there is no one grouping across the pool.
const activeSite =
active.isAll || !active.siteId ? null : getSite(active.siteId, paths);
- const channels = activeSite
+ const inSite = activeSite
? all.filter((c) => siteChannelSlugs(activeSite).has(c.slug))
: all;
- const sections = activeSite
- ? buildChannelGroupSections(activeSite, channels, briefs, settings)
- : null;
+ // THE VOLUME BAR IS BUILT FROM THE SITE SCOPE, NOT FROM THE FILTERED LIST.
+ // A filter that erased every chip but the one you are standing on would be a
+ // one-way door, and the numbers on the other chips are exactly what makes
+ // "which of these should I move" answerable.
+ const volumeIds = [INTERNAL_LOCATION_ID, ...locations.map((l) => l.id), ""];
+ const volumes: ChannelVolume[] = volumeIds
+ .map((id) => {
+ const rows = inSite.filter((c) => c.volumeId === id);
+ const measured = rows.filter((c) => c.mediaBytes !== null);
+ return {
+ id,
+ label:
+ id === INTERNAL_LOCATION_ID
+ ? INTERNAL_LOCATION_LABEL
+ : id === ""
+ ? "Elsewhere (unnamed root)"
+ : (locations.find((l) => l.id === id)?.label ?? id),
+ channels: rows.length,
+ bytes: measured.reduce((sum, c) => sum + (c.mediaBytes ?? 0), 0),
+ unmeasured: rows.length - measured.length,
+ freeBytes: freeByVolume[id],
+ };
+ })
+ // The unnamed-root chip only exists when something is actually on one.
+ .filter((v) => v.id !== "" || v.channels > 0);
+ // ?location=<id|internal>, alongside ?site=. Ignored when it names nothing on
+ // screen — a stale link from /storage after the last channel moved off a
+ // location must show the page, not an empty table with no way back.
+ const locationFilter =
+ locationParam !== undefined &&
+ volumes.some((v) => v.id === locationParam && v.channels > 0)
+ ? locationParam
+ : null;
+ const channels =
+ locationFilter === null
+ ? inSite
+ : inSite.filter((c) => c.volumeId === locationFilter);
+ // GROUPS ARE A PARTITION OF A SITE, so a volume filter and the grouped render
+ // cannot both be true: half a group is not a group. Filtering flattens.
+ const sections =
+ activeSite && locationFilter === null
+ ? buildChannelGroupSections(activeSite, channels, briefs, settings)
+ : null;
const shown = new Set(channels.map((c) => c.slug));
const freshness = summariseFreshness(briefs.filter((b) => shown.has(b.slug)));
return (
@@ -361,6 +453,13 @@ export default async function ChannelsPage({
columns={columns}
sections={sections}
siteId={activeSite?.siteId}
+ volumes={volumes}
+ locationFilter={locationFilter}
+ // /storage links here with `sort=size`, so the list it promised
+ // ("largest first") is the list that renders. Any other value is
+ // ignored; sorting is otherwise client state, deliberately (a
+ // router.replace races the global AutoRefresh and gets dropped).
+ initialSort={sortParam === "size" ? "size" : null}
// The configured storage locations, for the selection deck's
// destination select. Read here, not in the client component —
// /storage is their one writer. No probe: the deck names a place and
diff --git a/editor/app/jobs/components/JobProgressBars.tsx b/editor/app/jobs/components/JobProgressBars.tsx
@@ -32,6 +32,7 @@ const TASK_KIND_VERB: Record<JobTaskKind, string> = {
transcribe: "Transcribing",
digest: "Digesting",
backfill: "Backfilling",
+ relocate: "Moving media",
};
const METRIC_FILL_BY_TASK: Record<JobTaskKind, string> = {
@@ -39,6 +40,7 @@ const METRIC_FILL_BY_TASK: Record<JobTaskKind, string> = {
transcribe: "bg-success",
digest: "bg-info",
backfill: "bg-warning",
+ relocate: "bg-info/70",
};
export function TaskProgressBar({ task }: { task: JobRowTask }) {
diff --git a/editor/app/storage/actions.ts b/editor/app/storage/actions.ts
@@ -1,5 +1,6 @@
"use server";
+import path from "node:path";
import { revalidatePath } from "next/cache";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import {
@@ -19,8 +20,15 @@ import {
recordProbedIdentity,
resetStorageProbeMemo,
} from "yt-dlp-transcript-common/controller/storageLocations";
+import {
+ clearDirMarker,
+ readDirMarker,
+} from "yt-dlp-transcript-common/controller/relocateDir";
+import { savedVideosMarkerPath } from "yt-dlp-transcript-common/controller/relocateSavedVideos";
import { channelMediaBusyReason } from "../channels/lib/mediaBusy";
import { enqueueRepointJob } from "./lib/repointJob";
+import { enqueueSavedVideosRelocation } from "./lib/savedVideosJob";
+import { savedVideosStoreBusyReason } from "./lib/storeBusy";
// THE SIX THINGS AN OPERATOR MAY DO TO A STORAGE LOCATION.
//
@@ -172,6 +180,22 @@ export async function deleteStorageLocationAction(
`Move them back in place, or onto another location, first.`,
};
}
+ // AND THE SAVED-VIDEO STORE IS A RESIDENT TOO. The channel check above exists
+ // because deleting the location erases the only record of which disk those
+ // absolute paths belong to; the store is on the location by exactly the same
+ // kind of record (`settings.storage.savedVideosLocationId`) and would be
+ // orphaned by exactly the same delete — reachable only through a symlink
+ // whose target nothing in the corpus can any longer name.
+ if ((settings.storage.savedVideosLocationId ?? "") === id) {
+ return {
+ ok: false,
+ error:
+ `The saved-video store is on ${location.root}. Move it back in place ` +
+ `(or onto another location) on this page first — deleting the location ` +
+ `would leave the store reachable only through a symlink nothing here ` +
+ `remembers the name of.`,
+ };
+ }
const locations = settings.storage.locations.filter((l) => l.id !== id);
await writeSettings({
...settings,
@@ -276,3 +300,86 @@ export async function repointStorageLocationAction(
}
return enqueueRepointJob({ locationId: id, newRoot: root });
}
+
+// ---------------------------------------------------------------------------
+// The saved-video store
+// ---------------------------------------------------------------------------
+
+// MOVE THE STORE, RESUME AN INTERRUPTED MOVE, OR THROW ITS MARKER AWAY — the
+// same three verbs a channel's Storage panel has, for the same three states,
+// and for the reason that file gives: without the last two, a killed copy
+// leaves a marker nothing will ever clear and the only fix is deleting a
+// dotfile over SSH.
+//
+// The store is not a channel, so `channelMediaBusyReason` has nothing to ask
+// about it. What guards it instead is the shared relocation queue key: the
+// registry caps it at concurrency 1, so a second move waits rather than racing,
+// and the controller re-reads the disk at every phase regardless.
+export async function relocateSavedVideosAction(
+ locationId: string,
+): Promise<StreamActionResult> {
+ // THE RELOCATION QUEUE KEY ONLY SERIALISES RELOCATIONS. It stops a second
+ // move, a channel move and a re-point from running at once, and it says
+ // nothing at all about the download that is about to persist a source
+ // container into the directory this is about to copy, verify, rename and
+ // reclaim. See lib/storeBusy.ts for the three ways that ends badly, and
+ // lib/savedVideoStore.ts for the guard that makes it refuse rather than lose
+ // a container — this is the half that answers before the operator commits to
+ // a multi-hour copy.
+ const busy = savedVideosStoreBusyReason("moving the saved-video store");
+ if (busy) return { ok: false, error: busy };
+ return enqueueSavedVideosRelocation({ locationId });
+}
+
+// RESUME IS THE MARKER'S DIRECTION, NOT THE OPERATOR'S. A marker records which
+// way the interrupted run was going; resuming it the other way would swap the
+// wrong way round (the controller refuses that by name, and this never asks it
+// to). "out" needs the location it was going to, which is the one the settings
+// still record — the record is written on SUCCESS, so mid-move it still names
+// where the store was, and for a resumed move-out that is where it is going.
+export async function resumeSavedVideosRelocationAction(): Promise<StreamActionResult> {
+ const paths = getPaths();
+ const marker = await readDirMarker(savedVideosMarkerPath(paths));
+ if (!marker) {
+ return {
+ ok: false,
+ error:
+ "The saved-video store has no relocation marker — there is no " +
+ "interrupted move to resume.",
+ };
+ }
+ const busy = savedVideosStoreBusyReason("resuming the store's move");
+ if (busy) return { ok: false, error: busy };
+ if (marker.direction === "back") {
+ return enqueueSavedVideosRelocation({ locationId: "" });
+ }
+ // The target is `<root>/saved-videos`, so the root is its parent — checked
+ // rather than assumed, exactly as resumeRelocationAction checks a channel's.
+ const root = path.dirname(marker.target);
+ const loc = getSettings().storage.locations.find((l) => l.root === root);
+ if (!loc) {
+ return {
+ ok: false,
+ error:
+ `The marker points at ${marker.target}, whose root ${root} is not a ` +
+ `configured storage location. Add it on /storage, or clear the marker ` +
+ `and start again.`,
+ };
+ }
+ return enqueueSavedVideosRelocation({ locationId: loc.id });
+}
+
+// THE LAST RESORT, and the only one of the three that is not a move. It removes
+// the marker file and NOTHING else: no link, no settings, no bytes. Whatever
+// the store reads as afterwards is the truth the disk was already telling
+// underneath it — which may well be `inconsistent`, and that is the honest
+// answer rather than a repair nobody asked for.
+export async function clearSavedVideosMarkerAction(): Promise<LocationResult> {
+ try {
+ await clearDirMarker(savedVideosMarkerPath(getPaths()));
+ } catch (e) {
+ return { ok: false, error: (e as Error).message };
+ }
+ revalidatePath("/storage");
+ return { ok: true, note: "Marker cleared. Nothing was moved." };
+}
diff --git a/editor/app/storage/buildStorage.ts b/editor/app/storage/buildStorage.ts
@@ -1,11 +1,19 @@
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import { getSettings } from "yt-dlp-transcript-common/lib/settings";
import { getRegistry } from "yt-dlp-transcript-common/jobs/registry";
+import { getFreeBytes } from "yt-dlp-transcript-common/lib/diskSpace";
import { udisksctlAvailable } from "yt-dlp-transcript-common/lib/storageVolumes";
+import { listChannelBriefs } from "yt-dlp-transcript-common/controller/channels";
import {
channelsOnLocation,
probeAllLocations,
} from "yt-dlp-transcript-common/controller/storageLocations";
+import { measureTree } from "yt-dlp-transcript-common/controller/relocateDir";
+import {
+ inspectSavedVideosStore,
+ savedVideosMarkerPath,
+} from "yt-dlp-transcript-common/controller/relocateSavedVideos";
+import { pathExists } from "yt-dlp-transcript-common/controller/relocateDir";
import {
buildStorageRows,
type StorageRowsPayload,
@@ -27,15 +35,60 @@ export async function buildStorage(): Promise<StorageRowsPayload> {
const paths = getPaths();
const settings = getSettings();
const locations = settings.storage.locations;
- const [probes, rollups, udisksctl] = await Promise.all([
+ // THE SIZES COME OFF THE REPORTS, not off a walk. `listChannelBriefs` is one
+ // config + one snapshot per channel (6.5 MB of JSON across 71 channels on the
+ // production corpus) and it is what /channels already pays; walking 523 GB of
+ // media to size this page would be the opposite trade. A channel whose
+ // snapshot predates `totalMediaBytes` contributes to `unknownBytes` and the
+ // row says so rather than under-reporting.
+ const briefs = await listChannelBriefs(paths);
+ const mediaBytes: Record<string, number | undefined> = {};
+ for (const b of briefs) mediaBytes[b.slug] = b.snapshot?.totalMediaBytes;
+ const configs = briefs.map((b) => ({ slug: b.slug, config: b.config }));
+ const [probes, rollups, udisksctl, freeOnCorpus] = await Promise.all([
probeAllLocations(locations, paths),
- channelsOnLocation({ paths, locations }),
+ channelsOnLocation({
+ paths,
+ locations,
+ configs,
+ mediaBytes,
+ includeInternal: true,
+ }),
// Memoised per binary path inside storageVolumes, so this is one
// `--version` for the life of the process.
locations.length > 0 ? udisksctlAvailable(paths) : Promise.resolve(false),
+ getFreeBytes(paths.channelsDir),
]);
+ // THE STORE IS WALKED, and that is affordable because of what it holds: one
+ // persisted container per PINNED or kept-latest video, not one per video. It
+ // is a few dozen files on the production corpus, against 523 GB of channel
+ // media that is never walked here (that comes off the reports). If the store
+ // ever grows to corpus scale, this is the line that has to change — and
+ // `listSavedVideos` (which reads the pointers, each carrying its own `bytes`)
+ // is the cheaper answer waiting.
+ const store = await inspectSavedVideosStore(paths, settings);
+ const storeMeasured =
+ store.status === "unreachable" || store.status === "in-transition"
+ ? { bytes: 0, files: 0 }
+ : await measureTree(paths.savedVideosDir);
return buildStorageRows({
locations,
+ savedVideos: {
+ dir: store.dir,
+ at: store.target ?? store.dir,
+ locationId: store.locationId,
+ status: store.status,
+ ...(store.detail ? { detail: store.detail } : {}),
+ bytes: storeMeasured.bytes,
+ files: storeMeasured.files,
+ hasMarker: await pathExists(savedVideosMarkerPath(paths)),
+ },
+ // The corpus volume, always: it is where every unrelocated channel's media
+ // is, and it is the row the operator is actually trying to empty.
+ internal: {
+ root: paths.channelsDir,
+ ...(Number.isFinite(freeOnCorpus) ? { freeBytes: freeOnCorpus } : {}),
+ },
defaultLocationId: settings.storage.defaultLocationId,
probes,
rollups,
diff --git a/editor/app/storage/components/SavedVideosStoreCard.tsx b/editor/app/storage/components/SavedVideosStoreCard.tsx
@@ -0,0 +1,199 @@
+"use client";
+
+import Link from "next/link";
+import { useState } from "react";
+import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog";
+import { formatBytes } from "yt-dlp-transcript-common/lib/format";
+import type { SavedVideosView } from "yt-dlp-transcript-common/views/storage";
+import { cancelJobAction } from "../../jobs/actions";
+import {
+ clearSavedVideosMarkerAction,
+ relocateSavedVideosAction,
+ resumeSavedVideosRelocationAction,
+} from "../actions";
+
+// THE SAVED-VIDEO STORE — the one large thing in the corpus no channel move
+// could ever reach.
+//
+// It gets a card rather than a row in the locations table because it is not a
+// location: nothing lives "on" it, it has no volume identity of its own and
+// there is nothing to re-point. What it has is a place (in the corpus, or on a
+// location) and one verb for changing it.
+//
+// ⚠️ NO RUN PANEL IS EVER UNMOUNTED BY ITS OWN RESULT. StreamActionLog holds
+// its streamed log in React state and calls router.refresh() the instant a run
+// ends (plans/FACTS.md) — and that refresh re-renders this card with the store
+// on its new location: a different destination list, and for a Resume the very
+// marker the button was offered for now gone. The Move panel is therefore
+// rendered UNCONDITIONALLY (only its `disabled` changes), and the Resume
+// section follows the documented `ranHere` shape — a flag set inside its own
+// trigger, so it stays for its log with the now-cleared condition fed to
+// `disabled` rather than becoming a second Run button.
+
+export function SavedVideosStoreCard({ store }: { store: SavedVideosView }) {
+ const [destId, setDestId] = useState(() => store.destinations[0]?.id ?? "");
+ const [ranResumeHere, setRanResumeHere] = useState(false);
+ const [note, setNote] = useState<string | null>(null);
+ const [error, setError] = useState<string | null>(null);
+
+ // DERIVED, not the raw state: a finished move changes the destination list
+ // under this card, and a select still naming the location the store is now
+ // ON would be offering a no-op.
+ const valid = store.destinations.some((d) => d.id === destId);
+ const effectiveId = valid ? destId : (store.destinations[0]?.id ?? "");
+ const canMove =
+ store.busy === null &&
+ store.destinations.length > 0 &&
+ store.status !== "in-transition" &&
+ store.status !== "inconsistent";
+
+ return (
+ <article
+ aria-label="saved video store"
+ className="flex flex-col gap-3 rounded-xl border border-border bg-card px-4 py-3"
+ >
+ <div className="flex flex-wrap items-center gap-3">
+ <h2 className="text-base font-semibold">Saved-video store</h2>
+ <span
+ aria-label="saved videos status"
+ className={`rounded-full border px-2 py-0.5 text-xs font-medium ${
+ store.status === "in-place" || store.status === "ok"
+ ? "border-border bg-muted"
+ : "border-destructive/50 bg-destructive/5 text-destructive"
+ }`}
+ >
+ {store.statusLabel}
+ </span>
+ </div>
+
+ <p className="text-sm text-muted-foreground max-w-3xl">
+ Where persisted source containers live. No channel move reaches it — it
+ belongs to the corpus, not to a channel — so it moves on its own, by the
+ same mechanism: the copy is verified before the source is touched, and a
+ symlink is left behind so{" "}
+ <Link href="/saved-videos" className="underline">
+ every reader
+ </Link>{" "}
+ keeps working unchanged.
+ </p>
+
+ <dl className="grid grid-cols-[max-content_1fr] gap-x-4 gap-y-1 text-sm">
+ <dt className="text-muted-foreground">On</dt>
+ <dd aria-label="saved videos location">{store.locationLabel}</dd>
+ <dt className="text-muted-foreground">Path</dt>
+ <dd className="font-mono text-xs break-all" aria-label="saved videos path">
+ {store.at}
+ {store.at !== store.dir ? ` (read as ${store.dir})` : ""}
+ </dd>
+ <dt className="text-muted-foreground">Size</dt>
+ <dd aria-label="saved videos bytes">
+ {store.status === "unreachable" || store.status === "in-transition"
+ ? "—"
+ : `${formatBytes(store.bytes)} in ${store.files.toLocaleString()} file(s)`}
+ </dd>
+ </dl>
+
+ {store.detail && (
+ <p
+ role="status"
+ aria-label="saved videos detail"
+ className="text-sm rounded border border-border bg-muted px-3 py-2"
+ >
+ {store.detail}
+ </p>
+ )}
+ {store.busy && (
+ <p role="status" className="text-sm rounded border border-border bg-muted px-3 py-2">
+ {store.busy}
+ </p>
+ )}
+ {note && (
+ <p role="status" aria-label="saved videos result" className="text-sm">
+ {note}
+ </p>
+ )}
+ {error && (
+ <p role="alert" aria-label="saved videos error" className="text-sm text-destructive">
+ {error}
+ </p>
+ )}
+
+ <section className="flex flex-col gap-2">
+ <label className="flex flex-col gap-1 text-sm max-w-md">
+ <span className="font-medium">Move the store to…</span>
+ <select
+ aria-label="saved videos destination"
+ value={effectiveId}
+ disabled={!canMove}
+ onChange={(e) => setDestId(e.target.value)}
+ className="rounded border border-border bg-card px-2 py-1 text-sm"
+ >
+ {store.destinations.map((d) => (
+ <option key={d.id || "internal"} value={d.id}>
+ {d.label}
+ </option>
+ ))}
+ </select>
+ </label>
+ <StreamActionLog
+ key="saved-videos-move-log"
+ trigger={() => relocateSavedVideosAction(effectiveId)}
+ cancelAction={cancelJobAction}
+ buttonLabel="Move the store"
+ runningLabel="Moving the store…"
+ label="Move the store"
+ disabled={!canMove}
+ />
+ </section>
+
+ {/* THE TWO WAYS OUT OF A MARKER WHOSE RUN IS GONE — finish it, or throw
+ it away. Offered only when a marker is present AND nothing is running:
+ with a live job the marker is not stale, it belongs to that run. */}
+ {(store.canResume || ranResumeHere) && (
+ <section className="flex flex-col gap-2 rounded border border-destructive/50 bg-destructive/5 px-3 py-2">
+ <h3 className="text-sm font-semibold">
+ Finish the interrupted move, or clear its marker
+ </h3>
+ {store.canResume && (
+ <p className="text-xs text-muted-foreground">
+ <strong>Resume</strong> runs the same move again from where it
+ stopped — whatever already copied correctly is not copied twice,
+ and the source is not touched until the copy verifies. That is the
+ usual answer. <strong>Clear marker</strong> removes the marker file
+ and nothing else: no link, no settings, no bytes.
+ </p>
+ )}
+ <div>
+ <button
+ type="button"
+ aria-label="clear saved videos marker"
+ disabled={!store.canResume}
+ onClick={async () => {
+ setError(null);
+ setNote(null);
+ const r = await clearSavedVideosMarkerAction();
+ if (r.ok) setNote(r.note ?? null);
+ else setError(r.error);
+ }}
+ className="px-3 py-1.5 rounded-md border border-destructive text-destructive text-sm font-medium disabled:opacity-50"
+ >
+ Clear marker
+ </button>
+ </div>
+ <StreamActionLog
+ key="saved-videos-resume-log"
+ trigger={() => {
+ setRanResumeHere(true);
+ return resumeSavedVideosRelocationAction();
+ }}
+ cancelAction={cancelJobAction}
+ buttonLabel="Resume move"
+ runningLabel="Resuming…"
+ label="Resume move"
+ disabled={!store.canResume}
+ />
+ </section>
+ )}
+ </article>
+ );
+}
diff --git a/editor/app/storage/components/StorageLocationsTable.tsx b/editor/app/storage/components/StorageLocationsTable.tsx
@@ -1,5 +1,6 @@
"use client";
+import Link from "next/link";
import { useState } from "react";
import { useRouter } from "next/navigation";
import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog";
@@ -19,6 +20,7 @@ import {
repointStorageLocationAction,
} from "../actions";
import { LocationForm } from "./LocationForm";
+import { SavedVideosStoreCard } from "./SavedVideosStoreCard";
// THE LOCATIONS, ONE CARD EACH, AND WHAT MAY BE DONE TO THEM.
//
@@ -67,6 +69,13 @@ export function StorageLocationsTable({ payload }: { payload: StorageRowsPayload
))}
</section>
+ {/* AFTER THE LOCATIONS, BEFORE THE FORM. The store is a thing that lives
+ on a location, so it reads after the list of them — and before "Add a
+ location", which is the page's trailing affordance. */}
+ {payload.savedVideos && (
+ <SavedVideosStoreCard store={payload.savedVideos} />
+ )}
+
<section className="flex flex-col gap-2">
<h2 className="text-base font-semibold">Add a location</h2>
{adding ? (
@@ -181,10 +190,37 @@ function LocationCard({
</dd>
<dt className="text-muted-foreground">Volume</dt>
<dd className="text-xs" aria-label="volume identity">
- {row.identity ?? "unknown — nothing here can ask (no findmnt, or a container)"}
+ {row.kind === "internal"
+ ? "the corpus volume"
+ : (row.identity ??
+ "unknown — nothing here can ask (no findmnt, or a container)")}
</dd>
<dt className="text-muted-foreground">Channels</dt>
+ {/* THE COUNT AND THE LINK ARE TWO ELEMENTS, not one. `location channels`
+ is the read-out's accessible name and the suite asserts its exact
+ text; a link folded into it appends "— list them" to that text and
+ breaks every caller. The link gets its own `<dd>` on the next row of
+ the grid. */}
<dd aria-label="location channels">{row.channelsText}</dd>
+ {row.channels.total > 0 && (
+ <>
+ <dt className="text-muted-foreground">Browse</dt>
+ <dd>
+ {/* THE ROW'S OWN LIST, largest first. A summary that cannot be
+ opened is a number the operator has to go and re-derive by
+ hand, which on this page is opening 71 channel pages. */}
+ <Link
+ href={row.channelsHref}
+ aria-label={`list channels on ${row.id}`}
+ className="underline hover:text-foreground"
+ >
+ the {row.channels.total} channel(s) on this location
+ </Link>
+ </dd>
+ </>
+ )}
+ <dt className="text-muted-foreground">Media</dt>
+ <dd aria-label="location media bytes">{row.bytesText}</dd>
<dt className="text-muted-foreground">Free</dt>
<dd aria-label="location free space">
{row.freeBytes === undefined ? "—" : formatBytes(row.freeBytes)}
@@ -218,6 +254,11 @@ function LocationCard({
</p>
)}
+ {row.kind === "internal" ? (
+ <p className="text-xs text-muted-foreground" aria-label="internal note">
+ {actionOf(row, "edit")?.withheld}
+ </p>
+ ) : (
<div className="flex flex-wrap items-center gap-2">
<button
type="button"
@@ -268,13 +309,14 @@ function LocationCard({
Delete
</button>
</div>
- {del && !del.offered && (
+ )}
+ {row.kind !== "internal" && del && !del.offered && (
<p className="text-xs text-muted-foreground" aria-label="delete withheld">
{del.withheld}
</p>
)}
- {editing && (
+ {row.kind !== "internal" && editing && (
<LocationForm
mode="edit"
initial={{
@@ -316,7 +358,11 @@ function LocationCard({
/>
</section>
)}
- {repoint && !repoint.offered && !ranRepointHere && row.status !== "available" && (
+ {row.kind !== "internal" &&
+ repoint &&
+ !repoint.offered &&
+ !ranRepointHere &&
+ row.status !== "available" && (
<p className="text-xs text-muted-foreground" aria-label="repoint withheld">
{repoint.withheld}
</p>
diff --git a/editor/app/storage/lib/savedVideosJob.ts b/editor/app/storage/lib/savedVideosJob.ts
@@ -0,0 +1,80 @@
+import { revalidatePath } from "next/cache";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { relocationQueueKey } from "yt-dlp-transcript-common/lib/queueKeys";
+import {
+ runManagedFunction,
+ type StreamActionResult,
+} from "yt-dlp-transcript-common/jobs/streamCommand";
+import { formatBytes } from "yt-dlp-transcript-common/lib/format";
+import { relocateSavedVideos } from "yt-dlp-transcript-common/controller/relocateSavedVideos";
+
+// ONE ENQUEUE OF THE SAVED-VIDEO STORE MOVE, for the two callers that have one:
+// the Move control on /storage and its Resume.
+//
+// Deliberately NOT in `actions.ts` — that file carries "use server", where
+// every non-type export is a server action, so a shared helper exported from it
+// would put an unguarded enqueue on the wire under its own endpoint. Same
+// reason `lib/repointJob.ts` and `channels/lib/relocationJob.ts` exist.
+//
+// THE QUEUE KEY IS THE SHARED RELOCATION ONE. A store move, a channel move and
+// a re-point all rewrite symlinks under the same roots and all run their space
+// check when they START; the registry caps a key at concurrency 1, so one at a
+// time across the three is the whole point of sharing it.
+//
+// THE PROGRESS IS THE CHANNEL MOVE'S, for the reason the code is: it is the
+// same rsync, reported the same way, so /jobs draws the same bar.
+export async function enqueueSavedVideosRelocation(opts: {
+ // "" moves the store back in place.
+ locationId: string;
+}): Promise<StreamActionResult> {
+ const paths = getPaths();
+ const back = opts.locationId.trim() === "";
+ return runManagedFunction({
+ kind: "relocate-saved-videos",
+ queueKey: relocationQueueKey(),
+ paths,
+ fn: async (onLog, signal, _setProgress, ctx) => {
+ const taskId = "relocate:saved-videos";
+ let started = false;
+ const result = await relocateSavedVideos({
+ paths,
+ locationId: opts.locationId,
+ onLog,
+ // Added on the FIRST frame, not at start: the preflight and a resumed
+ // run's verify pass transfer nothing, and a bar sitting at 0 % through
+ // them would be claiming a copy had begun.
+ onProgress: (p) => {
+ if (!started) {
+ started = true;
+ ctx?.addTask({
+ id: taskId,
+ label: back
+ ? "saved-video store → in place"
+ : `saved-video store → ${opts.locationId}`,
+ kind: "relocate",
+ startedAt: Date.now(),
+ });
+ }
+ ctx?.updateTask(taskId, { fraction: p.fraction, detail: p.detail });
+ },
+ signal,
+ }).finally(() => {
+ if (started) ctx?.removeTask(taskId);
+ });
+ onLog(
+ `${back ? "Moved back" : "Moved"} ${result.files} file(s) / ` +
+ `${formatBytes(result.bytes)} — ${result.target}` +
+ (result.resumed ? " (resumed an interrupted move)" : "") +
+ (result.retried
+ ? " (one directory timestamp settled by a second pass)"
+ : ""),
+ );
+ // No snapshot regen — the kind is in NO_REGEN_KINDS. The move changes
+ // where the store's bytes are, not what any channel has: every count in
+ // every report is identical afterwards. What DOES change is what these
+ // pages read per render.
+ revalidatePath("/storage");
+ revalidatePath("/saved-videos");
+ },
+ });
+}
diff --git a/editor/app/storage/lib/storeBusy.ts b/editor/app/storage/lib/storeBusy.ts
@@ -0,0 +1,80 @@
+import { getRegistry } from "yt-dlp-transcript-common/jobs/registry";
+import { getAutoRunnerStatus } from "yt-dlp-transcript-common/controller/autoRunner";
+
+// IS ANYTHING WRITING INTO THE SAVED-VIDEO STORE RIGHT NOW?
+//
+// `channelMediaBusyReason` asks the same question about one channel's `data/`.
+// This is its twin for the store, and the difference is the one that matters:
+// THE STORE HAS NO SLUG. A persist belongs to a video, the video belongs to a
+// channel, and the container lands in the ONE store — so "is this channel
+// busy" answers nothing here and the question has to be asked of the whole
+// machine.
+//
+// WHY A BUSY CHECK AT ALL WHEN THE MARKER ALREADY REFUSES THE WRITE.
+// `assertSavedVideosStoreWritable` (lib/savedVideoStore.ts) is the guard: it
+// runs at the moment of the persist, it covers every caller including ones that
+// do not exist yet, and it is what makes a raced write impossible rather than
+// merely unlikely. What it CANNOT do is give the operator the answer before
+// they commit — a download that starts a persist thirty seconds into a
+// three-hour copy is refused correctly, and the operator finds out from a job
+// log. So this is the courtesy half: a sentence, before the move, naming what
+// is running.
+//
+// THE KIND LIST IS DELIBERATELY NOT EXHAUSTIVE, and that is safe precisely
+// because the marker is the guard. It names the kinds that can reach
+// `persistSourceVideo` / the store today; a kind that escapes it costs a
+// refused persist (logged, container left in the data dir, picked up next
+// time), not a lost container.
+//
+// SERVER-SIDE ONLY: it reaches the controller, which imports execa
+// transitively. `next build` proves nothing client-side imports it.
+
+// Everything that runs `downloadOneManaged` (which persists a kept source
+// container at the end of a download), plus the four kinds whose whole subject
+// is the store.
+const STORE_TOUCHING_KINDS = new Set([
+ // Downloads, every entry point.
+ "auto-download",
+ "auto-download-unit",
+ "download-from-playlist",
+ "download-missing",
+ "download-missing-subs",
+ "import-one",
+ "redownload-archive",
+ "redownload-incomplete-bucket",
+ "retry-bucket",
+ // One kind, channel-scoped: a sync downloads.
+ "sync",
+ // The store's own jobs.
+ "persist-kept",
+ "check-kept-deleted",
+ "backup-saved-videos",
+ "verify-saved-video-backup",
+]);
+
+// A sentence naming what is holding the store, or null. The caller supplies the
+// verb, so the same reason reads as an instruction wherever it appears.
+export function savedVideosStoreBusyReason(what?: string): string | null {
+ const jobs = getRegistry()
+ .list()
+ .filter(
+ (j) =>
+ STORE_TOUCHING_KINDS.has(j.kind) &&
+ (j.status === "running" || j.status === "queued"),
+ );
+ // THE DOWNLOAD LANE'S UNITS MAKE NO JOB RECORD — the omnimirror lesson, and
+ // the reason `channelMediaBusyReason` exists in the shape it does. A unit
+ // that is mid-download is a persist that has not happened yet. No slug
+ // filter: every one of them writes into the same store.
+ const units = getAutoRunnerStatus("download").inFlight.length;
+ if (jobs.length === 0 && units === 0) return null;
+
+ const parts: string[] = [];
+ if (jobs.length > 0) {
+ const kinds = [...new Set(jobs.map((j) => j.kind))].slice(0, 3).join(", ");
+ parts.push(`${jobs.length} running/queued download job(s) (${kinds})`);
+ }
+ if (units > 0) parts.push(`${units} auto-download unit(s) in flight`);
+ const subject = `${parts.join(" and ")} — any of them can persist a source video into the store`;
+ return what ? `Finish or cancel ${subject}, before ${what}.` : subject;
+}
diff --git a/editor/app/widget/components/MonitorWidget.tsx b/editor/app/widget/components/MonitorWidget.tsx
@@ -1095,6 +1095,8 @@ const TASK_KIND_VERB: Record<JobTaskKind, string> = {
transcribe: "\u270e",
digest: "\u00b6",
backfill: "\u21ba",
+ // A right arrow: a relocate task is a channel's media moving to another disk.
+ relocate: "\u21e2",
};
const TASK_KIND_FILL: Record<JobTaskKind, string> = {
@@ -1102,6 +1104,7 @@ const TASK_KIND_FILL: Record<JobTaskKind, string> = {
transcribe: "bg-success",
digest: "bg-info",
backfill: "bg-warning",
+ relocate: "bg-info/70",
};
// Per-metric glyph for the compact widget line. A Record over JobProgressMetric
diff --git a/editor/e2e/channel-priority.spec.ts b/editor/e2e/channel-priority.spec.ts
@@ -316,7 +316,21 @@ test("a row selection focuses those channels and bulk-sets their tier", async ({
.toEqual({ kind: "channels", slugs: ["slow-a"] });
// Applying a tier to both rows at once writes both entries in one save.
- await page.getByLabel("select all channels").check();
+ //
+ // The focus save above triggers a refresh of the rows; a tick that lands while
+ // that re-render is in flight is dropped (seen once under a loaded full suite:
+ // "Clicking the checkbox did not change its state"). Selecting all is
+ // idempotent, so retry until the checkbox reports it.
+ const selectAll = page.getByLabel("select all channels");
+ await expect
+ .poll(
+ async () => {
+ await selectAll.check({ timeout: 5_000 }).catch(() => {});
+ return selectAll.isChecked();
+ },
+ { timeout: 30_000, intervals: [250, 500, 1_000, 1_000, 2_000] },
+ )
+ .toBe(true);
await page.getByLabel("bulk tier").selectOption("low");
await page
.getByLabel("channel priority bulk")
diff --git a/editor/e2e/channel-storage.spec.ts b/editor/e2e/channel-storage.spec.ts
@@ -116,6 +116,18 @@ test("relocate a channel's media to another root, and move it back", async ({
await expect(page.getByLabel("Move media output")).toContainText("Moved", {
timeout: 60_000,
});
+ // THE COPY SAYS HOW FAR IT GOT. rsync's `--info=progress2` frames used to go
+ // into the log as several thousand carriage-return redraws of one line; the
+ // controller parses them and writes one line per decile instead. The fixture
+ // is two small files, so the copy is over in one frame and the assertion is
+ // on the DECILE LINE that frame produces — a percentage-and-rate line that
+ // could only have come from a parsed frame. (The live task bar on /jobs is
+ // fed by the same parse; there is no honest way to catch a 3 ms copy
+ // mid-flight from a browser, which is what the unit tests in
+ // jobs/progressParsers.test.ts are for.)
+ await expect(page.getByLabel("Move media output")).toContainText(
+ /Copying… .* · \d+ % · /,
+ );
// data/ is a symlink now, and config.json records the target — written only
// by the job, on success, after the copy verified.
@@ -157,15 +169,26 @@ test("relocate a channel's media to another root, and move it back", async ({
page.getByLabel(/^media location: Media relocated/),
).toBeVisible();
- // --- back ---------------------------------------------------------------
+ // --- back, WHICH IS A DESTINATION AND NOT A SECOND BUTTON ----------------
+ // "Move back in place" was its own section with its own button; it is now
+ // the one destination the select offers while the media is on a location,
+ // and it runs the same action (moveChannelMediaBackAction) it always did.
await page.goto(channelStage(SLUG, "storage"));
await expect(page.getByLabel("media path")).toHaveText(target);
+ const destination = page.getByLabel("destination location");
+ await expect(destination).toHaveValue("__internal");
+ // The only option: a location-to-location move is refused by the controller,
+ // so the select does not offer one.
+ await expect(destination.locator("option")).toHaveCount(1);
+ await expect(destination.locator("option")).toHaveText(
+ "Internal (in place)",
+ );
const backButton = page.getByRole("button", {
name: "Move back in place",
});
await expect(backButton).toBeEnabled();
await backButton.click();
- await expect(page.getByLabel("Move back in place output")).toContainText(
+ await expect(page.getByLabel("Move media output")).toContainText(
"Moved back",
{ timeout: 60_000 },
);
diff --git a/editor/e2e/channels-storage-columns.spec.ts b/editor/e2e/channels-storage-columns.spec.ts
@@ -0,0 +1,221 @@
+import { mkdir, rename, symlink } from "node:fs/promises";
+import { join } from "node:path";
+import { test, expect } from "@playwright/test";
+import { baseUrl } from "./baseUrl";
+import {
+ generateReport,
+ readJson,
+ resetData,
+ resolvePath,
+ writeChannelConfig,
+ writeSettings,
+} from "./helpers";
+
+// /channels AS THE STORAGE WORKING SURFACE.
+//
+// Operator ask, 2026-09-20, verbatim: "I'd like to see disk space and current
+// storage volume as columns on the channels menu, and let me filter by volume
+// for easy checking of things that may need moving or other processing."
+//
+// So: a Location column, a Size column, one free-space read-out per VOLUME (not
+// per row — 71 copies of one number is what the focus bar taught us not to do),
+// a `?location=` filter beside the existing `?site=`, and a helper that ticks
+// the largest in-place channels until a target is met.
+//
+// THE SIZES COME OFF THE REPORTS, so every test here generates one first. That
+// is not ceremony: a channel with no report has NO measured size, and the
+// distinction between "0 bytes" and "nobody measured" is the thing half of this
+// feature exists to keep straight.
+//
+// `minFreeDiskGB: 0` on every settings write — a spec that does not say so
+// inherits the 5 GB default and fails on a nearly full disk (repo memory,
+// 549d).
+
+const SLUG = "test-youtube";
+
+// Put a channel's media on `root` the way a finished relocation leaves it: the
+// real directory on the "drive", an absolute symlink at channels/<slug>/data,
+// and config.dataDir naming the target. Built directly rather than by running a
+// move — what is under test is the COLUMN, and a real rsync buys nothing here.
+async function relocateOnDisk(slug: string, root: string): Promise<string> {
+ const channelDir = resolvePath(`test-transcripts/channels/${slug}`);
+ const target = join(root, slug, "data");
+ await mkdir(join(root, slug), { recursive: true });
+ await rename(join(channelDir, "data"), target);
+ await symlink(target, join(channelDir, "data"));
+ const config = await readJson<Record<string, unknown>>(
+ `test-transcripts/channels/${slug}/config.json`,
+ );
+ await writeChannelConfig(slug, { ...config, dataDir: target });
+ await fetch(`${baseUrl}/api/test/invalidate-cache`).catch(() => {});
+ return target;
+}
+
+test("the Location and Size columns say which disk and how much", async ({
+ page,
+}, testInfo) => {
+ test.setTimeout(120_000);
+ await resetData("one-youtube-channel-with-data");
+ const root = testInfo.outputPath("platter");
+ await mkdir(root, { recursive: true });
+ await writeSettings({
+ adminTitle: "Test Admin",
+ minFreeDiskGB: 0,
+ storage: {
+ locations: [{ id: "cold", label: "Cold", root, autoRepoint: false }],
+ defaultLocationId: "cold",
+ },
+ });
+ await generateReport(page, SLUG);
+
+ await page.goto("/channels");
+ // IN PLACE READS "Internal", not blank. A channel whose media has never moved
+ // is on the corpus volume, and that is a fact about it, not an absence.
+ const location = page.getByLabel(`media location for ${SLUG}`);
+ await expect(location).toContainText("Internal");
+ // The fixture is two small files, so the figure is not zero and not "—": the
+ // report measured it.
+ const size = page.getByLabel(`media size for ${SLUG}`);
+ await expect(size).not.toHaveText("—");
+ await expect(size).toContainText(/B|KB|MB/);
+
+ // --- moved, and the column follows it ------------------------------------
+ await relocateOnDisk(SLUG, root);
+ await page.goto("/channels");
+ await expect(page.getByLabel(`media location for ${SLUG}`)).toContainText(
+ "Cold",
+ );
+
+ // --- the volume bar, once per volume -------------------------------------
+ const volumes = page.getByLabel("storage volumes");
+ await expect(volumes).toBeVisible();
+ await expect(volumes.getByLabel("volume cold")).toContainText("1 ch");
+ await expect(volumes.getByLabel("volume cold")).toContainText("free");
+ // FREE SPACE IS NOT A COLUMN. It is a fact about a disk; one read-out per
+ // volume, not one per row.
+ // (Matched on TEXT, not on the accessible name: each header's name comes from
+ // its sort button's aria-label, as channels-sort.spec.ts discovered.)
+ await expect(
+ page.getByRole("columnheader").filter({ hasText: /free/i }),
+ ).toHaveCount(0);
+});
+
+test("a channel with no report reads as unmeasured, never as empty", async ({
+ page,
+}) => {
+ await resetData("one-youtube-channel-with-data");
+ await writeSettings({ adminTitle: "Test Admin", minFreeDiskGB: 0 });
+ // Deliberately NO generateReport. A zero here would rank a 400 GB channel
+ // bottom of the very list the operator opened to find it.
+ await page.goto("/channels");
+ await expect(page.getByLabel(`media size for ${SLUG}`)).toHaveText("—");
+});
+
+test("filter by volume, sort by size, and free up N GB", async ({
+ page,
+}, testInfo) => {
+ test.setTimeout(180_000);
+ await resetData("one-youtube-channel-with-data");
+ const root = testInfo.outputPath("platter");
+ await mkdir(root, { recursive: true });
+
+ // A second channel, so a filter has something to exclude and a sort has
+ // something to order.
+ const SECOND = "second-mover";
+ await writeChannelConfig(SECOND, {
+ handling: "youtube",
+ name: "Second Mover",
+ url: "https://www.youtube.com/@second",
+ });
+ await mkdir(
+ resolvePath(`test-transcripts/channels/${SECOND}/data/20240102_second0001`),
+ { recursive: true },
+ );
+ await writeSettings({
+ adminTitle: "Test Admin",
+ minFreeDiskGB: 0,
+ storage: {
+ locations: [{ id: "cold", label: "Cold", root, autoRepoint: false }],
+ defaultLocationId: "cold",
+ },
+ });
+ await generateReport(page, SLUG);
+ await generateReport(page, SECOND);
+ await relocateOnDisk(SECOND, root);
+
+ // --- ?location= ----------------------------------------------------------
+ await page.goto("/channels?location=internal");
+ await expect(page.getByLabel(`select ${SLUG}`)).toBeVisible();
+ await expect(page.getByLabel(`select ${SECOND}`)).toHaveCount(0);
+
+ await page.goto("/channels?location=cold");
+ await expect(page.getByLabel(`select ${SECOND}`)).toBeVisible();
+ await expect(page.getByLabel(`select ${SLUG}`)).toHaveCount(0);
+
+ // The chip is the filter, and it carries `aria-current` when it is the one
+ // in force — so the operator can see which list they are looking at.
+ await expect(
+ page.getByLabel("storage volumes").getByLabel("volume cold"),
+ ).toHaveAttribute("aria-current", "true");
+
+ // --- ?sort=size ----------------------------------------------------------
+ // /storage links here with it, so the "largest first" list it promised is the
+ // list that renders.
+ await page.goto("/channels?sort=size");
+ const sizeHeader = page
+ .getByRole("columnheader")
+ .filter({ hasText: /^Size/ });
+ await expect(sizeHeader).toHaveAttribute("aria-sort", "descending");
+ // And clicking the header flips it, like every other sortable column.
+ await expect
+ .poll(
+ async () => {
+ await page
+ .getByRole("button", { name: "sort by Size" })
+ .click({ timeout: 5_000 })
+ .catch(() => {});
+ return sizeHeader.getAttribute("aria-sort");
+ },
+ { timeout: 30_000, intervals: [250, 500, 1_000, 1_000, 2_000] },
+ )
+ .toBe("ascending");
+
+ // --- free up N GB --------------------------------------------------------
+ await page.goto("/channels");
+ // Nothing ticked, so the deck is not there — which is why the control lives
+ // in the volume bar and not on the deck.
+ await expect(page.getByLabel("channel priority bulk")).toHaveCount(0);
+ // CLICK UNTIL IT TOOK. Every control here is a React handler attached after
+ // the server HTML arrives; Playwright's actionability check is satisfied by
+ // the BUTTON, not by the listener, so a click landing in that window is
+ // silently dropped. The action is idempotent — the same selection is computed
+ // from the same rows — so retrying until the consequence appears is both safe
+ // and the honest way to wait for a listener the DOM cannot advertise.
+ const target = page.getByLabel("free up target GB");
+ const selectLargest = page.getByLabel("select largest channels to free up");
+ await expect
+ .poll(
+ async () => {
+ await target.fill("1").catch(() => {});
+ await selectLargest.click({ timeout: 5_000 }).catch(() => {});
+ return page.getByLabel(`select ${SLUG}`).isChecked();
+ },
+ { timeout: 30_000, intervals: [250, 500, 1_000, 1_000, 2_000] },
+ )
+ .toBe(true);
+
+ // It ticked the in-place channel and NOT the one already on the platter —
+ // moving that frees nothing on the disk being emptied.
+ await expect(page.getByLabel(`select ${SLUG}`)).toBeChecked();
+ await expect(page.getByLabel(`select ${SECOND}`)).not.toBeChecked();
+ // The deck is up, with the note beside the Move button that will act on it.
+ const deck = page.getByLabel("channel priority bulk");
+ await expect(deck).toBeVisible();
+ await expect(deck.getByLabel("free up selection note")).toContainText(
+ "selected",
+ );
+ // 1 GB is more than the fixture holds, so it says so rather than pretending.
+ await expect(deck.getByLabel("free up selection note")).toContainText(
+ "short of the target",
+ );
+});
diff --git a/editor/e2e/ops-api.spec.ts b/editor/e2e/ops-api.spec.ts
@@ -0,0 +1,494 @@
+// /api/ops — the HTTP door onto the editor's server actions.
+//
+// What this spec is really pinning is the claim the layer rests on: that an
+// agent driving the editor over HTTP and an operator clicking the same button
+// get the SAME answer from the SAME code. So the assertions are deliberately
+// about the shared sentences — the download-filter message that
+// title-filter.spec.ts reads off the form, the busy-channel refusal the Storage
+// panel shows — and about the files on disk, not about the routes' own shapes.
+//
+// THE 503 BRANCH IS NOT REACHABLE FROM HERE. The test server runs with
+// WORKER_TOKEN=test-worker-token (editor/package.json, dev:test) and there is one
+// server for the whole suite, so no spec can observe the endpoint disabled. 401
+// (missing and wrong) is covered below; the 503 is authorizeWorkerRequest's own
+// first branch, shared with /api/worker/* and unit-tested by nothing else
+// either. See plans/FACTS.md.
+
+import { readdir, rm } from "node:fs/promises";
+import { test, expect, type APIRequestContext } from "@playwright/test";
+import { baseUrl } from "./baseUrl";
+import {
+ generateReport,
+ pathExists,
+ readJson,
+ resetData,
+ resolvePath,
+ writeChannelConfig,
+ writeSettings,
+} from "./helpers";
+
+const TOKEN = "test-worker-token";
+const AUTH = { authorization: `Bearer ${TOKEN}` };
+
+type OpsResponse = {
+ ok?: boolean;
+ error?: string;
+ jobId?: string;
+ queued?: string[];
+ skipped?: { slug: string; reason: string }[];
+};
+
+async function ops(
+ request: APIRequestContext,
+ action: string,
+ data: Record<string, unknown>,
+): Promise<{ status: number; body: OpsResponse }> {
+ const res = await request.post(`${baseUrl}/api/ops/${action}`, {
+ headers: AUTH,
+ data,
+ });
+ return { status: res.status(), body: (await res.json()) as OpsResponse };
+}
+
+// EVERY SPEC HERE NAMES THE DISK FLOOR. Without it the merged fixture default
+// applies, and a spec that starts a download-shaped job on a nearly-full host
+// would be refused by lowDiskError() with a returned { ok: false } no assertion
+// reads. 0 is the fixture's own value; naming it makes that a decision.
+async function settings(): Promise<void> {
+ await writeSettings({ minFreeDiskGB: 0 });
+}
+
+test("the token gate answers 401 for a missing and for a wrong bearer", async ({
+ request,
+}) => {
+ await resetData("empty");
+ for (const headers of [undefined, { authorization: "Bearer wrong" }]) {
+ const post = await request.post(`${baseUrl}/api/ops/refresh-report`, {
+ ...(headers ? { headers } : {}),
+ data: { all: true },
+ });
+ expect(post.status(), JSON.stringify(headers)).toBe(401);
+ // The READ side is gated by the same token, not merely the write side.
+ const get = await request.get(`${baseUrl}/api/ops/channel/anything`, {
+ ...(headers ? { headers } : {}),
+ });
+ expect(get.status(), JSON.stringify(headers)).toBe(401);
+ }
+});
+
+test("an unknown body key is a 400 that names the accepted keys", async ({
+ request,
+}) => {
+ await resetData("empty");
+ // A misspelled key would otherwise get a cheerful { ok: true } and a channel
+ // that did not change.
+ const { status, body } = await ops(request, "sync", {
+ slug: "x",
+ fullSweep: true,
+ });
+ expect(status).toBe(400);
+ expect(body.ok).toBe(false);
+ expect(body.error).toContain("unknown key(s): fullSweep");
+ expect(body.error).toContain("full");
+
+ // So is a nested one, on the route whose body carries an object.
+ const patch = await ops(request, "channel-config", {
+ slug: "x",
+ patch: { downloadFilterExcluded: "rerun" },
+ });
+ expect(patch.status).toBe(400);
+ expect(patch.body.error).toContain("downloadFilterExcluded");
+});
+
+test("a traversing slug is refused at the door, on every route that takes one", async ({
+ request,
+}) => {
+ await resetData("title-filter-channel");
+ const channelsDir = resolvePath("test-transcripts/channels");
+ const before = (await readdir(channelsDir)).sort();
+ expect(before).toEqual(["test-filter"]);
+
+ // EVERY SLUG BELOW REACHES A path.join UNDER channelsDir, and the readers
+ // swallow their own errors — so an unchecked traversing segment would fail
+ // SILENTLY (an empty config read as "channel not found") rather than loudly,
+ // and any future writer on that path would land outside the corpus. reqSlug
+ // is one check for all of them; this is the assertion that it is wired to
+ // each.
+ const cases: [string, Record<string, unknown>][] = [
+ ["metadata-scan", { slug: "../../escape" }],
+ ["sync", { slug: "../../escape" }],
+ ["download-missing", { slug: "../../escape" }],
+ ["import-video", { slug: "../../escape", url: "https://example.com/v" }],
+ ["retry-bucket", { slug: "../../escape", bucket: "noTranscript" }],
+ ["refresh-report", { slug: "../../escape" }],
+ ["channel-config", { slug: "../../escape", patch: { cookieMode: "always" } }],
+ ["channel-priority", { slugs: ["../../escape"], tier: "paused" }],
+ ["relocate", { slugs: ["../../escape"], root: "/tmp/ops-api-never" }],
+ ["relocate-back", { slugs: ["../../escape"] }],
+ ];
+ for (const [action, data] of cases) {
+ const { status, body } = await ops(request, action, data);
+ expect(status, action).toBe(400);
+ expect(body.error, action).toMatch(/is not a valid channel slug/);
+ }
+
+ // The read route takes its slug as a path SEGMENT rather than in a body, so
+ // it spells the same check out. A slash-bearing value is not the case to
+ // assert here — the router never matches one to a single dynamic segment, so
+ // it 404s before the handler exists. What DOES reach the handler is a
+ // one-segment name CHANNEL_SLUG_RE still refuses, and a leading dot is the
+ // one that matters: it is how a dotfile beside the channels dir would be
+ // named at.
+ const read = await request.get(`${baseUrl}/api/ops/channel/.escape`, {
+ headers: AUTH,
+ });
+ expect(read.status()).toBe(400);
+ expect(((await read.json()) as OpsResponse).error).toMatch(
+ /is not a valid channel slug/,
+ );
+
+ // NOTHING WAS TOUCHED: the corpus still holds exactly the fixture channel,
+ // and the fixture's own config is byte-identical.
+ expect((await readdir(channelsDir)).sort()).toEqual(before);
+ expect(
+ await readJson<Record<string, unknown>>(
+ "test-transcripts/channels/test-filter/config.json",
+ ),
+ ).toEqual({
+ handling: "youtube",
+ name: "Test Title Filter",
+ url: "https://www.youtube.com/@example/videos",
+ downloadFilter: { include: "guest" },
+ });
+});
+
+test("channel-config round-trips a download filter and refuses a bad regex", async ({
+ page,
+ request,
+}) => {
+ test.setTimeout(120_000);
+ await resetData("title-filter-channel");
+ await settings();
+ const SLUG = "test-filter";
+ const CONFIG = `test-transcripts/channels/${SLUG}/config.json`;
+ // TWO FIELDS THE CLEAR-THEN-LAYER PATH ACTUALLY THREATENS. name/handling/url
+ // survive a patch trivially — they are re-posted by the serializer because
+ // parseChannelForm requires the first two and CHANNEL_FORM_FIELDS spares
+ // none of the rest. `cookieMode` and `keepLatest` are in CHANNEL_FORM_FIELDS,
+ // so updateChannelAction deletes them from the baseline before layering, and
+ // a patch that failed to re-post them would silently clear both.
+ //
+ // `keepLatest: 0` is the sharp one: 0 is the explicit "disabled" sentinel, and
+ // a serializer that tested the value for truthiness rather than for
+ // null/undefined would drop it and read as "inherit" on the next save.
+ await writeChannelConfig(SLUG, {
+ handling: "youtube",
+ name: "Test Title Filter",
+ url: "https://www.youtube.com/@example/videos",
+ downloadFilter: { include: "guest" },
+ cookieMode: "always",
+ keepLatest: 0,
+ });
+ const before = await readJson<Record<string, unknown>>(CONFIG);
+ expect(before.downloadFilter).toEqual({ include: "guest" });
+ expect(before.cookieMode).toBe("always");
+ expect(before.keepLatest).toBe(0);
+
+ // THE SAME SENTENCE THE FORM SHOWS. title-filter.spec.ts reads this off the
+ // page after typing "elf(" into the exclude input; the route reaches it
+ // through the same parseChannelForm, which is the whole point of routing a
+ // patch through a FormData rather than writing config.json directly.
+ const bad = await ops(request, "channel-config", {
+ slug: SLUG,
+ patch: { downloadFilterExclude: "elf(" },
+ });
+ expect(bad.status).toBe(400);
+ expect(bad.body.error).toMatch(
+ /Download filter exclude is not a valid regular expression/,
+ );
+ // And nothing was written.
+ expect(
+ (await readJson<Record<string, unknown>>(CONFIG)).downloadFilter,
+ ).toEqual({ include: "guest" });
+
+ const good = await ops(request, "channel-config", {
+ slug: SLUG,
+ patch: { downloadFilterInclude: "guest|special", downloadFilterExclude: "rerun" },
+ });
+ expect(good.body).toEqual({ ok: true });
+ const after = await readJson<Record<string, unknown>>(CONFIG);
+ expect(after.downloadFilter).toEqual({
+ include: "guest|special",
+ exclude: "rerun",
+ });
+ // A PATCH IS A PATCH. updateChannelAction clears every form-managed key
+ // before layering the parse result on, so a route that posted only the patch
+ // would have silently dropped every one of these.
+ expect(after.name).toBe(before.name);
+ expect(after.handling).toBe(before.handling);
+ expect(after.url).toBe(before.url);
+ expect(after.cookieMode).toBe("always");
+ expect(after.keepLatest).toBe(0);
+
+ // "" clears a field, exactly as clearing the input does.
+ const cleared = await ops(request, "channel-config", {
+ slug: SLUG,
+ patch: { downloadFilterInclude: "", downloadFilterExclude: "" },
+ });
+ expect(cleared.body).toEqual({ ok: true });
+ const emptied = await readJson<Record<string, unknown>>(CONFIG);
+ expect("downloadFilter" in emptied).toBe(false);
+ // Clearing one field clears ONLY that field.
+ expect(emptied.cookieMode).toBe("always");
+ expect(emptied.keepLatest).toBe(0);
+
+ // The read route sees the same config, and says whether the media is there.
+ await generateReport(page, SLUG);
+ const read = await request.get(`${baseUrl}/api/ops/channel/${SLUG}`, {
+ headers: AUTH,
+ });
+ expect(read.status()).toBe(200);
+ const view = (await read.json()) as {
+ ok: boolean;
+ config: { name: string };
+ media: { status: string };
+ report: { totals: { videos: number } } | null;
+ };
+ expect(view.ok).toBe(true);
+ expect(view.config.name).toBe(before.name);
+ expect(view.media.status).toBe("in-place");
+ expect(view.report?.totals.videos).toBeGreaterThanOrEqual(0);
+
+ const missing = await request.get(`${baseUrl}/api/ops/channel/nope`, {
+ headers: AUTH,
+ });
+ expect(missing.status()).toBe(404);
+});
+
+test("channel-priority writes a per-operation override", async ({ request }) => {
+ await resetData("title-filter-channel");
+ await settings();
+ const SLUG = "test-filter";
+
+ const pinned = await ops(request, "channel-priority", {
+ slugs: [SLUG],
+ operation: "download",
+ tier: "paused",
+ });
+ expect(pinned.body).toEqual({ ok: true });
+ await expect
+ .poll(async () => {
+ const s = await readJson<{
+ channelPriority?: {
+ channels?: Record<string, { overrides?: Record<string, string> }>;
+ };
+ }>("test-settings.json");
+ return s.channelPriority?.channels?.[SLUG]?.overrides?.download ?? null;
+ })
+ .toBe("paused");
+
+ // null clears the override — and only with an operation named, because a bare
+ // null tier has no meaning for the BASE tier.
+ const bare = await ops(request, "channel-priority", {
+ slugs: [SLUG],
+ tier: null,
+ });
+ expect(bare.status).toBe(400);
+ expect(bare.body.error).toMatch(/clears an operation override/);
+
+ const cleared = await ops(request, "channel-priority", {
+ slugs: [SLUG],
+ operation: "download",
+ tier: null,
+ });
+ expect(cleared.body).toEqual({ ok: true });
+ await expect
+ .poll(async () => {
+ const s = await readJson<{
+ channelPriority?: {
+ channels?: Record<string, { overrides?: Record<string, string> }>;
+ };
+ }>("test-settings.json");
+ return s.channelPriority?.channels?.[SLUG]?.overrides?.download ?? null;
+ })
+ .toBe(null);
+
+ const bogus = await ops(request, "channel-priority", {
+ slugs: [SLUG],
+ tier: "urgent",
+ });
+ expect(bogus.status).toBe(400);
+ expect(bogus.body.error).toMatch(/normal, low, paused/);
+});
+
+test("metadata-scan starts a job, and the job says it is a metadata-scan", async ({
+ request,
+}) => {
+ test.setTimeout(120_000);
+ await resetData("title-filter-channel");
+ await settings();
+ const SLUG = "test-filter";
+
+ const { status, body } = await ops(request, "metadata-scan", { slug: SLUG });
+ expect(status).toBe(200);
+ expect(body.ok).toBe(true);
+ expect(body.jobId).toBeTruthy();
+
+ // THE ROUTE DOES NOT STREAM, so the id is the whole contract: the caller
+ // follows the same log endpoint the editor's own panel polls.
+ // The sidecar is written with a `void` promise right after enqueue, so poll.
+ const metaPath = `test-transcripts/.jobs/${body.jobId}.meta.json`;
+ await expect
+ .poll(async () => {
+ const meta = await readJson<{ kind: string; channelSlug: string }>(
+ metaPath,
+ ).catch(() => null);
+ return meta ? `${meta.kind}/${meta.channelSlug}` : null;
+ })
+ .toBe(`metadata-scan/${SLUG}`);
+
+ const log = await request.get(
+ `${baseUrl}/api/jobs/${body.jobId}/log?from=0`,
+ );
+ expect(log.status()).toBe(200);
+ const payload = (await log.json()) as { status: string };
+ expect(
+ ["queued", "running", "done", "failed", "cancelled"].includes(
+ payload.status,
+ ),
+ ).toBe(true);
+
+ const unknownChannel = await ops(request, "metadata-scan", { slug: "nope" });
+ expect(unknownChannel.status).toBe(400);
+ expect(unknownChannel.body.error).toContain('Channel "nope" not found');
+});
+
+test("refresh-report regenerates snapshot.json", async ({ request }) => {
+ test.setTimeout(120_000);
+ await resetData("title-filter-channel");
+ await settings();
+ const SLUG = "test-filter";
+ const SNAP = `test-transcripts/channels/${SLUG}/snapshot.json`;
+ // Removed rather than assumed absent: a debounced regen armed by the previous
+ // spec can land between resetData's copy and here, and the point of the
+ // assertion below is the ROUTE's effect, not the scheduler's.
+ await rm(resolvePath(SNAP), { force: true });
+ expect(await pathExists(SNAP)).toBe(false);
+
+ // No page, no click: the route IS the refresh. It regenerates SYNCHRONOUSLY
+ // (a filesystem scan, not a job), so { ok: true } means the file is there.
+ const first = await ops(request, "refresh-report", { slug: SLUG });
+ expect(first.body).toEqual({ ok: true });
+ const snapshot = await readJson<{ generatedAt: string; totals: { videos: number } }>(SNAP);
+ expect(snapshot.generatedAt).toBeTruthy();
+ expect(snapshot.totals.videos).toBeGreaterThanOrEqual(0);
+
+ // Re-running REWRITES it. Polled through the action itself because two scans
+ // of a six-video fixture can land in the same millisecond.
+ await expect
+ .poll(async () => {
+ await ops(request, "refresh-report", { slug: SLUG });
+ return (await readJson<{ generatedAt: string }>(SNAP)).generatedAt;
+ })
+ .not.toBe(snapshot.generatedAt);
+
+ // The bulk form queues a job per channel and reports both lists.
+ const all = await ops(request, "refresh-report", { all: true });
+ expect(all.status).toBe(200);
+ expect(all.body.ok).toBe(true);
+ expect([...(all.body.queued ?? []), ...(all.body.skipped ?? []).map((s) => s.slug)]).toContain(
+ SLUG,
+ );
+
+ const both = await ops(request, "refresh-report", { slug: SLUG, all: true });
+ expect(both.status).toBe(400);
+ expect(both.body.error).toMatch(/not both/);
+});
+
+test("relocate refuses a busy channel with the sentence the panel shows", async ({
+ request,
+}, testInfo) => {
+ test.setTimeout(120_000);
+ await resetData("slow-pipeline-channel");
+ await settings();
+ const SLUG = "slow-channel";
+
+ // --test-slow makes the fake yt-dlp sleep 30s, so the channel is genuinely
+ // busy for the length of this assertion rather than racily so.
+ const started = await ops(request, "sync", { slug: SLUG });
+ expect(started.body.ok).toBe(true);
+ await expect
+ .poll(async () => {
+ const res = await request.get(`${baseUrl}/api/jobs/active`);
+ const body = (await res.json()) as
+ | { channelSlug?: string; status: string }[]
+ | { jobs?: { channelSlug?: string; status: string }[] };
+ const jobs = Array.isArray(body) ? body : (body.jobs ?? []);
+ return jobs.filter(
+ (j) =>
+ j.channelSlug === SLUG &&
+ (j.status === "running" || j.status === "queued"),
+ ).length;
+ })
+ .toBeGreaterThan(0);
+
+ const root = testInfo.outputPath("media-root");
+ const refused = await ops(request, "relocate", { slugs: [SLUG], root });
+ expect(refused.status).toBe(200);
+ // A SKIP IS NOT A FAILURE — the bulk bar renders both numbers, and so does
+ // this. The reason is channelMediaBusyReason's, word for word.
+ expect(refused.body.queued).toEqual([]);
+ expect(refused.body.skipped?.[0]?.slug).toBe(SLUG);
+ expect(refused.body.skipped?.[0]?.reason).toMatch(
+ /running\/queued job\(s\) for this channel/,
+ );
+
+ // Exactly one destination, and it must be absolute.
+ const neither = await ops(request, "relocate", { slugs: [SLUG] });
+ expect(neither.status).toBe(400);
+ expect(neither.body.error).toMatch(/exactly one of "locationId".*or "root"/);
+});
+
+test("lane flips a hold, and /api/auto-queue/status agrees", async ({
+ request,
+}) => {
+ await resetData("empty");
+ await settings();
+
+ const held = async (): Promise<boolean> => {
+ const res = await request.get(`${baseUrl}/api/auto-queue/status`);
+ const body = (await res.json()) as Record<string, { held?: boolean }>;
+ return body.download?.held === true;
+ };
+ expect(await held()).toBe(false);
+
+ const hold = await ops(request, "lane", { lane: "download", held: true });
+ expect(hold.body).toEqual({ ok: true, lane: "download" });
+ await expect.poll(held).toBe(true);
+
+ const release = await ops(request, "lane", { lane: "download", held: false });
+ expect(release.body).toEqual({ ok: true, lane: "download" });
+ await expect.poll(held).toBe(false);
+
+ // `enabled` is the policy's master switch and is NOT the gate — two controls
+ // in the UI, two keys here.
+ const enable = await ops(request, "lane", { lane: "download", enabled: true });
+ expect(enable.body.ok).toBe(true);
+ await expect
+ .poll(async () => {
+ const s = await readJson<{
+ autoQueue?: Record<string, { enabled?: boolean }>;
+ }>("test-settings.json");
+ return s.autoQueue?.download?.enabled ?? null;
+ })
+ .toBe(true);
+ await ops(request, "lane", { lane: "download", enabled: false });
+
+ const nothing = await ops(request, "lane", { lane: "download" });
+ expect(nothing.status).toBe(400);
+ expect(nothing.body.error).toMatch(/nothing to do/);
+
+ const bogus = await ops(request, "lane", { lane: "transcode", held: true });
+ expect(bogus.status).toBe(400);
+ expect(bogus.body.error).toMatch(/transcription, download, digest, backfill/);
+});
diff --git a/editor/e2e/storage-locations.spec.ts b/editor/e2e/storage-locations.spec.ts
@@ -1,8 +1,18 @@
-import { mkdir, readlink, rename, rm, symlink, writeFile } from "node:fs/promises";
+import {
+ mkdir,
+ readFile,
+ readlink,
+ rename,
+ rm,
+ symlink,
+ writeFile,
+} from "node:fs/promises";
import { join } from "node:path";
import { test, expect, type Page } from "@playwright/test";
import { baseUrl } from "./baseUrl";
import {
+ generateReport,
+ pathExists,
readJson,
resetData,
resolvePath,
@@ -285,3 +295,212 @@ test("a volume that came up somewhere else is re-pointed in one click", async ({
"1 ok / 0 unreachable / 0 moving",
);
});
+
+// --- THE CORPUS VOLUME, AND THE SAVED-VIDEO STORE ---------------------------
+
+test("the corpus volume is a row of its own, with nothing to configure", async ({
+ page,
+}, testInfo) => {
+ test.setTimeout(120_000);
+ await resetData("one-youtube-channel-with-data");
+ const root = testInfo.outputPath("platter");
+ await mkdir(root, { recursive: true });
+ await writeSettings({
+ adminTitle: "Test Admin",
+ minFreeDiskGB: 0,
+ storage: {
+ locations: [{ id: "cold", label: "Cold", root, autoRepoint: false }],
+ defaultLocationId: "cold",
+ },
+ });
+ // The sizes come off the reports, so the row can only be honest after one.
+ await generateReport(page, SLUG);
+
+ await page.goto("/storage");
+ const internal = row(page, "internal");
+ await expect(internal).toBeVisible();
+ // IT IS FIRST. 523 GB on a disk with 67 left is not a footnote under the
+ // locations that were added to fix it.
+ const ids = await page
+ .getByLabel(/^storage location: /)
+ .evaluateAll((els) =>
+ els.map((e) => e.getAttribute("aria-label")?.replace("storage location: ", "")),
+ );
+ expect(ids[0]).toBe("internal");
+
+ await expect(internal.getByLabel("location status")).toHaveText("Available");
+ await expect(internal.getByLabel("location channels")).toContainText("1 ok");
+ await expect(internal.getByLabel("location media bytes")).not.toHaveText("—");
+ await expect(internal.getByLabel("location media bytes")).not.toContainText(
+ "size unknown",
+ );
+ // It links to its own list, largest first.
+ await expect(internal.getByLabel("list channels on internal")).toHaveAttribute(
+ "href",
+ "/channels?location=internal&sort=size",
+ );
+ // NOTHING TO REFRESH, RE-POINT, MOUNT, EDIT OR DELETE — and the sentence is
+ // said once rather than as five greyed buttons.
+ await expect(internal.getByLabel("refresh internal")).toHaveCount(0);
+ await expect(internal.getByLabel("delete internal")).toHaveCount(0);
+ await expect(internal.getByLabel("internal note")).toContainText(
+ "not a configured location",
+ );
+
+ // A location with NOTHING on it is genuinely empty, and says so plainly —
+ // "0 B" is a measurement here, not a missing one. (The "size unknown" wording
+ // is for a location whose channels have never had a report; that case is
+ // pinned by views/storage.test.ts, which does not need a browser.)
+ const cold = row(page, "cold");
+ await expect(cold.getByLabel("location channels")).toHaveText(
+ "0 ok / 0 unreachable / 0 moving",
+ );
+ await expect(cold.getByLabel("location media bytes")).toHaveText("0 B");
+ // Nothing on it, so it has no list to link to.
+ await expect(cold.getByLabel("list channels on cold")).toHaveCount(0);
+});
+
+test("the saved-video store moves onto a location, and back", async ({
+ page,
+}, testInfo) => {
+ test.setTimeout(180_000);
+ await resetData("one-youtube-channel-with-data");
+ const root = testInfo.outputPath("platter");
+ await mkdir(root, { recursive: true });
+ // Something to move. The store is <root>/<slug>/<videoId>/<container>.
+ const storeDir = resolvePath("test-transcripts/saved-videos");
+ await mkdir(join(storeDir, SLUG, VIDEO), { recursive: true });
+ await writeFile(
+ join(storeDir, SLUG, VIDEO, "source-media.mp4"),
+ "not really an mp4",
+ );
+ await writeSettings({
+ adminTitle: "Test Admin",
+ minFreeDiskGB: 0,
+ storage: {
+ locations: [{ id: "cold", label: "Cold", root, autoRepoint: false }],
+ defaultLocationId: "cold",
+ },
+ });
+
+ await page.goto("/storage");
+ const card = page.getByLabel("saved video store");
+ await expect(card).toBeVisible();
+ await expect(card.getByLabel("saved videos status")).toHaveText("In place");
+ await expect(card.getByLabel("saved videos location")).toHaveText(
+ "Internal (in place)",
+ );
+ await expect(card.getByLabel("saved videos bytes")).toContainText("1 file(s)");
+
+ // --- out ----------------------------------------------------------------
+ // NO selectOption: the only destination a store in place can go to is the one
+ // configured location, and the select opens on it. An option that would be a
+ // no-op is never offered (see savedVideosView), which is what makes the
+ // default correct in both directions here.
+ await expect(card.getByLabel("saved videos destination")).toHaveValue("cold");
+ await page.getByRole("button", { name: "Move the store" }).click();
+ await expect(page.getByLabel("Move the store output")).toContainText("Moved", {
+ timeout: 60_000,
+ });
+
+ // A symlink where the store was, the bytes on the "drive", and the record
+ // written only on success.
+ const target = join(root, "saved-videos");
+ expect(await readlink(storeDir)).toBe(target);
+ const after = await readJson<{
+ storage: { savedVideosLocationId?: string };
+ }>("test-settings.json");
+ expect(after.storage.savedVideosLocationId).toBe("cold");
+
+ // EVERY READER KEEPS WORKING, THROUGH THE LINK, UNCHANGED — which is the
+ // whole claim the symlink design rests on, and it is a filesystem fact, not a
+ // page one. (`/saved-videos` lists POINTER sidecars written into each video
+ // dir by the retention rule, not the store's contents, so a store seeded
+ // directly has nothing for it to list — that page is not the reader this
+ // proves.)
+ expect(
+ await readFile(
+ join(storeDir, SLUG, VIDEO, "source-media.mp4"),
+ "utf8",
+ ),
+ ).toBe("not really an mp4");
+
+ // --- back ---------------------------------------------------------------
+ await page.goto("/storage");
+ const card2 = page.getByLabel("saved video store");
+ await expect(card2.getByLabel("saved videos status")).toHaveText(
+ "Relocated · reachable",
+ );
+ await expect(card2.getByLabel("saved videos location")).toHaveText("Cold");
+ // And on the way back the only destination is the corpus volume, for the same
+ // reason: a location-to-location move of the store is not offered.
+ await expect(card2.getByLabel("saved videos destination")).toHaveValue("");
+ await page.getByRole("button", { name: "Move the store" }).click();
+ await expect(page.getByLabel("Move the store output")).toContainText(
+ "Moved back",
+ { timeout: 60_000 },
+ );
+ const back = await readJson<{
+ storage: { savedVideosLocationId?: string };
+ }>("test-settings.json");
+ expect(back.storage.savedVideosLocationId).toBe(undefined);
+ // The target was reclaimed, and the store is a real directory again.
+ expect(await pathExists("test-transcripts/saved-videos/" + SLUG)).toBe(true);
+});
+
+// A KILLED COPY LEAVES A MARKER NOTHING WILL EVER CLEAR, and without Resume the
+// only fix is deleting a dotfile over SSH.
+test("an interrupted store move is resumable from the page", async ({
+ page,
+}, testInfo) => {
+ test.setTimeout(120_000);
+ await resetData("one-youtube-channel-with-data");
+ const root = testInfo.outputPath("platter");
+ await mkdir(root, { recursive: true });
+ const storeDir = resolvePath("test-transcripts/saved-videos");
+ await mkdir(join(storeDir, SLUG, VIDEO), { recursive: true });
+ await writeFile(
+ join(storeDir, SLUG, VIDEO, "source-media.mp4"),
+ "not really an mp4",
+ );
+ await writeSettings({
+ adminTitle: "Test Admin",
+ minFreeDiskGB: 0,
+ storage: {
+ locations: [{ id: "cold", label: "Cold", root, autoRepoint: false }],
+ defaultLocationId: "cold",
+ },
+ });
+ // What a killed copy leaves behind: a partial target and a phase-copy marker.
+ const target = join(root, "saved-videos");
+ await mkdir(join(target, SLUG, VIDEO), { recursive: true });
+ await writeFile(
+ resolvePath("test-transcripts/.relocating-saved-videos.json"),
+ JSON.stringify(
+ {
+ target,
+ direction: "out",
+ startedAt: new Date().toISOString(),
+ phase: "copy",
+ },
+ null,
+ 2,
+ ),
+ );
+ await fetch(`${baseUrl}/api/test/invalidate-cache`).catch(() => {});
+
+ await page.goto("/storage");
+ const card = page.getByLabel("saved video store");
+ await expect(card.getByLabel("saved videos status")).toHaveText(
+ "Move in flight",
+ );
+ await page.getByRole("button", { name: "Resume move" }).click();
+ await expect(page.getByLabel("Resume move output")).toContainText(
+ "resumed an interrupted move",
+ { timeout: 60_000 },
+ );
+ expect(await readlink(storeDir)).toBe(target);
+ expect(
+ await pathExists("test-transcripts/.relocating-saved-videos.json"),
+ ).toBe(false);
+});
diff --git a/editor/instrumentation.ts b/editor/instrumentation.ts
@@ -108,6 +108,25 @@ export async function register() {
// Everything past here STARTS work. On an idle boot, nothing does.
if (idle) return;
+ // THE DRIVE WATCH. Every guard the corpus has for an unreachable channel runs
+ // at the start of a piece of work — none of them is a DETECTOR, so a channel
+ // whose drive vanished sits there being refused with nothing saying why. This
+ // is the thing that looks, on a five-minute cadence, and auto-pauses (and
+ // later restores) the channels on a location that is not there.
+ //
+ // BELOW THE IDLE GATE, deliberately, and unlike the boot probe above: this
+ // one WRITES settings.channelPriority, and a container pointed at somebody
+ // else's corpus for the first time has no business rewriting that corpus's
+ // priority document. See common/controller/storageWatch.ts.
+ try {
+ const { startStorageWatch } = await import(
+ "yt-dlp-transcript-common/controller/storageWatch"
+ );
+ startStorageWatch({ log: (line) => console.log(line) });
+ } catch {
+ /* a watch that fails to arm must not block server readiness */
+ }
+
// Start the automatic priority-queue runners — all four lanes — if their
// policies are enabled. Each is a self-managed registry job; this only
// kicks them off and returns. Best-effort — a failure here must not stop the
diff --git a/package.json b/package.json
@@ -22,7 +22,8 @@
"wt": "node scripts/worktree.mjs",
"e2e:sharded": "node scripts/run-sharded-e2e.mjs",
"test:scripts": "node --test scripts/*.test.mjs umtool/report-to-video/*.test.mjs",
- "lint": "pnpm --filter export run lint"
+ "lint": "pnpm --filter export run lint",
+ "ops": "node scripts/archilyzer-ops.mjs"
},
"devDependencies": {
"tsx": "^4.21.0"
diff --git a/plans/FACTS.md b/plans/FACTS.md
@@ -4227,13 +4227,291 @@ umtool asks the editor for a clip window instead of running yt-dlp
the cache misses forever.
- **`clips/` HAS NO GARBAGE COLLECTION, and its bytes are counted by nothing.**
Nothing prunes a window once it is fetched (the retention sweep is
- pointer-driven and `pruneSavedVideos` never sees it), and the snapshot's
- `totalMediaBytes` walk is flat over a video dir, so a `clips/` subdirectory
- adds to the disk without adding to the number the operator reads. Deliberate
- for now — a window is seconds, not a recording — but a channel walked many
- times by many reports will accumulate them silently. A later sweep owes both:
- a count on the storage page and an eviction rule.
+ pointer-driven, so `pruneSavedVideos` never sees one), and
+ `channelSnapshot.ts` (the `mediaBytes` loop, ~:773) does `if (!st.isFile())
+ continue` — its comment says "a video dir is flat", **which this feature makes
+ untrue**. So a `clips/` subdirectory adds to the platter without adding to
+ `snapshot.totalMediaBytes`, which is the number the storage page reads and the
+ number a relocation estimate is drawn from — while `rsync -a` of the data dir
+ moves the windows regardless. Deliberate for now (a window is seconds, not a
+ recording) but it is a real under-count, and a channel walked many times by
+ many reports accumulates them silently. A later sweep owes three things: the
+ recursion or an explicit clips total, a count on the storage page, and an
+ eviction rule.
- **`SavedVideoPointer.origin` is separate from `keepReason` on purpose.**
`keepReason` stays `override`/`pin` so `pruneSavedVideos` never evicts a
container somebody asked for; `origin` is who asked. `parseSavedVideoPointer`
is tolerant, so a legacy pointer parses unchanged (`clipWindow.test.ts`).
+
+---
+
+## Storage locations S5–S6 (verified 2026-09-20, branch `storage/locations-s5-s6`)
+
+Six things, in the order they were built. Trust these over re-deriving them.
+
+### Bytes: `snapshot.totalMediaBytes`
+
+- **EVERY byte under `data/<id>/`**, not the audio. `totalAudioBytes` is a
+ different question (what a CLEANUP could reclaim); this is what a volume holds
+ and what a move carries, and on a real channel the two differ by the whole
+ transcript/cues/metadata mass.
+- Gathered in the EXISTING per-video walk in `channelSnapshot.ts` (~746): the
+ `audioSizes` loop became one loop over `files.entries`, statting each, feeding
+ both numbers. No second walk; the added cost is a stat per non-audio entry,
+ warm inode cache, on a pass that already reads several sidecars per video.
+ Sub-directories count as nothing rather than recursing — a video dir is flat.
+- **OPTIONAL, and the distinction is load-bearing.** A snapshot written before
+ the field lacks it, and every reader must render that as "size unknown until
+ Refresh report", NEVER as 0: a zero ranks a 400 GB channel bottom of the very
+ list the operator opened to find it. `LocationRollup` therefore carries
+ `bytes` AND `unknownBytes`, and `storageBytesText(bytes, unknown)` in
+ `views/storage.ts` is the one wording.
+
+### The corpus volume as a row: `INTERNAL_LOCATION_ID = "internal"`
+
+- **SYNTHETIC, never stored in `settings.storage.locations`.** There is nothing
+ to configure (the root is wherever the corpus is), nothing to re-point, and —
+ the load-bearing reason — a stored entry would make `locationOfDataDir` match
+ every unrelocated channel, breaking the rule the whole design rests on: a
+ channel is on location L iff its `dataDir` is under `L.root`, and an in-place
+ channel HAS no `dataDir`.
+- Assembled in `views/storage.ts` (`internalRow`) from `StorageRowsInputs.internal
+ = { root, freeBytes? }`, which the shell fills. It sorts FIRST, is
+ `available` by construction, and every one of its five actions is present with
+ `offered: false` and the same sentence — a greyed button with no reason is
+ what that page exists not to be.
+- `channelsOnLocation({ includeInternal: true })` rolls the no-`dataDir` channels
+ under that id. A `dataDir` under a root NOBODY named is counted in NEITHER; the
+ nudge is to name that root on /storage.
+- `StorageRowsPayload` exposes `bytesOnLocation` (by row id, internal included)
+ and `bytesInPlace`.
+
+### The Storage panel has ONE verb
+
+- `MoveOut`/`MoveBack` folded into `MoveMedia` (`StorageStage.tsx`). The corpus
+ volume is a destination (`__internal`) and picking it calls
+ `moveChannelMediaBackAction` — unchanged, by that name, so the `/api/ops/*`
+ routes on `feat/ops-api` keep working.
+- While the media IS on a location, `__internal` is the ONLY option offered: the
+ controller refuses a location-to-location move by name ("already relocated to
+ X. Move it back in place first."), and a select of options that all refuse is
+ worse than one that works plus a sentence. No preview gate on the way back.
+- One log, one accessible name: `getByLabel("Move media output")` for both
+ directions. The button LABEL still reads "Move back in place" when the
+ destination is internal.
+
+### `/channels` is the storage working surface
+
+- Two columns: **Location** (`aria-label="media location for <slug>"`, the
+ volume label with the existing reachability badge beside it) and **Size**
+ (`aria-label="media size for <slug>"`, sort key `size`, first click biggest
+ first, "—" and never "0 B" when unmeasured). Sort keys `location` and `size`
+ joined `SortKey`; `colSpan` went 10 → 12 + columns.
+- **Free space is NOT a column.** It is a fact about a disk; 71 copies of one
+ number is what the focus bar already taught us not to do. It lives in
+ `ChannelVolumeBar` (`aria-label="storage volumes"`), one chip per volume
+ (`aria-label="volume <id>"`), and the chip IS the filter.
+- **`?location=<id|internal>`**, a URL param beside `?site=`, validated on the
+ server against what is on screen (a stale /storage link must show the page,
+ not an empty table). A volume filter FLATTENS the grouped render: a group is a
+ partition of a site, and half a group is not a group.
+- **`?sort=size`** seeds the sort ONCE. Sorting is otherwise client state,
+ deliberately — a `router.replace` races the global AutoRefresh's
+ `router.refresh()` and gets dropped.
+- **"Free up N GB"** lives in the volume bar, not the deck: the deck only exists
+ once something is ticked, and this is the control that does the ticking. The
+ deck shows the resulting note (`aria-label="free up selection note"`). The
+ rule is pure in `common/views/freeUpSelection.ts`: only in-place channels
+ (moving one already on the platter frees nothing on the disk being emptied),
+ unmeasured ones excluded AND COUNTED, largest first, last pick overshoots.
+- **Per-volume free space is `volumeFreeBytes` (controller), two syscalls per
+ root and never a probe.** The `stat` is what stops `getFreeBytes` reporting an
+ unmounted root's PARENT volume — which on this machine is the disk being
+ emptied. Tables do not shell out.
+
+### Relocate progress
+
+- `rsync --info=progress2` was ALREADY on the copy; every frame went to the job
+ log as a carriage-return redraw. `parseRsyncProgress` (jobs/progressParsers.ts)
+ reads one frame — `30,000,000 85% 1.03GB/s 0:00:03 (xfr#1, to-chk=1/3)` —
+ into `{bytes, percent, rate, etaSeconds}`. Grouping separators are stripped
+ with `\D`, not assumed to be commas.
+- **The fraction divides by the MEASURED tree, not by rsync's percentage.** Under
+ incremental recursion (the default) rsync's percentage is of what it has
+ enumerated so far and walks BACKWARDS. The controller already measured the
+ whole tree for its space check.
+- `relocateChannelMedia`/`relocateSavedVideos` take `onProgress`, keep frames out
+ of the log, and write one decile line: `Copying… 12.3 GB of 45.6 GB · 27 % ·
+ 110.50MB/s · ETA 5:32`. Same wording in the job's task detail.
+- The job publishes it as a **`JobTask` of kind `"relocate"`** (new member of
+ `JobTaskKind`; the two `Record<JobTaskKind, …>` tables in
+ `JobProgressBars.tsx` and `MonitorWidget.tsx` are the compile errors that
+ catch a third). NOT a `JobProgress` metric — that counts artifacts
+ re-countable from disk, and a copy in flight is neither. The task is added on
+ the FIRST FRAME, not at start: the preflight and a resumed run's verify pass
+ transfer nothing.
+
+### `relocateDir.ts` and the movable saved-video store
+
+- `common/controller/relocateDir.ts` is `relocateChannelMedia.ts`'s core lifted
+ out UNCHANGED: `measureTree`, `linkOrDirState`, the marker read/write/clear,
+ `rsyncTree` (takes `rsyncBin`, not `Paths`), `COPY_ARGS`, `verifyCopy` (dry run
+ + re-measure, one `.d..t` retry) and `makeProgressSink`. What stayed behind is
+ everything channel-specific. The channel mover's 24 tests pass against it.
+- `relocateSavedVideos({ paths, locationId, io? })`: `""` moves back.
+ `<store>` becomes a symlink to `<root>/saved-videos`;
+ `savedVideoRoot()` (lib/savedVideo.ts) is untouched and every reader follows
+ the link. Marker: `transcripts/.relocating-saved-videos.json`, the SAME
+ `{target, direction, startedAt, phase}` shape a channel's uses.
+- **`settings.storage.savedVideosLocationId`** is a RECORD of where bytes are,
+ not a preference — so unlike `defaultLocationId` it is NOT fallen back to
+ another location when the named one is deleted; it sanitizes to `""`.
+- `ChannelConfig.savedVideosDir` (the per-channel override) is neither read nor
+ rewritten: it is an absolute path the operator set.
+- Job kind `relocate-saved-videos`, in `NO_REGEN_KINDS`, on
+ `relocationQueueKey()` with the channel move and the re-point. A store move or
+ a re-point freezes every /storage row (`runningRepoint` checks both kinds); a
+ CHANNEL move deliberately does not.
+- `/storage` grows `SavedVideosStoreCard` — size, location, Move / Resume /
+ Clear marker. The store is WALKED (`measureTree`) because it holds one
+ container per pinned or kept-latest video, not one per video; if it ever grows
+ to corpus scale, `listSavedVideos` (pointers carry their own `bytes`) is the
+ cheaper answer waiting.
+
+### The drive watch (S5)
+
+- `common/controller/storageWatch.ts`. Every existing guard runs at the START of
+ a piece of work; none is a DETECTOR, so a channel on a vanished drive sits
+ being refused with nothing saying why. `runStorageWatchPass` probes every
+ location (`refresh: true` — the 10 s memo is for page renders), auto-pauses the
+ channels it cannot reach and restores them when it can.
+ `startStorageWatch()` arms it at `STORAGE_WATCH_INTERVAL_MS` (5 min).
+- **`ChannelPriorityEntry.autoPaused = {reason: "storage", since, previousTier}`.**
+ `autoPauseForMedia` no-ops on an already-auto-paused channel (a flapping drive
+ must not overwrite `previousTier` with `paused`) and on one the OPERATOR
+ paused. `restoreAfterMedia` no-ops without a record. `clearAutoPause` is called
+ by the one priority writer on every manual tier change — the operator's word
+ always wins.
+- **The sanitizer now KEEPS a rank on an auto-paused channel.** It still drops
+ one from a channel paused everywhere, which is right for a pause the operator
+ meant and catastrophic for a temporary one: an unplugged cable would otherwise
+ destroy the queue order and restore the channel unranked.
+- **THE PASS PAYS THE LEGACY SEED.** `laneDispatchRoot` is all-or-nothing on
+ `isDefaultChannelPriority`, so auto-pausing one channel on a corpus with an
+ empty document would switch it off its hand-made lane trees. Same condition and
+ same remedy as `saveChannelPriorityAction` (S2/S3 review, finding 3), and the
+ lanes are recompiled in the SAME `writeSettings`.
+- Writes at most once per pass, and only on a transition — a quiet pass bumps no
+ pulse revision. `in-transition` (a relocation marker) is never a reason to
+ pause: the relocate job is what an auto-pause would be refusing.
+- Armed in `instrumentation.ts` BELOW the idle gate, unlike the boot probe above
+ it: the probe is read-only, the WRITE is work, and `ARCHILYZER_IDLE_BOOT`
+ refuses work.
+- The flag: `autoPauseReasonOf` is the one sentence, rendered as a "storage" chip
+ by `ChannelTierSelect` (`aria-label="auto-paused reason for <slug>"`).
+
+### Review fixes, 2026-09-20 — the five that were not optional
+
+- **An auto-pause/restore cycle used to destroy per-operation overrides.**
+ `sanitizeOverrides` normalises away any override equal to the BASE tier, and
+ while the machine's pause stands the base on the entry is the FORCED
+ `paused` — so every `{op:"paused"}` fence stopped being an exception and was
+ deleted. The record is now computed BEFORE the overrides and they are
+ normalised against `previousTier`. Test: a pause→restore round trip preserving
+ tier, rank AND overrides.
+- **`common/lib/savedVideoStore.ts` is the store's guard, and it is in lib/ on
+ purpose.** `savedVideo-server.ts` is what persists a container, it is lib, and
+ lib may not import controller — so the marker's name, its reader and
+ `assertSavedVideosStoreWritable` live there and the controller that WRITES the
+ marker imports them. `persistSourceVideo`/`unpersistSavedVideo` take an
+ optional `paths` and consult it; `downloadOneManaged` passes it. A caller that
+ omits it opts out, which is right for a per-channel store elsewhere.
+ `editor/app/storage/lib/storeBusy.ts` is the courtesy half — a sentence before
+ the operator commits — and its kind list is deliberately NOT exhaustive,
+ because the marker is the guard.
+- **The swap links defensively and records only once the link reads back.**
+ `moveFileCrossDevice` does an unconditional `mkdir -p`, so a persist between
+ the rename and the symlink recreates `saved-videos` as a real dir; an EMPTY
+ one is removed and linked, a non-empty one refuses, and
+ `savedVideosLocationId` is never written without a link.
+- **Two consecutive down passes before the watch pauses; one up pass to
+ restore.** Availability is a bare `stat` with a blanket catch, so EIO or a
+ spun-down disk reads as "not mounted". The pending count is MODULE state, not
+ settings (`resetStorageWatchSuspicion()` is the test seam). The watch also
+ calls `maybeAutoRepoint` now, so a drive that comes up elsewhere while the
+ editor is running no longer waits for a reboot.
+- **Move-back deletes the target only when the run can vouch for the local
+ copy** — it did the swap, the marker says a previous run got past it
+ (`phase: "reclaim"`), or `measureTree(store) >= measureTree(target)`.
+ Otherwise it finishes, clears the marker and says what it did not delete: "a
+ real directory" is also what `rsync --copy-links` produces.
+- Smaller, same commit: a resumed copy's bar is offset by `measureTree(dest)`
+ taken before the copy (rsync counts only what IT sent, so a resume topped out
+ at 40 %); `rsyncTree` buffers the trailing segment across chunks and flushes it
+ at exit, so a torn frame never parses as `bytes=0`; the parked name is
+ `saved-videos.relocated-<ts>` swept by prefix; a store that never existed is
+ created empty rather than failing rsync 23 behind a stuck marker; both sides of
+ the containment check are realpath-resolved; and deleting a location the store
+ is on is refused in the action AND withheld on the row.
+
+---
+
+## The ops API (`/api/ops/*`) — added 2026-09-20
+
+**It is adapters, and nothing else.** Each route under
+`editor/app/api/ops/<action>/route.ts` is ~5 lines: validate a JSON body, call
+ONE existing server action, map its result. No route contains a rule the UI does
+not already enforce — the point of the layer is that an agent over HTTP and an
+operator clicking the same button get the same refusal, with the same sentence,
+from the same code. A check written in a route would be a second opinion nobody
+maintains. `editor/app/api/ops/_lib.ts` holds auth, body parsing and the three
+result mappers (`jobResponse` / `actionResponse` / `queueResponse`).
+
+**`WORKER_TOKEN` is shared with `/api/worker/*` by design.** It already means
+"this instance accepts instructions from something that is not the browser in
+front of it", and the failure modes are identical, so the gate is: unset → 503
+(the surface is off until you opt in), wrong → 401. `authorizeWorkerRequest`
+(`common/lib/workerToken.ts`) is the single implementation; nothing new was
+written. A second secret would be a second thing to distribute, rotate and leave
+unset.
+
+**A job-starting route returns `{ ok: true, jobId }` and NEVER streams.**
+`runManagedFunction` hands back a `ReadableStream` the browser consumes; an HTTP
+caller wants to hang up and poll. Every job adapter calls `result.stream.cancel()`
+— which stops pushing into the controller and leaves the on-disk log running
+(`streamCommand.ts`'s `cancel()` note) — and the caller follows
+`/api/jobs/<id>/log`. Returning the id is also the honest answer: the queue may
+hold the job behind other work for hours, so "started" is not "running".
+
+**Unknown body keys are a 400, never a silent ignore.** The allow-list passed to
+`ops()` IS the route's documented body shape. A caller that misspells
+`downloadFilterExclude` would otherwise get `{ ok: true }` and a channel that
+still downloads everything.
+
+**`ops/channel-config`'s patch keys are the FORM's field names, not
+`ChannelConfig`'s** — `downloadFilterInclude` / `downloadFilterExclude` rather
+than a `downloadFilter` object, `ytdlpExtraArgs` as a string or an array of
+lines. That is what routes them through `parseChannelForm`'s validators. The
+patch is laid over the channel's CURRENT form representation
+(`editor/app/channels/components/channelConfigToForm.ts`) rather than posted
+alone, because `updateChannelAction` deletes every `CHANNEL_FORM_FIELDS` key from
+the stored config before layering the parse result on — a FormData carrying only
+a patch would clear everything the patch did not name.
+
+**`GET /api/ops/channel/<slug>`'s live counts are opt-in (`?counts=1`).**
+`readChannelStat` walks every video directory — eleven thousand readdirs on the
+largest channel here — so a poll loop asking for a channel's state would be the
+thing hammering the platter. The report's totals answer the same question from a
+file. The field is `counts` and is null without the flag.
+
+**No action needed a refactor to be callable from a route.** `revalidatePath` is
+supported in Route Handlers (Next 16 —
+`docs/01-app/03-api-reference/04-functions/revalidatePath.md:10`), no adapted
+action calls `cookies()`, and the only actions that `redirect()`
+(`createChannelAction`, `gotoVideoAction`) are deliberately not exposed.
+
+**The 503-when-unset branch is not covered by e2e.** The editor test server runs
+with `WORKER_TOKEN=test-worker-token` in `editor/package.json`'s `dev:test`, one
+server for the whole suite, so no spec can observe the disabled state.
+`editor/e2e/ops-api.spec.ts` covers 401 (missing and wrong); the 503 is
+`authorizeWorkerRequest`'s own first branch, shared with `/api/worker/*`.
diff --git a/plans/STATE.md b/plans/STATE.md
@@ -3,7 +3,23 @@
The working memory for the local-AI derived-corpus work. Rewritten at the end of every
session, before context is cleared. See [`README.md`](README.md) for the protocol.
-**Last updated:** 2026-09-15 (Phase 3 slice 1 and the rack landed; see the Next block) — 2026-09-13: **gate A passed and `main` moved; the interlude shipments and one-core Phase 2 are all merged
+**Last updated:** 2026-09-20 — **storage locations S5 + S6 shipped** on branch
+`storage/locations-s5-s6` (worktree `/home/user/Projects/storage-locations-s5-s6`, off `main`
+@ `60183f0`, unmerged). The operator's two asks that evening — *"disk space and current storage
+volume as columns on the channels menu, and let me filter by volume"* and *"progress on
+relocate jobs since rsync gives progress"* — plus the four things around them: per-channel
+media bytes in the snapshot (`totalMediaBytes`), the corpus volume as a first-class `/storage`
+row (`internal`, synthetic), the saved-video store made movable (`relocateDir.ts` is the
+channel mover's factored core; `relocateSavedVideos` is new), and a five-minute drive watch
+that auto-pauses a channel whose drive went away and restores its tier when it comes back
+(`ChannelPriorityEntry.autoPaused`). Slice record and what was deliberately left out:
+[`storage-locations.md`](storage-locations.md#s6--the-storage-surfaces-the-operator-asked-for-shipped-2026-09-20).
+Seams: [`FACTS.md`](FACTS.md#storage-locations-s5s6-verified-2026-09-20-branch-storagelocations-s5-s6).
+**Left out, named:** `assertRelocationRootPresent` (a move can still `mkdir` under an absent
+mount), a `/review` section for auto-paused channels, and the per-unit reachability re-check in
+the two lane runners.
+
+**Previously:** 2026-09-15 (Phase 3 slice 1 and the rack landed; see the Next block) — 2026-09-13: **gate A passed and `main` moved; the interlude shipments and one-core Phase 2 are all merged
on one branch**, `integrate/2026-09-storage-priority`, tip **`bd3d4ec`**, unmerged. Off
`e74f005`: `relocate-channel-media` (from `storage/relocate-media`), then `channel-priority`
(from `channel-priority/s5`), then Phase 2's five slices in the order their reviews cleared —
diff --git a/plans/storage-locations.md b/plans/storage-locations.md
@@ -1,6 +1,7 @@
# Storage locations: named, refreshable, re-pointable places a channel's media lives
-Status: **S0–S4 shipped 2026-09-19** (`4b55e3c`); S5 written, not started. Facts pinned to `main` @ `6b4f25b`. Slices ship on
+Status: **S0–S4 shipped 2026-09-19** (`4b55e3c`); **S5 + S6 shipped 2026-09-20** on
+`storage/locations-s5-s6` (unmerged) — see the two status blocks at the foot of this file. Facts pinned to `main` @ `6b4f25b`. Slices ship on
`storage/locations-s{0,1,2,3,4}` branches; this file is updated as each lands.
## Context
@@ -317,5 +318,51 @@ The S3/S4 gate list. Full suite on the final tip. Offline against the real host:
### Status
+- 2026-09-20 — **S5 and S6 shipped** on `storage/locations-s5-s6` (unmerged). What landed
+ differs from the design above in three ways, all deliberate:
+ - **(a) is a watch, not a per-unit re-check.** The plan put an `inspectChannelMedia` beside
+ the marker check in `autoRunner.run(picked)` and `operationBatch`'s GUARD 5. The operator's
+ ask was "an automatic disable and flag", and a flag is a STATE — a per-unit refusal
+ produces none, it just declines work more often. `common/controller/storageWatch.ts` is a
+ five-minute pass that probes, auto-pauses and restores; the existing start-of-work guards
+ are untouched, and the per-unit re-check is still available as a follow-up if a lane is ever
+ seen writing into a dangling link.
+ - **(c) is `reason: "storage"`, not `"media-unreachable"`**, and it lives beside the tier as
+ a chip on `/channels` rather than as a `/review` section. `/review` is a follow-up.
+ - **(b), `assertRelocationRootPresent`, is NOT done.** The absolute-path `mkdir` in moveOut
+ and moveBack can still materialise a mount when the volume is absent. It is the one piece
+ of S5 left and it is self-contained.
- 2026-09-18 — written; dispatch after S4 merges (`storage/locations-s5`).
- 2026-09-19 — S4 (`storage/locations-s4`, `c4384bd`→`f007263`) reviewed and merged as `4b55e3c`: badge names the location, destination by name with Resume move, bulk by name, docs + changelog. Full suite on the branch tip 527/538 under load, all 11 rerun green after two spec fixes (`a939e0e`, `f007263`). **S0–S4 complete.** Next: S5 (unreachable media) — see below.
+
+
+## S6 — the storage surfaces the operator asked for (shipped 2026-09-20)
+
+Two asks on the evening of 2026-09-20, verbatim: *"I'd like to see disk space and current
+storage volume as columns on the channels menu, and let me filter by volume for easy checking
+of things that may need moving or other processing"* and *"It'd be nice to see a progress on
+relocate jobs since rsync gives progress"* — against an internal disk 96 % full (67 GB free,
+523 GB of channel `data/` on it) with a 2 TB platter registered as `platter`.
+
+Shipped on `storage/locations-s5-s6`, in this order:
+
+1. **Bytes.** `snapshot.totalMediaBytes` in the existing per-video walk;
+ `channelsOnLocation` sums it with an `unknownBytes` count beside it;
+ `views/storage.ts` exposes `bytesOnLocation` / `bytesInPlace` / per-row `freeBytes`.
+2. **The corpus volume is a first-class row**, synthetic, id `internal`, first, with every
+ action withheld and a link to its own channel list. The channel Storage panel's
+ destination select offers it and routes to `moveChannelMediaBackAction`; "Move back in
+ place" is no longer a second section.
+3. **`/channels` is the working surface**: Location and Size columns, `?location=` filter,
+ `?sort=size`, one free-space read-out per volume in a volume bar, and "free up N GB".
+4. **Relocate progress**: `parseRsyncProgress` + a `relocate` JobTask + one log line per
+ decile.
+5. **The saved-video store is movable**: `relocateDir.ts` (the factored core),
+ `relocateSavedVideos`, `settings.storage.savedVideosLocationId`, a `/storage` card.
+
+Every seam is pinned in
+[`FACTS.md`](FACTS.md#storage-locations-s5s6-verified-2026-09-20-branch-storagelocations-s5-s6).
+
+**Not done, and named rather than forgotten:** `assertRelocationRootPresent` (S5(b)) — the
+move can still `mkdir` a target under an absent mount; a `/review` section for auto-paused
+channels; and the per-unit reachability re-check in the two lane runners.
diff --git a/scripts/archilyzer-ops.mjs b/scripts/archilyzer-ops.mjs
@@ -0,0 +1,245 @@
+#!/usr/bin/env node
+// archilyzer-ops — drive a running editor over HTTP, without a browser.
+//
+// Every editor gesture used to be reachable only as a server action, which meant
+// an agent that wanted to sync a channel or fix a download filter had to drive
+// Playwright. /api/ops is a thin adapter layer over those same actions, and this
+// is its client.
+//
+// USAGE
+//
+// pnpm ops <action> [--json '<body>'] [--wait] [--quiet]
+// pnpm ops get channel <slug> [--counts]
+// pnpm ops list
+//
+// ARCHILYZER_EDITOR_URL editor base URL (default http://localhost:3001)
+// WORKER_TOKEN the shared secret the editor is running with.
+// Unset on the SERVER => every route 503s; unset here
+// => every route 401s.
+//
+// EXAMPLES
+//
+// pnpm ops sync --json '{"slug":"the-quartering"}' --wait
+// pnpm ops metadata-scan --json '{"slug":"the-quartering"}'
+// pnpm ops channel-config --json '{"slug":"x","patch":{"downloadFilterExclude":"rerun"}}'
+// pnpm ops channel-priority --json '{"slugs":["x"],"operation":"download","tier":"paused"}'
+// pnpm ops lane --json '{"lane":"download","held":true}'
+// pnpm ops refresh-report --json '{"all":true}'
+// pnpm ops relocate --json '{"slugs":["x"],"locationId":"platter"}'
+// pnpm ops get channel the-quartering
+//
+// --wait follows /api/jobs/<jobId>/log to the end for a job-starting action and
+// exits 0 only if the job finished `done`. Without it the command returns as
+// soon as the job is QUEUED, which is the honest answer: the queue may hold it
+// behind other work for hours.
+//
+// The response JSON is printed verbatim on stdout (log lines from --wait go to
+// stderr), so `pnpm ops … | jq` works.
+
+const DEFAULT_URL = "http://localhost:3001";
+
+// The read-side routes, reachable as `get <noun> <arg>`. Kept tiny and explicit:
+// an ops API that let a caller assemble arbitrary GET paths would be a proxy,
+// not an adapter.
+const GETTERS = {
+ // --counts adds the LIVE on-disk counts, which walk every video directory —
+ // opt-in for the same reason the route makes it opt-in.
+ channel: (slug, counts) =>
+ `/api/ops/channel/${encodeURIComponent(slug)}${counts ? "?counts=1" : ""}`,
+};
+
+const ACTIONS = [
+ "channel-priority",
+ "channel-config",
+ "metadata-scan",
+ "import-video",
+ "refresh-report",
+ "sync",
+ "download-missing",
+ "retry-bucket",
+ "build-index",
+ "build-deploy",
+ "build-site",
+ "relocate",
+ "relocate-back",
+ "lane",
+];
+
+export function parseArgs(argv) {
+ const positional = [];
+ let json = null;
+ let wait = false;
+ let quiet = false;
+ let counts = false;
+ for (let i = 0; i < argv.length; i++) {
+ const arg = argv[i];
+ if (arg === "--wait") {
+ wait = true;
+ } else if (arg === "--quiet") {
+ quiet = true;
+ } else if (arg === "--counts") {
+ counts = true;
+ } else if (arg === "--json") {
+ json = argv[++i];
+ if (json === undefined) {
+ return { error: "--json needs a JSON object argument" };
+ }
+ } else if (arg.startsWith("--json=")) {
+ json = arg.slice("--json=".length);
+ } else if (arg === "--help" || arg === "-h") {
+ return { help: true };
+ } else if (arg.startsWith("-")) {
+ return { error: `unknown flag: ${arg}` };
+ } else {
+ positional.push(arg);
+ }
+ }
+ if (positional.length === 0) return { help: true };
+ let body = {};
+ if (json !== null) {
+ try {
+ body = JSON.parse(json);
+ } catch (e) {
+ return { error: `--json is not valid JSON: ${e.message}` };
+ }
+ if (typeof body !== "object" || body === null || Array.isArray(body)) {
+ return { error: "--json must be a JSON object" };
+ }
+ }
+ if (positional[0] === "list") {
+ return { list: true };
+ }
+ if (positional[0] === "get") {
+ const noun = positional[1];
+ if (!noun || !GETTERS[noun]) {
+ return {
+ error: `get: unknown noun "${noun ?? ""}" — known: ${Object.keys(GETTERS).join(", ")}`,
+ };
+ }
+ if (!positional[2]) return { error: `get ${noun}: needs an argument` };
+ return {
+ method: "GET",
+ path: GETTERS[noun](positional[2], counts),
+ wait: false,
+ quiet,
+ };
+ }
+ const action = positional[0];
+ if (!ACTIONS.includes(action)) {
+ return {
+ error: `unknown action "${action}" — known: ${ACTIONS.join(", ")}`,
+ };
+ }
+ if (positional.length > 1) {
+ return {
+ error: `"${action}" takes no positional arguments — pass its body with --json`,
+ };
+ }
+ return { method: "POST", path: `/api/ops/${action}`, body, wait, quiet };
+}
+
+export function usage() {
+ return [
+ "Usage: pnpm ops <action> [--json '<body>'] [--wait]",
+ " pnpm ops get channel <slug> [--counts]",
+ " pnpm ops list",
+ "",
+ `Actions: ${ACTIONS.join(", ")}`,
+ "",
+ "Env: ARCHILYZER_EDITOR_URL (default http://localhost:3001), WORKER_TOKEN",
+ ].join("\n");
+}
+
+function baseUrl() {
+ return (process.env.ARCHILYZER_EDITOR_URL ?? DEFAULT_URL).replace(/\/+$/, "");
+}
+
+function authHeaders() {
+ const token = process.env.WORKER_TOKEN ?? "";
+ return token ? { authorization: `Bearer ${token}` } : {};
+}
+
+// Follow a job's log to its terminal state. Returns the status string.
+// Deliberately polls the SAME endpoint the editor's own log panel does, so a
+// job started here and a job started by a click are observed identically.
+async function followJob(jobId, quiet) {
+ let from = 0;
+ for (;;) {
+ const res = await fetch(
+ `${baseUrl()}/api/jobs/${encodeURIComponent(jobId)}/log?from=${from}`,
+ { headers: authHeaders() },
+ );
+ if (!res.ok) throw new Error(`log poll failed: HTTP ${res.status}`);
+ const payload = await res.json();
+ if (payload.content && !quiet) process.stderr.write(payload.content);
+ from = payload.nextOffset ?? from;
+ const status = payload.status;
+ if (status !== "queued" && status !== "running") return status;
+ await new Promise((r) => setTimeout(r, 1000));
+ }
+}
+
+async function main() {
+ const parsed = parseArgs(process.argv.slice(2));
+ if (parsed.help) {
+ console.log(usage());
+ return 0;
+ }
+ if (parsed.error) {
+ console.error(parsed.error);
+ console.error("");
+ console.error(usage());
+ return 2;
+ }
+ if (parsed.list) {
+ console.log(ACTIONS.join("\n"));
+ return 0;
+ }
+ const url = `${baseUrl()}${parsed.path}`;
+ const res = await fetch(url, {
+ method: parsed.method,
+ headers: {
+ ...authHeaders(),
+ ...(parsed.method === "POST" ? { "content-type": "application/json" } : {}),
+ },
+ ...(parsed.method === "POST" ? { body: JSON.stringify(parsed.body) } : {}),
+ });
+ const text = await res.text();
+ let payload;
+ try {
+ payload = JSON.parse(text);
+ } catch {
+ console.error(`HTTP ${res.status}: ${text.slice(0, 500)}`);
+ return 1;
+ }
+ console.log(JSON.stringify(payload, null, 2));
+ if (!res.ok || payload.ok === false) return 1;
+ if (!parsed.wait) return 0;
+ const jobIds = payload.jobId
+ ? [payload.jobId]
+ : Array.isArray(payload.jobs)
+ ? payload.jobs.map((j) => j.jobId)
+ : [];
+ if (jobIds.length === 0) {
+ // Not a job-starting action (or it queued nothing). --wait is satisfied.
+ return 0;
+ }
+ let worst = 0;
+ for (const jobId of jobIds) {
+ const status = await followJob(jobId, parsed.quiet);
+ console.error(`[${jobId}] ${status}`);
+ if (status !== "done") worst = 1;
+ }
+ return worst;
+}
+
+// Importable for the arg-parsing tests; only the CLI entry point runs main().
+if (process.argv[1] && import.meta.url === `file://${process.argv[1]}`) {
+ main().then(
+ (code) => process.exit(code),
+ (e) => {
+ console.error(e.message);
+ process.exit(1);
+ },
+ );
+}
diff --git a/scripts/archilyzer-ops.test.mjs b/scripts/archilyzer-ops.test.mjs
@@ -0,0 +1,72 @@
+// Arg parsing for scripts/archilyzer-ops.mjs. No network: parseArgs is pure and
+// returns the request it WOULD make, which is the whole surface worth pinning —
+// the routes themselves are covered by editor/e2e/ops-api.spec.ts.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import test from "node:test";
+import { parseArgs, usage } from "./archilyzer-ops.mjs";
+
+test("no arguments prints usage", () => {
+ assert.equal(parseArgs([]).help, true);
+ assert.match(usage(), /pnpm ops <action>/);
+});
+
+test("an action becomes a POST to its route", () => {
+ const p = parseArgs(["sync", "--json", '{"slug":"x","full":true}']);
+ assert.equal(p.method, "POST");
+ assert.equal(p.path, "/api/ops/sync");
+ assert.deepEqual(p.body, { slug: "x", full: true });
+ assert.equal(p.wait, false);
+});
+
+test("--wait and --json= are both accepted", () => {
+ const p = parseArgs(["metadata-scan", '--json={"slug":"x"}', "--wait"]);
+ assert.equal(p.wait, true);
+ assert.deepEqual(p.body, { slug: "x" });
+});
+
+test("an action with no body posts an empty object", () => {
+ const p = parseArgs(["build-index"]);
+ assert.deepEqual(p.body, {});
+});
+
+test("an unknown action is refused by name, with the list", () => {
+ const p = parseArgs(["sinc"]);
+ assert.match(p.error, /unknown action "sinc"/);
+ assert.match(p.error, /metadata-scan/);
+});
+
+test("malformed --json is refused before any request", () => {
+ assert.match(parseArgs(["sync", "--json", "{"]).error, /not valid JSON/);
+ assert.match(parseArgs(["sync", "--json", "[1]"]).error, /must be a JSON object/);
+ assert.match(parseArgs(["sync", "--json"]).error, /needs a JSON object/);
+});
+
+test("positional arguments after an action are refused", () => {
+ // `pnpm ops sync the-quartering` reads naturally and would otherwise be a
+ // silent no-op body, so it is an error that names the fix.
+ assert.match(parseArgs(["sync", "the-quartering"]).error, /--json/);
+});
+
+test("get channel becomes a GET on the read route", () => {
+ const p = parseArgs(["get", "channel", "the quartering"]);
+ assert.equal(p.method, "GET");
+ assert.equal(p.path, "/api/ops/channel/the%20quartering");
+});
+
+test("get refuses an unknown noun and a missing argument", () => {
+ assert.match(parseArgs(["get", "site", "x"]).error, /unknown noun/);
+ assert.match(parseArgs(["get", "channel"]).error, /needs an argument/);
+});
+
+test("an unknown flag is refused", () => {
+ assert.match(parseArgs(["sync", "--force"]).error, /unknown flag/);
+});
+
+test("get channel --counts asks for the live on-disk counts", () => {
+ const p = parseArgs(["get", "channel", "x", "--counts"]);
+ assert.equal(p.path, "/api/ops/channel/x?counts=1");
+ // Off by default: the counts walk every video directory.
+ assert.equal(parseArgs(["get", "channel", "x"]).path, "/api/ops/channel/x");
+});
diff --git a/umtool/components/projects/ClipBench.tsx b/umtool/components/projects/ClipBench.tsx
@@ -629,6 +629,32 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
[draft, clip, save, needNote],
);
+ // ---- the window patch, one rule, two callers ------------------------------
+ //
+ // `save window` writes it on demand; a confirmation carries it when the edges
+ // have actually moved. Both need the same rule about a cut the new extent no
+ // longer contains, so the rule lives here rather than in each of them.
+ const windowMoved =
+ Math.abs(round2(sel.from) - clip.start) > 0.02 || Math.abs(round2(sel.to) - clip.end) > 0.02;
+
+ const windowPatch = useCallback((): {
+ start: number;
+ end: number;
+ cutStart?: string;
+ cutEnd?: string;
+ } => {
+ const start = round2(sel.from);
+ const end = round2(sel.to);
+ const cutOutside =
+ clip.cutStart != null &&
+ clip.cutEnd != null &&
+ (clip.cutStart < start - 0.02 || clip.cutEnd > end + 0.02);
+ // The extent is the judgement being made right now; the cut was derived
+ // from a wider one and is no longer inside it. Clearing it in the SAME
+ // patch is what keeps the writer's rule and the screen agreeing.
+ return cutOutside ? { start, end, cutStart: "", cutEnd: "" } : { start, end };
+ }, [sel.from, sel.to, clip.cutStart, clip.cutEnd]);
+
// ---- the walk's verdict ---------------------------------------------------
//
// "Is this clip what the report says it is" is the question the walk exists
@@ -637,12 +663,23 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
// case; saying no stays put, because the note has to be typed -- and then
// re-read, which is why it still stays put once it is saved.
const confirmClip = useCallback(async () => {
+ // A MOVED WINDOW RIDES ALONG. Confirming a clip whose edges you just nudged
+ // is a judgement about THAT window, and the advance would otherwise walk
+ // away from it -- so the edges go in the SAME patch as the verdict rather
+ // than needing `save window` pressed first. One write, one token.
+ const win = windowMoved ? windowPatch() : null;
// A note survives a confirmation. It stops being a complaint and becomes
// what it now says it is: why this clip is here in the shape it is in.
- const ok = await save({ verdict: "confirmed" });
+ const ok = await save({ verdict: "confirmed", ...(win ?? {}) });
if (ok) setNeedNote(false);
+ if (ok && win)
+ setNote(
+ win.cutStart !== undefined
+ ? "saved — the window moved with it, and the cut no longer fitted and was cleared"
+ : "saved — the window you moved was saved with it",
+ );
if (ok && data.next) router.push(`/browse/${data.project}/clip/${data.next}`);
- }, [save, router, data.project, data.next]);
+ }, [save, router, data.project, data.next, windowMoved, windowPatch]);
const rejectClip = useCallback(() => {
setNeedNote(true);
@@ -925,23 +962,12 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
/** Save the window, and drop a cut the new extent no longer contains. */
const saveWindow = useCallback(() => {
- const start = round2(sel.from);
- const end = round2(sel.to);
- const cutOutside =
- clip.cutStart != null &&
- clip.cutEnd != null &&
- (clip.cutStart < start - 0.02 || clip.cutEnd > end + 0.02);
- if (cutOutside) {
- // The extent is the judgement being made right now; the cut was derived
- // from a wider one and is no longer inside it. Clearing it in the SAME
- // patch is what keeps the writer's rule and the screen agreeing.
- void save({ start, end, cutStart: "", cutEnd: "" }).then((ok) => {
- if (ok) setNote("saved — the cut no longer fitted this window and was cleared");
- });
- return;
- }
- void save({ start, end });
- }, [sel.from, sel.to, clip.cutStart, clip.cutEnd, save]);
+ const patch = windowPatch();
+ void save(patch).then((ok) => {
+ if (ok && patch.cutStart !== undefined)
+ setNote("saved — the cut no longer fitted this window and was cleared");
+ });
+ }, [windowPatch, save]);
// ---- the warnings --------------------------------------------------------
const endCue = cues.find((c) => sel.to >= c.start - 0.02 && sel.to <= c.end + 0.02) ?? null;
diff --git a/umtool/docs/clip-bench.md b/umtool/docs/clip-bench.md
@@ -247,6 +247,11 @@ the cursor in the `correction` box and marks it **required**, because the note
clip. That save is ONE patch, `{correction, verdict: "incorrect"}`, so the
manifest never holds a complaint with no verdict.
+An edge you nudged but never saved is confirmed WITH the clip: if the selection
+differs from the stored window, <kbd>y</kbd> carries `start`/`end` (and clears a
+cut the new extent no longer contains) in the same patch as the verdict, so
+walking on never loses the window you just chose.
+
A note on a clip that is already **confirmed** is just a note — why its window
moved, a caveat for the writers — and writing one there leaves the verdict
alone. The state line reads *not yet reviewed*, *confirmed*, *confirmed · with
diff --git a/umtool/e2e/clip-bench.spec.ts b/umtool/e2e/clip-bench.spec.ts
@@ -619,6 +619,46 @@ test("`y` confirms the clip and walks on; the manifest says so", async ({ page,
await expect.poll(() => readClip("c01").verdict).toBeUndefined();
});
+test("`y` saves a window you nudged but never saved, in the same write", async ({
+ page,
+ request,
+}) => {
+ // Known ground on both clips the walk touches, so the only thing that moves
+ // an edge here is the key press below.
+ const { token: t0 } = await token(request, "c01");
+ await request.put("/api/report/window", {
+ data: { project: PROJECT, clip: "c01", start: 3, end: 6, verdict: "", token: t0 },
+ });
+ const { token: t1 } = await token(request, "c04");
+ await request.put("/api/report/window", {
+ data: { project: PROJECT, clip: "c04", verdict: "", token: t1 },
+ });
+
+ await page.goto(bench("c01"));
+ await keyboardLive(page);
+ // 6.00 -> 6.05, unsaved: `save window` is lit and nobody has pressed it.
+ await page.locator("body").press(".");
+ await expect(page.getByRole("button", { name: "save window" })).toBeEnabled();
+
+ await page.locator("body").press("y");
+
+ // ONE write carries both. Confirming a clip whose edge you just nudged is a
+ // judgement about THAT window, and the advance would otherwise walk away
+ // from it.
+ await expect.poll(() => readClip("c01").end).toBe(6.05);
+ expect(readClip("c01").start).toBe(3);
+ expect(readClip("c01").verdict).toBe("confirmed");
+ // And it still advances, over c02 and c03, which are not fetched.
+ await expect(page.locator("[data-bench=c04]")).toBeVisible();
+
+ // Put c01 back where the rest of this file found it.
+ const { token: t2 } = await token(request, "c01");
+ await request.put("/api/report/window", {
+ data: { project: PROJECT, clip: "c01", start: 3, end: 9, verdict: "", token: t2 },
+ });
+ await expect.poll(() => readClip("c01").verdict).toBeUndefined();
+});
+
test("`x` requires the note, writes both fields at once, and stays put", async ({ page }) => {
await page.goto(bench("c03"));
// The key IS the assertion: `x` answers "no" by putting the cursor where the
