commit 043a3aa506835641cee357b2b68852a2f042c83d
parent fda0cad3b05e81e65ee13da5da2616503d418c3c
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 17 Sep 2026 14:51:27 -0400
storage: a location knows its channels, and can be pointed at where the disk came up
The probe next door answers "is this disk here, and if not, where?" for one
location in isolation. This is the half that knows the corpus: which channels
sit on a location (`channelsOnLocation`, one pass for every location rather
than one pass each), whether a re-point is safe (`preflightRepoint`, which
writes nothing and makes every refusal the job could make), and the re-point
itself.
A re-point moves no bytes. It rewrites n symlinks, n `config.dataDir` fields
and the location's root, for the case the relocate job cannot help with: the
media never moved, the DISK did. The ledger is renameChannel's, widened to
span channels — a failure on the second rolls the first one back, because a
location half re-pointed is a corpus nobody can reason about at 3am. The
settings write is last and is in the ledger too.
Two seams, both to keep the layer rules: the busy check is a callback (the
editor passes S1's `channelMediaBusyReason`, which reads two process
singletons a unit test has no business creating), and settings I/O is
injected, so these tests neither read the operator's settings.json nor point
the global Paths cache anywhere.
The view is pure and every one of its imports is a type import: the row type
is what the client table renders, and a value import of the controller would
put execa in the client bundle. A withheld action is DATA — "Delete" missing
from a row is indistinguishable from a bug; "Delete — 3 channels still have
their media under this root" is an instruction.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
6 files changed, 1886 insertions(+), 0 deletions(-)
diff --git a/common/controller/storageLocations.test.ts b/common/controller/storageLocations.test.ts
@@ -0,0 +1,498 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ chmod,
+ mkdir,
+ mkdtemp,
+ readlink,
+ rm,
+ symlink,
+ writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import type { SiteSettings } from "../lib/settings";
+import { defaultStorage, sanitizeStorage } from "../lib/settings";
+import type { StorageLocation } from "../lib/storageLocations";
+import {
+ channelsOnLocation,
+ preflightRepoint,
+ probeLocationMemo,
+ recordProbedIdentity,
+ repointStorageLocation,
+ resetStorageProbeMemo,
+ type SettingsIO,
+} from "./storageLocations";
+import { readChannelConfig } from "./channels";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/storageLocations.test.ts
+//
+// Everything happens inside one mkdtemp holding a corpus and two "drives" as
+// SIBLINGS — never nested. A root inside the corpus is refused by design
+// (relocationRootProblem), so a harness that nested them would be exercising a
+// shape the controller rejects.
+//
+// SETTINGS ARE INJECTED, not written to a file: `SettingsIO` is the seam, so
+// these tests never touch the operator's settings.json and never have to point
+// the process-global Paths cache anywhere.
+
+type Harness = {
+ paths: Paths;
+ dir: string;
+ rootA: string;
+ rootB: string;
+ io: SettingsIO;
+ writes: SiteSettings[];
+ settings: () => SiteSettings;
+};
+
+function settingsWith(locations: StorageLocation[]): SiteSettings {
+ return {
+ storage: { locations, defaultLocationId: locations[0]?.id ?? "" },
+ } as unknown as SiteSettings;
+}
+
+async function withTmp(fn: (h: Harness) => Promise<void>): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-storage-loc-"));
+ const transcriptsDir = path.join(dir, "corpus");
+ const paths = {
+ transcriptsDir,
+ channelsDir: path.join(transcriptsDir, "channels"),
+ } as Paths;
+ const rootA = path.join(dir, "platter-a");
+ const rootB = path.join(dir, "platter-b");
+ await mkdir(paths.channelsDir, { recursive: true });
+ await mkdir(rootA, { recursive: true });
+ await mkdir(rootB, { recursive: true });
+ let current = settingsWith([]);
+ const writes: SiteSettings[] = [];
+ const io: SettingsIO = {
+ read: () => current,
+ write: async (next) => {
+ // Round-trip through the real sanitizer: a controller that wrote a shape
+ // settings.ts would drop is a controller whose write silently did
+ // nothing on the next read.
+ current = {
+ ...next,
+ storage: sanitizeStorage(next.storage),
+ } as SiteSettings;
+ writes.push(current);
+ },
+ };
+ const h: Harness = {
+ paths,
+ dir,
+ rootA,
+ rootB,
+ io,
+ writes,
+ settings: () => current,
+ };
+ // Seeded through the same setter the tests use, so `current` is never a
+ // shape the sanitizer would have rejected.
+ h.io.write = io.write;
+ try {
+ await fn(h);
+ } finally {
+ await chmod(path.join(paths.channelsDir), 0o755).catch(() => {});
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+// A channel whose media lives under `root`: a real dir on the "drive", an
+// absolute symlink at channels/<slug>/data, and config.dataDir naming it —
+// exactly what a finished relocation leaves behind.
+async function seedRelocated(
+ h: Harness,
+ slug: string,
+ root: string,
+ opts: { link?: boolean; media?: boolean } = {},
+): Promise<string> {
+ const target = path.join(root, slug, "data");
+ if (opts.media !== false) {
+ await mkdir(path.join(target, "20240101_aaaaaaaaaaa"), { recursive: true });
+ await writeFile(
+ path.join(target, "20240101_aaaaaaaaaaa", "transcript.en.vtt"),
+ "WEBVTT\n",
+ );
+ }
+ const channelDir = path.join(h.paths.channelsDir, slug);
+ await mkdir(channelDir, { recursive: true });
+ await writeFile(
+ path.join(channelDir, "config.json"),
+ JSON.stringify(
+ { handling: "transcribe", url: `https://example.com/${slug}`, dataDir: target },
+ null,
+ 2,
+ ) + "\n",
+ );
+ if (opts.link !== false) {
+ await symlink(target, path.join(channelDir, "data"));
+ }
+ return target;
+}
+
+async function setLocations(h: Harness, locations: StorageLocation[]) {
+ await h.io.write(settingsWith(locations));
+ h.writes.length = 0;
+}
+
+function loc(id: string, root: string, extra: Partial<StorageLocation> = {}): StorageLocation {
+ return { id, label: id, root, autoRepoint: false, ...extra };
+}
+
+test("channelsOnLocation buckets ok / unreachable / moving and ignores channels elsewhere", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "alpha", h.rootA);
+ // Media dir never created: the link dangles, which is what an unmounted
+ // drive looks like.
+ await seedRelocated(h, "beta", h.rootA, { media: false });
+ await seedRelocated(h, "gamma", h.rootA);
+ await writeFile(
+ path.join(h.paths.channelsDir, "gamma", ".relocating.json"),
+ JSON.stringify({ target: path.join(h.rootA, "gamma", "data"), phase: "copy" }),
+ );
+ // On the other drive, and one plain in-place channel with no dataDir.
+ await seedRelocated(h, "delta", h.rootB);
+ await mkdir(path.join(h.paths.channelsDir, "plain", "data"), {
+ recursive: true,
+ });
+ await writeFile(
+ path.join(h.paths.channelsDir, "plain", "config.json"),
+ JSON.stringify({ handling: "transcribe", url: "https://x/y" }) + "\n",
+ );
+
+ const rollups = await channelsOnLocation({
+ paths: h.paths,
+ locations: [loc("a", h.rootA), loc("b", h.rootB)],
+ });
+ assert.deepEqual(rollups.a.slugs, ["alpha", "beta", "gamma"]);
+ assert.equal(rollups.a.total, 3);
+ assert.equal(rollups.a.ok, 1);
+ assert.equal(rollups.a.unreachable, 1);
+ assert.equal(rollups.a.moving, 1);
+ assert.deepEqual(rollups.b.slugs, ["delta"]);
+ assert.equal(rollups.b.ok, 1);
+ });
+});
+
+test("re-point rewrites both channels' links and configs, then the location", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "alpha", h.rootA);
+ await seedRelocated(h, "beta", h.rootA);
+ // The media is already on drive B — the disk moved, the bytes did not.
+ await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
+ await mkdir(path.join(h.rootB, "beta", "data"), { recursive: true });
+ await setLocations(h, [
+ loc("cold", h.rootA, {
+ volume: {
+ uuid: "1111",
+ mountpoint: h.rootA,
+ relPath: "",
+ fstype: "ext4",
+ },
+ }),
+ ]);
+
+ const lines: string[] = [];
+ const result = await repointStorageLocation({
+ paths: h.paths,
+ locationId: "cold",
+ newRoot: h.rootB,
+ io: h.io,
+ onLog: (l) => lines.push(l),
+ });
+
+ assert.deepEqual(result.channels, ["alpha", "beta"]);
+ for (const slug of ["alpha", "beta"]) {
+ const target = path.join(h.rootB, slug, "data");
+ assert.equal(
+ await readlink(path.join(h.paths.channelsDir, slug, "data")),
+ target,
+ );
+ assert.equal((await readChannelConfig(h.paths, slug))?.dataDir, target);
+ }
+ // The settings write is last and carries the new root; the identity is
+ // re-anchored so root === join(mountpoint, relPath) still holds.
+ const stored = h.settings().storage.locations[0];
+ assert.equal(stored.root, h.rootB);
+ assert.equal(stored.volume?.uuid, "1111");
+ assert.equal(stored.volume?.mountpoint, h.rootB);
+ assert.equal(h.writes.length, 1);
+ assert.ok(lines.some((l) => l.includes("Done.")));
+ });
+});
+
+test("re-point refuses a target that has no media for a channel, naming the slug", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "alpha", h.rootA);
+ await seedRelocated(h, "beta", h.rootA);
+ await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
+ await setLocations(h, [loc("cold", h.rootA)]);
+
+ const pre = await preflightRepoint({
+ paths: h.paths,
+ locationId: "cold",
+ newRoot: h.rootB,
+ io: h.io,
+ });
+ assert.equal(pre.ok, false);
+ assert.equal(pre.problems.length, 1);
+ assert.match(pre.problems[0], /beta/);
+ assert.doesNotMatch(pre.problems[0], /alpha/);
+
+ await assert.rejects(
+ repointStorageLocation({
+ paths: h.paths,
+ locationId: "cold",
+ newRoot: h.rootB,
+ io: h.io,
+ }),
+ /beta/,
+ );
+ // Nothing was written: not the link that COULD have moved, not settings.
+ assert.equal(
+ await readlink(path.join(h.paths.channelsDir, "alpha", "data")),
+ path.join(h.rootA, "alpha", "data"),
+ );
+ assert.equal(h.writes.length, 0);
+ });
+});
+
+test("a failure on the second channel rolls the first one back", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "alpha", h.rootA);
+ await seedRelocated(h, "beta", h.rootA);
+ await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
+ await mkdir(path.join(h.rootB, "beta", "data"), { recursive: true });
+ await setLocations(h, [loc("cold", h.rootA)]);
+
+ // A READ-ONLY CHANNEL DIR is the cheapest real failure: the unlink of
+ // beta's `data` link fails with EACCES, which is precisely the shape
+ // (ENOSPC, EROFS, a lost mount) the ledger exists for.
+ const betaDir = path.join(h.paths.channelsDir, "beta");
+ await chmod(betaDir, 0o555);
+ try {
+ await assert.rejects(
+ repointStorageLocation({
+ paths: h.paths,
+ locationId: "cold",
+ newRoot: h.rootB,
+ io: h.io,
+ }),
+ /beta: failed while/,
+ );
+ } finally {
+ await chmod(betaDir, 0o755);
+ }
+
+ // alpha is back where it started — link AND config, and it reads as the
+ // pre-job `unreachable`, never `inconsistent`.
+ assert.equal(
+ await readlink(path.join(h.paths.channelsDir, "alpha", "data")),
+ path.join(h.rootA, "alpha", "data"),
+ );
+ assert.equal(
+ (await readChannelConfig(h.paths, "alpha"))?.dataDir,
+ path.join(h.rootA, "alpha", "data"),
+ );
+ // And the location never moved.
+ assert.equal(h.settings().storage.locations[0].root, h.rootA);
+ assert.equal(h.writes.length, 0);
+ });
+});
+
+test("re-point refuses a busy channel and names it", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "alpha", h.rootA);
+ await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
+ await setLocations(h, [loc("cold", h.rootA)]);
+
+ const pre = await preflightRepoint({
+ paths: h.paths,
+ locationId: "cold",
+ newRoot: h.rootB,
+ io: h.io,
+ isBusy: (slug) => (slug === "alpha" ? "1 running/queued job(s)" : null),
+ });
+ assert.equal(pre.ok, false);
+ assert.match(pre.problems[0], /^alpha: 1 running/);
+ });
+});
+
+test("a rerun after a crash finishes the channels that were left", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "alpha", h.rootA);
+ await seedRelocated(h, "beta", h.rootA);
+ await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
+ await mkdir(path.join(h.rootB, "beta", "data"), { recursive: true });
+ await setLocations(h, [loc("cold", h.rootA)]);
+
+ // A CRASH, not a rollback: alpha was re-pointed and the process died before
+ // beta and before the settings write. Rebuilt by hand, which is exactly the
+ // state on disk.
+ const alphaLink = path.join(h.paths.channelsDir, "alpha", "data");
+ await rm(alphaLink);
+ await symlink(path.join(h.rootB, "alpha", "data"), alphaLink);
+ const alphaConfig = await readChannelConfig(h.paths, "alpha");
+ await writeFile(
+ path.join(h.paths.channelsDir, "alpha", "config.json"),
+ JSON.stringify(
+ { ...alphaConfig, dataDir: path.join(h.rootB, "alpha", "data") },
+ null,
+ 2,
+ ) + "\n",
+ );
+
+ const result = await repointStorageLocation({
+ paths: h.paths,
+ locationId: "cold",
+ newRoot: h.rootB,
+ io: h.io,
+ });
+ // alpha is NOT on the old root any more, so the rerun does not touch it.
+ assert.deepEqual(result.channels, ["beta"]);
+ assert.equal(
+ await readlink(path.join(h.paths.channelsDir, "beta", "data")),
+ path.join(h.rootB, "beta", "data"),
+ );
+ assert.equal(h.settings().storage.locations[0].root, h.rootB);
+
+ // And running it a third time is refused, not repeated: the location is
+ // already there.
+ const pre = await preflightRepoint({
+ paths: h.paths,
+ locationId: "cold",
+ newRoot: h.rootB,
+ io: h.io,
+ });
+ assert.equal(pre.ok, false);
+ assert.match(pre.problems[0], /already at/);
+ });
+});
+
+test("preflight refuses an in-transition channel and a root that is not a directory", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "alpha", h.rootA);
+ await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
+ await writeFile(
+ path.join(h.paths.channelsDir, "alpha", ".relocating.json"),
+ JSON.stringify({ target: path.join(h.rootA, "alpha", "data"), phase: "copy" }),
+ );
+ await setLocations(h, [loc("cold", h.rootA)]);
+
+ const marker = await preflightRepoint({
+ paths: h.paths,
+ locationId: "cold",
+ newRoot: h.rootB,
+ io: h.io,
+ });
+ assert.equal(marker.ok, false);
+ assert.match(marker.problems[0], /alpha: a media relocation is in flight/);
+
+ const nowhere = await preflightRepoint({
+ paths: h.paths,
+ locationId: "cold",
+ newRoot: path.join(h.dir, "not-there"),
+ io: h.io,
+ });
+ assert.equal(nowhere.ok, false);
+ assert.match(nowhere.problems[0], /is not a directory/);
+
+ const relative = await preflightRepoint({
+ paths: h.paths,
+ locationId: "cold",
+ newRoot: "relative/root",
+ io: h.io,
+ });
+ assert.match(relative.problems[0], /absolute path/);
+
+ const missing = await preflightRepoint({
+ paths: h.paths,
+ locationId: "nope",
+ newRoot: h.rootB,
+ io: h.io,
+ });
+ assert.match(missing.problems[0], /no storage location "nope"/);
+ });
+});
+
+test("recordProbedIdentity writes only when the identity changed", async () => {
+ await withTmp(async (h) => {
+ await setLocations(h, [loc("cold", h.rootA)]);
+ const probe = {
+ status: "available" as const,
+ identity: {
+ known: true as const,
+ uuid: "abcd",
+ fstype: "ext4",
+ mountpoint: h.rootA,
+ relPath: "",
+ },
+ };
+ assert.equal(
+ await recordProbedIdentity({ locationId: "cold", probe, io: h.io }),
+ true,
+ );
+ assert.equal(h.settings().storage.locations[0].volume?.uuid, "abcd");
+ // Same answer a second time: no write, so no pulse revision bump.
+ assert.equal(
+ await recordProbedIdentity({ locationId: "cold", probe, io: h.io }),
+ false,
+ );
+ assert.equal(h.writes.length, 1);
+ // An unknown identity is not a fact and never overwrites one.
+ assert.equal(
+ await recordProbedIdentity({
+ locationId: "cold",
+ probe: { status: "missing", identity: { known: false } },
+ io: h.io,
+ }),
+ false,
+ );
+ assert.equal(h.settings().storage.locations[0].volume?.uuid, "abcd");
+ });
+});
+
+test("the probe memo answers twice from one probe, and refresh bypasses it", async () => {
+ await withTmp(async (h) => {
+ resetStorageProbeMemo();
+ let calls = 0;
+ // A findmnt that counts its invocations. The location's root EXISTS, so
+ // probeLocation takes the `available` branch and calls it.
+ const bins = {
+ findmntBin: path.join(h.dir, "counting-findmnt"),
+ udisksctlBin: path.join(h.dir, "no-such-udisksctl"),
+ };
+ await writeFile(
+ bins.findmntBin,
+ "#!/bin/sh\necho '{\"filesystems\":[]}'\n",
+ { mode: 0o755 },
+ );
+ const spyBins = {
+ get findmntBin() {
+ calls += 1;
+ return bins.findmntBin;
+ },
+ udisksctlBin: bins.udisksctlBin,
+ };
+ const location = loc("cold", h.rootA);
+ await probeLocationMemo(location, spyBins, { findmntTimeoutMs: 1_000 });
+ const first = calls;
+ assert.ok(first > 0);
+ await probeLocationMemo(location, spyBins, { findmntTimeoutMs: 1_000 });
+ assert.equal(calls, first, "the second call inside the window probes nothing");
+ await probeLocationMemo(location, spyBins, {
+ findmntTimeoutMs: 1_000,
+ refresh: true,
+ });
+ assert.ok(calls > first, "refresh bypasses the memo");
+ resetStorageProbeMemo();
+ });
+});
+
+test("defaultStorage is still the empty list (the harness's assumption)", () => {
+ assert.deepEqual(defaultStorage(), { locations: [], defaultLocationId: "" });
+});
diff --git a/common/controller/storageLocations.ts b/common/controller/storageLocations.ts
@@ -0,0 +1,779 @@
+import path from "node:path";
+import { stat, symlink, unlink } from "node:fs/promises";
+import { getPaths, type Paths } from "../lib/paths";
+import {
+ getSettings,
+ writeSettings,
+ type SiteSettings,
+} from "../lib/settings";
+import {
+ locationOfDataDir,
+ type StorageLocation,
+ type StorageVolume,
+} from "../lib/storageLocations";
+import {
+ probeLocation,
+ type ProbeOptions,
+ type StorageLocationProbe,
+ type VolumeBins,
+} from "../lib/storageVolumes";
+import { inspectChannelMedia, relocatedDataDir } from "../lib/channelMedia";
+import { relocationQueueKey } from "../lib/queueKeys";
+import {
+ runManagedFunction,
+ type StreamActionResult,
+} from "../jobs/streamCommand";
+import type { ChannelConfig } from "../lib/channelConfig";
+import {
+ listChannelConfigs,
+ readChannelConfig,
+ writeChannelConfig,
+} from "./channels";
+import { relocationRootProblem } from "./relocateChannelMedia";
+
+// WHAT A STORAGE LOCATION KNOWS ABOUT THE CHANNELS ON IT, AND HOW IT MOVES.
+//
+// The probe next door (`lib/storageVolumes.ts`) answers "is this disk here, and
+// if not, where?" for ONE location in isolation. This module is the half that
+// knows the corpus: which channels are on a location, whether a re-point is
+// safe, and the re-point itself.
+//
+// RE-POINT MOVES NO BYTES. It rewrites each channel's `data/` symlink and its
+// `config.dataDir`, then the location's `root`. That is the entire operation —
+// the media is already where it is going, because the DISK came up somewhere
+// else and took it along. The relocate job (which does move bytes) is a
+// different thing entirely, and the two share a queue key precisely so they can
+// never run at once.
+//
+// THE BUSY CHECK IS INJECTED, and this is the one design choice worth naming.
+// `channelMediaBusyReason` (S1) lives in `editor/app/channels/lib/mediaBusy.ts`
+// because it reads two PROCESS singletons — the job registry and the auto-queue
+// runner status — and a view-model layer that constructs singletons is the rule
+// one-core phase 3 exists to hold (see `common/views/inputs.ts`). Moving it down
+// here would make every unit test of this controller create a registry and four
+// lane runners just to be told nothing is running. So the seam is a callback:
+// the editor passes its helper, the tests pass a stub, and the default is "not
+// busy" — which is correct for every non-editor caller, none of which has an
+// in-process runner to be busy with.
+
+// Why a channel cannot be touched right now, or null. The editor passes
+// `channelMediaBusyReason`; a bin script passes nothing.
+export type BusyCheck = (slug: string, what?: string) => string | null;
+
+// Reading and writing settings, injectable so a unit test neither reads the
+// operator's settings.json nor has to point the global Paths cache at a tmp
+// dir. Production callers pass nothing.
+export type SettingsIO = {
+ read: () => SiteSettings;
+ write: (next: SiteSettings) => Promise<void>;
+};
+
+const DEFAULT_SETTINGS_IO: SettingsIO = {
+ read: getSettings,
+ write: writeSettings,
+};
+
+// ---------------------------------------------------------------------------
+// The roll-up: which channels are on which location, and how they read
+// ---------------------------------------------------------------------------
+
+export type LocationRollup = {
+ locationId: string;
+ // Every channel whose `config.dataDir` is under this location's root, sorted.
+ slugs: string[];
+ total: number;
+ // `inspectChannelMedia` status, bucketed into the three numbers the page
+ // shows. `unreachable` DELIBERATELY ABSORBS `inconsistent`: both mean "this
+ // channel's media is not readable through its link right now", which is the
+ // question a storage page is asking, and a fourth column for a state the
+ // re-point preflight refuses by name anyway would be a number nobody acts on.
+ ok: number;
+ unreachable: number;
+ // `in-transition`: a relocation marker is present. A location with any of
+ // these is one no re-point may touch.
+ moving: number;
+};
+
+function emptyRollup(locationId: string): LocationRollup {
+ return { locationId, slugs: [], total: 0, ok: 0, unreachable: 0, moving: 0 };
+}
+
+// One pass over the corpus for EVERY location, not one pass per location: the
+// page draws a row per location and the channel list is the same list for all
+// of them. Cost is `listChannelConfigs` (one readdir + one config read per
+// channel) plus `inspectChannelMedia` (two stats) for the channels that are
+// actually on a location — an unrelocated channel has no `dataDir` and is
+// skipped before it costs a stat.
+export async function channelsOnLocation(opts: {
+ paths: Paths;
+ locations: readonly StorageLocation[];
+ // Already-read configs, when the caller has them (the /storage shell does
+ // not, the channels page does).
+ configs?: ReadonlyArray<{ slug: string; config: ChannelConfig }>;
+}): Promise<Record<string, LocationRollup>> {
+ const out: Record<string, LocationRollup> = {};
+ for (const loc of opts.locations) out[loc.id] = emptyRollup(loc.id);
+ if (opts.locations.length === 0) return out;
+
+ const configs = opts.configs ?? (await listChannelConfigs(opts.paths));
+ for (const { slug, config } of configs) {
+ const dataDir = config.dataDir?.trim();
+ if (!dataDir) continue;
+ const loc = locationOfDataDir(dataDir, opts.locations as StorageLocation[]);
+ if (!loc) continue;
+ const roll = out[loc.id];
+ roll.slugs.push(slug);
+ roll.total += 1;
+ const media = await inspectChannelMedia(opts.paths, slug, config);
+ if (media.status === "ok") roll.ok += 1;
+ else if (media.status === "in-transition") roll.moving += 1;
+ else roll.unreachable += 1;
+ }
+ for (const roll of Object.values(out)) roll.slugs.sort();
+ return out;
+}
+
+// ---------------------------------------------------------------------------
+// The probe memo
+// ---------------------------------------------------------------------------
+
+export type MemoizedProbe = StorageLocationProbe & {
+ // When this answer was taken, ms since epoch. The page renders it as an age.
+ probedAt: number;
+};
+
+export const PROBE_MEMO_MS = 10_000;
+
+type MemoEntry = { root: string; probedAt: number; probe: StorageLocationProbe };
+const probeMemo = new Map<string, MemoEntry>();
+
+// Test seam, and the escape hatch for a process that has just written a root.
+export function resetStorageProbeMemo(): void {
+ probeMemo.clear();
+}
+
+// Probe a location, at most once per PROBE_MEMO_MS.
+//
+// KEYED BY ID, INVALIDATED BY ROOT. /storage renders on every navigation and a
+// probe is up to three subprocesses; ten seconds is short enough that a disk
+// the operator just plugged in shows up on the next reload and long enough that
+// a page with six locations does not fork eighteen processes per click. The
+// root is carried in the entry because a re-point changes it under the same id,
+// and answering for the old root would show the operator the state they just
+// left.
+//
+// `refresh` BYPASSES the memo — that is what the Refresh button is for. It is
+// not the same as a short TTL: the operator pressing Refresh has just done
+// something physical (plugged the disk in, mounted it) and is asking for an
+// answer taken after it.
+export async function probeLocationMemo(
+ loc: StorageLocation,
+ bins: VolumeBins,
+ opts: ProbeOptions & { refresh?: boolean; now?: number } = {},
+): Promise<MemoizedProbe> {
+ const now = opts.now ?? Date.now();
+ const hit = probeMemo.get(loc.id);
+ if (
+ !opts.refresh &&
+ hit &&
+ hit.root === loc.root &&
+ now - hit.probedAt < PROBE_MEMO_MS
+ ) {
+ return { ...hit.probe, probedAt: hit.probedAt };
+ }
+ const probe = await probeLocation(loc, bins, opts);
+ probeMemo.set(loc.id, { root: loc.root, probedAt: now, probe });
+ return { ...probe, probedAt: now };
+}
+
+export async function probeAllLocations(
+ locations: readonly StorageLocation[],
+ bins: VolumeBins,
+ opts: ProbeOptions & { refresh?: boolean; now?: number } = {},
+): Promise<Record<string, MemoizedProbe>> {
+ const out: Record<string, MemoizedProbe> = {};
+ await Promise.all(
+ locations.map(async (loc) => {
+ out[loc.id] = await probeLocationMemo(loc, bins, opts);
+ }),
+ );
+ return out;
+}
+
+function sameVolume(a?: StorageVolume, b?: StorageVolume): boolean {
+ if (!a || !b) return a === b;
+ return (
+ a.uuid === b.uuid &&
+ a.mountpoint === b.mountpoint &&
+ a.relPath === b.relPath &&
+ (a.fstype ?? "") === (b.fstype ?? "") &&
+ (a.label ?? "") === (b.label ?? "")
+ );
+}
+
+// IDENTITY ONLY, AND ONLY WHEN IT CHANGED.
+//
+// A refresh must never write availability — that would rewrite settings.json
+// (and so bump the pulse revision every reader polls) each time anybody loaded
+// the page. What a refresh MAY learn is the disk's identity: the uuid, the
+// mountpoint it is at, and the root's path relative to it. That is what turns
+// "the platter is gone" into "the platter is at /mnt/platter now", so it is
+// worth a write — but only on the probe that actually changed it.
+//
+// Returns whether it wrote.
+export async function recordProbedIdentity(opts: {
+ locationId: string;
+ probe: StorageLocationProbe;
+ io?: SettingsIO;
+}): Promise<boolean> {
+ const io = opts.io ?? DEFAULT_SETTINGS_IO;
+ const identity = opts.probe.identity;
+ if (!identity.known) return false;
+ const settings = io.read();
+ const loc = settings.storage.locations.find((l) => l.id === opts.locationId);
+ if (!loc) return false;
+ const volume: StorageVolume = {
+ uuid: identity.uuid,
+ ...(identity.fstype ? { fstype: identity.fstype } : {}),
+ ...(identity.label ? { label: identity.label } : {}),
+ mountpoint: identity.mountpoint,
+ relPath: identity.relPath,
+ };
+ if (sameVolume(loc.volume, volume)) return false;
+ await io.write({
+ ...settings,
+ storage: {
+ ...settings.storage,
+ locations: settings.storage.locations.map((l) =>
+ l.id === opts.locationId ? { ...l, volume } : l,
+ ),
+ },
+ });
+ return true;
+}
+
+// ---------------------------------------------------------------------------
+// The re-point preflight
+// ---------------------------------------------------------------------------
+
+export type RepointPreflight = {
+ ok: boolean;
+ // Every reason this re-point is refused, each naming the thing to fix.
+ problems: string[];
+ // The channels that would be re-pointed, in the order the job would do it.
+ channels: string[];
+ locationId: string;
+ oldRoot: string;
+ newRoot: string;
+};
+
+async function isDirectory(p: string): Promise<boolean> {
+ try {
+ return (await stat(p)).isDirectory();
+ } catch {
+ return false;
+ }
+}
+
+// NOTHING IS WRITTEN HERE. Every refusal the job can make is made here first,
+// so the operator reads it in the page rather than in a job log, and so
+// `maybeAutoRepoint` can decline silently instead of queueing a job that will
+// throw.
+export async function preflightRepoint(opts: {
+ paths: Paths;
+ locationId: string;
+ newRoot: string;
+ io?: SettingsIO;
+ bins?: VolumeBins;
+ probeOptions?: ProbeOptions;
+ isBusy?: BusyCheck;
+}): Promise<RepointPreflight> {
+ const io = opts.io ?? DEFAULT_SETTINGS_IO;
+ const settings = io.read();
+ const loc = settings.storage.locations.find((l) => l.id === opts.locationId);
+ const newRoot = opts.newRoot.trim().replace(/\/+$/, "") || opts.newRoot.trim();
+ const base: RepointPreflight = {
+ ok: false,
+ problems: [],
+ channels: [],
+ locationId: opts.locationId,
+ oldRoot: loc?.root ?? "",
+ newRoot,
+ };
+ if (!loc) {
+ base.problems.push(`There is no storage location "${opts.locationId}".`);
+ return base;
+ }
+ if (!path.isAbsolute(newRoot)) {
+ base.problems.push(
+ `The new root must be an absolute path (got "${opts.newRoot}").`,
+ );
+ return base;
+ }
+ if (newRoot === loc.root) {
+ base.problems.push(
+ `"${loc.label}" is already at ${loc.root} — there is nothing to re-point.`,
+ );
+ return base;
+ }
+ if (!(await isDirectory(newRoot))) {
+ base.problems.push(
+ `${newRoot} is not a directory. Mount the volume there first.`,
+ );
+ return base;
+ }
+
+ // IDENTITY IS A REFUSAL, NEVER A REQUIREMENT. Two KNOWN uuids that differ
+ // mean the operator is about to point a location at a different disk that
+ // happens to hold directories with the right names, and every channel's
+ // symlink would then silently read someone else's media. Unknown on either
+ // side (no probe yet, a container where block devices are invisible) is not
+ // evidence of anything and does not block: re-point by path is the whole
+ // story there. See RUNNING_IN_DOCKER.md §another drive.
+ if (opts.bins && loc.volume?.uuid) {
+ const probe = await probeLocation(
+ { ...loc, root: newRoot },
+ opts.bins,
+ opts.probeOptions ?? {},
+ );
+ if (probe.identity.known && probe.identity.uuid !== loc.volume.uuid) {
+ base.problems.push(
+ `${newRoot} is on a different disk: it is on UUID ` +
+ `${probe.identity.uuid}, and "${loc.label}" was last seen on ` +
+ `${loc.volume.uuid}. Re-point is for a volume that moved, not for ` +
+ `a different volume — move the media if that is what you meant.`,
+ );
+ return base;
+ }
+ }
+
+ const rollups = await channelsOnLocation({
+ paths: opts.paths,
+ locations: [loc],
+ });
+ const slugs = rollups[loc.id]?.slugs ?? [];
+ const missing: string[] = [];
+ for (const slug of slugs) {
+ const config = await readChannelConfig(opts.paths, slug);
+ const media = await inspectChannelMedia(opts.paths, slug, config);
+ if (media.status === "in-transition") {
+ base.problems.push(
+ `${slug}: a media relocation is in flight or was interrupted ` +
+ `(${media.detail ?? "marker present"}). Finish or clear it first.`,
+ );
+ continue;
+ }
+ if (media.status !== "ok" && media.status !== "unreachable") {
+ base.problems.push(
+ `${slug}: its media reads as ${media.status} — ` +
+ `${media.detail ?? "disk and config do not agree"}. A re-point ` +
+ `rewrites the link and would bake that disagreement in.`,
+ );
+ continue;
+ }
+ const busy = opts.isBusy?.(slug, "re-pointing its storage location");
+ if (busy) {
+ base.problems.push(`${slug}: ${busy}`);
+ continue;
+ }
+ const target = relocatedDataDir(newRoot, slug);
+ if (!(await isDirectory(target))) {
+ missing.push(slug);
+ continue;
+ }
+ const rootProblem = await relocationRootProblem({
+ paths: opts.paths,
+ slug,
+ root: newRoot,
+ });
+ if (rootProblem) {
+ base.problems.push(`${slug}: ${rootProblem}`);
+ continue;
+ }
+ base.channels.push(slug);
+ }
+
+ // THE MISSING TARGETS ARE ONE REFUSAL, NOT n. A root that holds none of the
+ // channels is the ordinary mistake (the wrong mountpoint, a subdirectory too
+ // deep), and n lines each saying the same thing buries the one fact that
+ // matters: which channels are not there.
+ if (missing.length > 0) {
+ base.problems.push(
+ `${missing.length} channel(s) have no media under ${newRoot}: ` +
+ `${missing.join(", ")}. Expected ` +
+ `${relocatedDataDir(newRoot, missing[0])} and friends.`,
+ );
+ }
+
+ base.ok = base.problems.length === 0;
+ return base;
+}
+
+// ---------------------------------------------------------------------------
+// The re-point itself
+// ---------------------------------------------------------------------------
+
+type ChannelLedgerEntry = {
+ slug: string;
+ oldTarget: string;
+ newTarget: string;
+ unlinked: boolean;
+ relinked: boolean;
+ configWritten: boolean;
+ oldConfig: ChannelConfig | null;
+};
+
+export type RepointResult = {
+ locationId: string;
+ oldRoot: string;
+ newRoot: string;
+ // Channels whose link and config were rewritten by THIS run. Empty on an
+ // idempotent rerun that only had the settings write left to do.
+ channels: string[];
+ settingsWritten: boolean;
+};
+
+// Undo one channel's three steps, in reverse, best-effort. The state it restores
+// is the one the job found: a link pointing at the OLD target, which with the
+// disk moved reads as `unreachable`. That is the honest restore — never
+// `inconsistent`, which is a state move-back refuses and which a half-rolled
+// channel (link removed, config rewritten) would leave behind.
+async function rollbackChannel(
+ paths: Paths,
+ entry: ChannelLedgerEntry,
+): Promise<void> {
+ if (entry.configWritten && entry.oldConfig) {
+ await writeChannelConfig(paths, entry.slug, {
+ ...entry.oldConfig,
+ dataDir: entry.oldTarget,
+ }).catch(() => {});
+ }
+ if (entry.relinked || entry.unlinked) {
+ const link = path.join(paths.channelsDir, entry.slug, "data");
+ await unlink(link).catch(() => {});
+ await symlink(entry.oldTarget, link).catch(() => {});
+ }
+}
+
+// THE JOB BODY. Sequential, per channel, on the renameChannel ledger pattern
+// (`controller/renameChannel.ts:139-201`): each step records that it RAN, and a
+// throw replays only what ran, in reverse — across every channel already done,
+// not just the one that failed. A location half re-pointed is a corpus where
+// some channels read from the new mountpoint and some from the old, which is
+// exactly the state nobody can reason about at 3am.
+//
+// The settings write is LAST and is in the ledger too: if it fails, every
+// channel goes back, because a location whose root still names the old path
+// while its channels point at the new one is the same split brain in the other
+// direction.
+//
+// IDEMPOTENT RERUN. A channel that was already re-pointed is no longer ON the
+// old root (`locationOfDataDir` reads its `dataDir`), so the preflight does not
+// list it and this does not touch it. A rerun after a crash finishes the rest;
+// a rerun after a complete run finds no channels and refuses with "already at".
+export async function repointStorageLocation(opts: {
+ paths: Paths;
+ locationId: string;
+ newRoot: string;
+ io?: SettingsIO;
+ bins?: VolumeBins;
+ probeOptions?: ProbeOptions;
+ isBusy?: BusyCheck;
+ onLog?: (line: string) => void;
+ signal?: AbortSignal;
+}): Promise<RepointResult> {
+ const io = opts.io ?? DEFAULT_SETTINGS_IO;
+ const log = opts.onLog ?? (() => {});
+ const pre = await preflightRepoint({
+ paths: opts.paths,
+ locationId: opts.locationId,
+ newRoot: opts.newRoot,
+ io,
+ bins: opts.bins,
+ probeOptions: opts.probeOptions,
+ isBusy: opts.isBusy,
+ });
+ if (!pre.ok) throw new Error(pre.problems.join("\n"));
+
+ const settings = io.read();
+ const loc = settings.storage.locations.find((l) => l.id === opts.locationId);
+ if (!loc) throw new Error(`There is no storage location "${opts.locationId}".`);
+
+ log(
+ `Re-pointing "${loc.label}" from ${pre.oldRoot} to ${pre.newRoot} — ` +
+ `${pre.channels.length} channel(s). No bytes move: this rewrites ` +
+ `${pre.channels.length} symlink(s), ${pre.channels.length} config ` +
+ `field(s) and the location's root.`,
+ );
+
+ const ledger: ChannelLedgerEntry[] = [];
+ try {
+ for (const slug of pre.channels) {
+ opts.signal?.throwIfAborted();
+ const fresh = await readChannelConfig(opts.paths, slug);
+ const oldTarget = fresh?.dataDir?.trim() ?? "";
+ const newTarget = relocatedDataDir(pre.newRoot, slug);
+ const entry: ChannelLedgerEntry = {
+ slug,
+ oldTarget,
+ newTarget,
+ unlinked: false,
+ relinked: false,
+ configWritten: false,
+ oldConfig: fresh,
+ };
+ ledger.push(entry);
+ const link = path.join(opts.paths.channelsDir, slug, "data");
+ try {
+ await unlink(link);
+ entry.unlinked = true;
+ await symlink(newTarget, link);
+ entry.relinked = true;
+ await writeChannelConfig(opts.paths, slug, {
+ ...(fresh as ChannelConfig),
+ dataDir: newTarget,
+ });
+ entry.configWritten = true;
+ } catch (err) {
+ const step = !entry.unlinked
+ ? "removing the old symlink"
+ : !entry.relinked
+ ? "creating the new symlink"
+ : "writing config.json";
+ throw new Error(
+ `${slug}: failed while ${step} — ${(err as Error).message}`,
+ { cause: err },
+ );
+ }
+ log(`${slug}: ${oldTarget} -> ${newTarget}`);
+ }
+
+ // The location last, from a FRESH read: the per-channel loop above wrote no
+ // settings, but a concurrent action (an edit on another tab) may have, and
+ // clobbering it with a snapshot taken before the job started would silently
+ // undo it.
+ const latest = io.read();
+ await io.write({
+ ...latest,
+ storage: {
+ ...latest.storage,
+ locations: latest.storage.locations.map((l) => {
+ if (l.id !== opts.locationId) return l;
+ // The identity, re-anchored to the new root. Written here rather than
+ // by a refresh because THIS is the moment
+ // `root === join(mountpoint, relPath)` holds again — a refresh that
+ // wrote it before the re-point would record a mountpoint the root is
+ // not under.
+ const volume = volumeForNewRoot(l, pre.newRoot);
+ return {
+ ...l,
+ root: pre.newRoot,
+ ...(volume ? { volume } : {}),
+ };
+ }),
+ },
+ });
+ } catch (err) {
+ for (const entry of [...ledger].reverse()) {
+ await rollbackChannel(opts.paths, entry);
+ }
+ log(
+ `Rolled back ${ledger.length} channel(s) — nothing on disk moved, and ` +
+ `every link points where it did before this run.`,
+ );
+ throw err;
+ }
+
+ resetStorageProbeMemo();
+ log(
+ `Done. "${loc.label}" is at ${pre.newRoot}; ${pre.channels.length} ` +
+ `channel(s) re-pointed.`,
+ );
+ return {
+ locationId: opts.locationId,
+ oldRoot: pre.oldRoot,
+ newRoot: pre.newRoot,
+ channels: pre.channels,
+ settingsWritten: true,
+ };
+}
+
+// The stored identity, re-anchored to the new root. The uuid/fstype/label are
+// the disk's and do not change; the mountpoint and relPath are the halves that
+// do, and they are recomputed from the new root so the invariant
+// `root === join(mountpoint, relPath)` survives the re-point. A location with
+// no recorded identity gets none.
+function volumeForNewRoot(
+ loc: StorageLocation,
+ newRoot: string,
+): StorageVolume | undefined {
+ const vol = loc.volume;
+ if (!vol) return undefined;
+ const rel = vol.relPath;
+ if (!rel) return { ...vol, mountpoint: newRoot, relPath: "" };
+ const suffix = `/${rel}`;
+ if (newRoot.endsWith(suffix)) {
+ return {
+ ...vol,
+ mountpoint: newRoot.slice(0, newRoot.length - suffix.length) || "/",
+ relPath: rel,
+ };
+ }
+ // The new root does not end in the recorded relative path — the operator
+ // pointed the location somewhere structurally different. Keep the uuid and
+ // record the root itself as the mountpoint rather than inventing a
+ // mountpoint the root is not under.
+ return { ...vol, mountpoint: newRoot, relPath: "" };
+}
+
+// ---------------------------------------------------------------------------
+// The job, auto re-point and the boot pass
+// ---------------------------------------------------------------------------
+
+// ONE ENQUEUE, for the three callers that have one: the /storage button, the
+// Refresh action's auto path, and the boot pass. Same reason
+// `editor/app/channels/lib/relocationJob.ts` exists — what must not drift
+// between them is the job record's shape, not the guards, which are the
+// caller's.
+//
+// The queue key is `relocationQueueKey()`, SHARED WITH THE RELOCATE JOB on
+// purpose: a re-point rewrites the same links a move is in the middle of
+// rewriting, and the registry caps a key at concurrency 1. One at a time,
+// across both kinds.
+export async function enqueueRepoint(opts: {
+ paths: Paths;
+ locationId: string;
+ newRoot: string;
+ bins?: VolumeBins;
+ isBusy?: BusyCheck;
+ io?: SettingsIO;
+ // Called after the body finishes, inside the job. The editor passes its
+ // revalidatePath calls here; nothing else has pages to invalidate.
+ afterDone?: () => void;
+}): Promise<StreamActionResult> {
+ return runManagedFunction({
+ kind: "repoint-storage-location",
+ queueKey: relocationQueueKey(),
+ paths: opts.paths,
+ fn: async (onLog, signal) => {
+ await repointStorageLocation({
+ paths: opts.paths,
+ locationId: opts.locationId,
+ newRoot: opts.newRoot,
+ bins: opts.bins,
+ isBusy: opts.isBusy,
+ io: opts.io,
+ onLog,
+ signal,
+ });
+ opts.afterDone?.();
+ },
+ });
+}
+
+export type AutoRepointOutcome =
+ | { started: true; jobId?: string; newRoot: string }
+ | { started: false; reason: string };
+
+// The opt-in. `autoRepoint` is per-location and off by default, because a
+// re-point rewrites every channel symlink on the location and that is not
+// something to do silently unless the operator asked for it.
+//
+// Refused unless ALL of: the flag is on, the volume is mounted somewhere else,
+// the probe named a candidate root, and the full preflight passes. The last one
+// is what makes this safe to call from a boot hook: an auto re-point that
+// cannot be done cleanly declines with a reason and leaves the manual button
+// exactly where it was.
+export async function maybeAutoRepoint(opts: {
+ paths: Paths;
+ location: StorageLocation;
+ probe: StorageLocationProbe;
+ bins?: VolumeBins;
+ isBusy?: BusyCheck;
+ io?: SettingsIO;
+ enqueue?: boolean;
+ afterDone?: () => void;
+}): Promise<AutoRepointOutcome> {
+ const loc = opts.location;
+ if (!loc.autoRepoint) return { started: false, reason: "auto re-point is off" };
+ if (opts.probe.status !== "mounted-elsewhere") {
+ return { started: false, reason: `status is ${opts.probe.status}` };
+ }
+ const candidate = opts.probe.candidateRoot;
+ if (!candidate) {
+ return { started: false, reason: "the probe named no candidate root" };
+ }
+ const pre = await preflightRepoint({
+ paths: opts.paths,
+ locationId: loc.id,
+ newRoot: candidate,
+ io: opts.io,
+ bins: opts.bins,
+ isBusy: opts.isBusy,
+ });
+ if (!pre.ok) return { started: false, reason: pre.problems.join("; ") };
+ if (opts.enqueue === false) {
+ return { started: false, reason: "enqueue is disabled (idle boot)" };
+ }
+ const result = await enqueueRepoint({
+ paths: opts.paths,
+ locationId: loc.id,
+ newRoot: candidate,
+ bins: opts.bins,
+ isBusy: opts.isBusy,
+ io: opts.io,
+ afterDone: opts.afterDone,
+ });
+ if (!result.ok) return { started: false, reason: result.error };
+ return { started: true, jobId: result.jobId, newRoot: candidate };
+}
+
+export type BootPassResult = {
+ probed: number;
+ started: string[];
+ declined: Array<{ locationId: string; reason: string }>;
+};
+
+// ONE PASS AT BOOT: probe every location, and for the ones the operator armed,
+// re-point to where the volume actually came up.
+//
+// `enqueue: false` — the idle boot (`ARCHILYZER_IDLE_BOOT`) — still PROBES. The
+// probe is read-only and its whole cost is a findmnt per location, and the memo
+// it fills is what makes the operator's first page load fast. What idle boot
+// refuses is starting work, and a re-point is work.
+export async function runStorageBootPass(opts: {
+ enqueue: boolean;
+ paths?: Paths;
+ bins?: VolumeBins;
+ isBusy?: BusyCheck;
+ io?: SettingsIO;
+}): Promise<BootPassResult> {
+ const io = opts.io ?? DEFAULT_SETTINGS_IO;
+ const settings = io.read();
+ const locations = settings.storage.locations;
+ const out: BootPassResult = { probed: 0, started: [], declined: [] };
+ if (locations.length === 0) return out;
+ const paths = opts.paths ?? getPaths();
+ const bins = opts.bins ?? paths;
+ for (const loc of locations) {
+ const probe = await probeLocationMemo(loc, bins);
+ out.probed += 1;
+ if (probe.identity.known) {
+ await recordProbedIdentity({ locationId: loc.id, probe, io }).catch(
+ () => {},
+ );
+ }
+ const outcome = await maybeAutoRepoint({
+ paths,
+ location: loc,
+ probe,
+ bins,
+ isBusy: opts.isBusy,
+ io,
+ enqueue: opts.enqueue,
+ });
+ if (outcome.started) out.started.push(loc.id);
+ else out.declined.push({ locationId: loc.id, reason: outcome.reason });
+ }
+ return out;
+}
diff --git a/common/jobs/jobKinds.ts b/common/jobs/jobKinds.ts
@@ -386,6 +386,34 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
queueKeyStrategy: "custom",
needsMedia: false,
},
+ // THE SAME LINKS, WITHOUT THE BYTES. A re-point rewrites every channel
+ // symlink on one storage location plus the location's root, for the case the
+ // relocation above cannot help with: the media never moved, the DISK did, and
+ // every `config.dataDir` on it now names a mountpoint that is not there.
+ //
+ // `needsMedia: false` for the relocation's reason, and more sharply: every
+ // channel this job touches is BY DEFINITION unreachable when it starts —
+ // that is the condition it exists to clear — so `true` here would refuse the
+ // one action that can fix it.
+ //
+ // Not drainable: there is no "stop starting new sub-operations" to honor. The
+ // work is n unlinks, n symlinks and n small writes, and a failure rolls the
+ // whole ledger back rather than leaving a partial run to resume. Not
+ // replayable: a replay carries no new root, and re-running against a location
+ // that has since been re-pointed is not a retry.
+ //
+ // THE QUEUE KEY IS relocationQueueKey(), SHARED WITH THE MOVE. The registry
+ // caps a key at concurrency 1, so a re-point can never run beside a
+ // relocation that is rewriting the very links it is about to rewrite — and
+ // only one re-point runs at a time.
+ "repoint-storage-location": {
+ kind: "repoint-storage-location",
+ label: "Re-point storage location",
+ drainable: false,
+ replayable: false,
+ queueKeyStrategy: "custom",
+ needsMedia: false,
+ },
// Social-post ingest for a `sourceKind: "social"` channel. Drainable (the
// fetcher stops paging on the drain signal and keeps what it already has) and
// replayable. queueKeyForUrl() routes x.com / bsky.app to
diff --git a/common/jobs/snapshotScheduler.ts b/common/jobs/snapshotScheduler.ts
@@ -60,6 +60,13 @@ const NO_REGEN_KINDS = new Set<string>([
// has one — so "move out, then move back" would be blocked by a report
// nobody needed.
"relocate-channel-media",
+ // A RE-POINT DOES NOT EVEN MOVE THE BYTES. It rewrites n symlinks and n
+ // `dataDir` fields so the media that came back on a different mountpoint is
+ // reachable again; every count in every report is what it was before. It
+ // also carries no channelSlug on its record — it is a job about a LOCATION,
+ // not a channel — so the central hook has nothing to regenerate anyway, and
+ // listing it here says that on purpose rather than by omission.
+ "repoint-storage-location",
]);
export function shouldRequestSnapshot(kind: string): boolean {
diff --git a/common/views/storage.test.ts b/common/views/storage.test.ts
@@ -0,0 +1,258 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import type { StorageLocation } from "../lib/storageLocations";
+import type { LocationRollup, MemoizedProbe } from "../controller/storageLocations";
+import type { RegistryReader } from "./inputs";
+import { buildStorageRows, type StorageActionKind, type StorageRow } from "./storage";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test views/storage.test.ts
+
+const NOW = 1_700_000_000_000;
+
+function loc(
+ id: string,
+ root: string,
+ extra: Partial<StorageLocation> = {},
+): StorageLocation {
+ return { id, label: id.toUpperCase(), root, autoRepoint: false, ...extra };
+}
+
+function rollup(partial: Partial<LocationRollup>): LocationRollup {
+ return {
+ locationId: partial.locationId ?? "x",
+ slugs: partial.slugs ?? [],
+ total: partial.total ?? 0,
+ ok: partial.ok ?? 0,
+ unreachable: partial.unreachable ?? 0,
+ moving: partial.moving ?? 0,
+ };
+}
+
+const NO_JOBS: RegistryReader = {
+ list: () => [],
+ get: () => undefined,
+} as unknown as RegistryReader;
+
+function registryWith(jobs: Array<Record<string, unknown>>): RegistryReader {
+ return { list: () => jobs, get: () => undefined } as unknown as RegistryReader;
+}
+
+function action(row: StorageRow, kind: StorageActionKind) {
+ const found = row.actions.find((a) => a.kind === kind);
+ assert.ok(found, `no ${kind} action on row ${row.id}`);
+ return found;
+}
+
+test("an available location reports identity, free bytes, counts and a fresh probe", () => {
+ const probe: MemoizedProbe = {
+ status: "available",
+ identity: {
+ known: true,
+ uuid: "09b5",
+ fstype: "ext4",
+ label: "platter",
+ mountpoint: "/run/media/user/09b5",
+ relPath: "archilyzer-media",
+ },
+ freeBytes: 1024,
+ warning: "automount — may not be present at boot",
+ probedAt: NOW - 4_000,
+ };
+ const { rows } = buildStorageRows({
+ locations: [loc("cold", "/run/media/user/09b5/archilyzer-media")],
+ defaultLocationId: "cold",
+ probes: { cold: probe },
+ rollups: { cold: rollup({ locationId: "cold", total: 3, ok: 2, unreachable: 1 }) },
+ registry: NO_JOBS,
+ now: NOW,
+ });
+ const row = rows[0];
+ assert.equal(row.statusLabel, "Available");
+ assert.equal(row.label, "COLD");
+ assert.equal(row.isDefault, true);
+ assert.equal(
+ row.identity,
+ "platter · ext4 · UUID 09b5 · at /run/media/user/09b5",
+ );
+ assert.equal(row.freeBytes, 1024);
+ assert.equal(row.lastProbeAgeMs, 4_000);
+ assert.equal(row.channelsText, "2 ok / 1 unreachable / 0 moving");
+ assert.match(row.warning ?? "", /automount/);
+ assert.equal(action(row, "refresh").offered, true);
+ assert.equal(action(row, "edit").offered, true);
+ // Nothing is wrong, so there is nothing to re-point or mount.
+ assert.equal(action(row, "repoint").offered, false);
+ assert.match(action(row, "repoint").withheld ?? "", /nothing to re-point/);
+ assert.equal(action(row, "mount").offered, false);
+});
+
+test("a probe with no freeBytes renders as absent, not as zero", () => {
+ const { rows } = buildStorageRows({
+ locations: [loc("cold", "/mnt/cold")],
+ defaultLocationId: "",
+ probes: {
+ cold: { status: "available", identity: { known: false }, probedAt: NOW },
+ },
+ rollups: {},
+ registry: NO_JOBS,
+ now: NOW,
+ });
+ assert.equal(rows[0].freeBytes, undefined);
+ // An unknown identity is not a fault: no problem is reported for it.
+ assert.equal(rows[0].identity, null);
+ assert.equal(rows[0].status, "available");
+});
+
+test("mounted-elsewhere offers Re-point named after the candidate root", () => {
+ const { rows } = buildStorageRows({
+ locations: [loc("cold", "/mnt/platter/media", {
+ volume: { uuid: "09b5", mountpoint: "/mnt/platter", relPath: "media" },
+ })],
+ defaultLocationId: "cold",
+ probes: {
+ cold: {
+ status: "mounted-elsewhere",
+ identity: {
+ known: true,
+ uuid: "09b5",
+ mountpoint: "/run/media/user/09b5",
+ relPath: "media",
+ },
+ candidateRoot: "/run/media/user/09b5/media",
+ probedAt: NOW,
+ },
+ },
+ rollups: { cold: rollup({ locationId: "cold", total: 2, unreachable: 2 }) },
+ registry: NO_JOBS,
+ now: NOW,
+ });
+ const row = rows[0];
+ assert.equal(row.statusLabel, "Mounted elsewhere");
+ assert.equal(row.candidateRoot, "/run/media/user/09b5/media");
+ const repoint = action(row, "repoint");
+ assert.equal(repoint.offered, true);
+ assert.equal(repoint.label, "Re-point to /run/media/user/09b5/media");
+ assert.equal(repoint.newRoot, "/run/media/user/09b5/media");
+ // Two channels live there, so Delete is refused with the count and the root.
+ const del = action(row, "delete");
+ assert.equal(del.offered, false);
+ assert.match(del.withheld ?? "", /2 channel\(s\) still have their media under \/mnt\/platter\/media/);
+});
+
+test("mounted-elsewhere with no candidate root withholds Re-point with the reason", () => {
+ const { rows } = buildStorageRows({
+ locations: [loc("cold", "/mnt/platter/media")],
+ defaultLocationId: "",
+ probes: {
+ cold: {
+ status: "mounted-elsewhere",
+ identity: { known: false },
+ probedAt: NOW,
+ },
+ },
+ rollups: {},
+ registry: NO_JOBS,
+ now: NOW,
+ });
+ const repoint = action(rows[0], "repoint");
+ assert.equal(repoint.offered, false);
+ assert.match(repoint.withheld ?? "", /could not work out where the root would be/);
+});
+
+test("Mount is offered only for an unmounted volume with udisksctl present", () => {
+ const inputs = {
+ locations: [loc("cold", "/mnt/platter/media", {
+ volume: { uuid: "09b5", mountpoint: "/mnt/platter", relPath: "media" },
+ })],
+ defaultLocationId: "",
+ probes: {
+ cold: {
+ status: "unmounted" as const,
+ identity: { known: false as const },
+ probedAt: NOW,
+ },
+ },
+ rollups: {},
+ registry: NO_JOBS,
+ now: NOW,
+ };
+ const withBin = buildStorageRows({ ...inputs, udisksctlAvailable: true });
+ const mount = action(withBin.rows[0], "mount");
+ assert.equal(mount.offered, true);
+ assert.equal(mount.uuid, "09b5");
+
+ const withoutBin = buildStorageRows({ ...inputs, udisksctlAvailable: false });
+ const refused = action(withoutBin.rows[0], "mount");
+ assert.equal(refused.offered, false);
+ assert.match(refused.withheld ?? "", /udisksctl is not available/);
+
+ // Absent means the disk is not attached, and the copy says so rather than
+ // blaming the binary.
+ const absent = buildStorageRows({
+ ...inputs,
+ udisksctlAvailable: true,
+ probes: {
+ cold: { status: "absent", identity: { known: false }, probedAt: NOW },
+ },
+ });
+ assert.match(action(absent.rows[0], "mount").withheld ?? "", /not attached/);
+});
+
+test("Delete is offered once no channel is left on the root", () => {
+ const { rows } = buildStorageRows({
+ locations: [loc("cold", "/mnt/cold")],
+ defaultLocationId: "",
+ probes: {
+ cold: { status: "available", identity: { known: false }, probedAt: NOW },
+ },
+ rollups: { cold: rollup({ locationId: "cold" }) },
+ registry: NO_JOBS,
+ now: NOW,
+ });
+ assert.equal(action(rows[0], "delete").offered, true);
+});
+
+test("a running re-point withholds every action on every row, with the job id", () => {
+ const { rows } = buildStorageRows({
+ locations: [loc("cold", "/mnt/cold"), loc("warm", "/mnt/warm")],
+ defaultLocationId: "cold",
+ probes: {
+ cold: { status: "available", identity: { known: false }, probedAt: NOW },
+ warm: {
+ status: "unmounted",
+ identity: { known: false },
+ probedAt: NOW,
+ },
+ },
+ rollups: {},
+ registry: registryWith([
+ { id: "job-7", kind: "repoint-storage-location", status: "running" },
+ ]),
+ udisksctlAvailable: true,
+ now: NOW,
+ });
+ for (const row of rows) {
+ assert.match(row.busy ?? "", /job-7/);
+ for (const a of row.actions) {
+ assert.equal(a.offered, false, `${row.id}/${a.kind} should be withheld`);
+ assert.match(a.withheld ?? "", /job-7/);
+ }
+ }
+});
+
+test("a location that has never been probed reads as missing, not as a gap", () => {
+ const { rows } = buildStorageRows({
+ locations: [loc("cold", "/mnt/cold")],
+ defaultLocationId: "",
+ probes: {},
+ rollups: {},
+ registry: NO_JOBS,
+ now: NOW,
+ });
+ assert.equal(rows[0].status, "missing");
+ assert.equal(rows[0].statusLabel, "Missing");
+ assert.equal(rows[0].identity, null);
+ assert.equal(rows[0].lastProbeAgeMs, 0);
+ assert.equal(rows[0].channelsText, "0 ok / 0 unreachable / 0 moving");
+});
diff --git a/common/views/storage.ts b/common/views/storage.ts
@@ -0,0 +1,316 @@
+import type { StorageLocation } from "../lib/storageLocations";
+import type {
+ StorageIdentity,
+ StorageLocationStatus,
+} from "../lib/storageVolumes";
+import type {
+ LocationRollup,
+ MemoizedProbe,
+} from "../controller/storageLocations";
+import type { RegistryReader } from "./inputs";
+
+// THE /storage PAYLOAD — one row per storage location, and for each row the
+// list of things the operator may do to it and why the rest are withheld.
+//
+// PURE, like everything in this directory (one-core phase 3 slice 1): no I/O,
+// no clock, no singletons. Every live fact arrives as an argument — the probes
+// from `probeAllLocations`, the channel counts from `channelsOnLocation`, the
+// jobs from the registry reader, the instant from `now`. The shell
+// (`editor/app/storage/buildStorage.ts`) is what touches disks and processes.
+//
+// EVERY IMPORT HERE IS A TYPE IMPORT, which is not an accident: the row type is
+// the wire shape the `"use client"` table renders, so this module is reachable
+// from the client bundle. `controller/storageLocations` and `lib/storageVolumes`
+// both pull in execa transitively, and a value import of either would fail
+// `next build` with `node:child_process` in a client component.
+//
+// WHY WITHHELD ACTIONS ARE DATA AND NOT ABSENCE. "Delete" missing from a row
+// is indistinguishable from a bug; "Delete — 3 channels still have their media
+// under this root" is an instruction. So every action is always present with an
+// `offered` flag and, when it is false, the sentence saying what to fix.
+
+export type StorageActionKind =
+ | "refresh"
+ | "repoint"
+ | "mount"
+ | "edit"
+ | "delete";
+
+export type StorageActionView = {
+ kind: StorageActionKind;
+ label: string;
+ offered: boolean;
+ // Set whenever `offered` is false. Never empty.
+ withheld?: string;
+ // "repoint" only: the root the button would move the location to.
+ newRoot?: string;
+ // "mount" only: the volume to mount.
+ uuid?: string;
+};
+
+export type StorageChannelCounts = {
+ ok: number;
+ unreachable: number;
+ moving: number;
+ total: number;
+};
+
+export type StorageRow = {
+ id: string;
+ label: string;
+ root: string;
+ isDefault: boolean;
+ autoRepoint: boolean;
+ status: StorageLocationStatus;
+ // Operator-facing name for the status; the badge's text.
+ statusLabel: string;
+ // One line of identity, or null when the probe could not learn any (no
+ // findmnt, a container, a volume with no UUID). NULL IS NOT A PROBLEM — see
+ // storageVolumes.ts's header — so the page renders it as "unknown", never as
+ // a fault.
+ identity: string | null;
+ channels: StorageChannelCounts;
+ // `n ok / n unreachable / n moving` — the cell's text, built here so the page
+ // and any future poll cannot word it differently.
+ channelsText: string;
+ // Absent when the location is not available, and ALSO when it is available
+ // but unmeasurable (getFreeBytes fails open to Infinity, which the probe
+ // drops rather than carry). Render "—" for both.
+ freeBytes?: number;
+ // How long ago the probe behind this row was taken, in ms.
+ lastProbeAgeMs: number;
+ warning?: string;
+ candidateRoot?: string;
+ // Why every action on this row is withheld, or null. One re-point runs at a
+ // time (the registry caps the shared `relocate` queue key at 1), so a running
+ // one freezes every row, not just its own.
+ busy: string | null;
+ actions: StorageActionView[];
+};
+
+export type StorageRowsPayload = {
+ rows: StorageRow[];
+ defaultLocationId: string;
+ // Whether `udisksctl` resolved in this process. False in a container, and the
+ // reason the Mount button is withheld there.
+ udisksctlAvailable: boolean;
+};
+
+export type StorageRowsInputs = {
+ locations: readonly StorageLocation[];
+ defaultLocationId: string;
+ // By location id. A location with no entry has never been probed in this
+ // process — treated as `missing` with an unknown identity rather than
+ // omitted, because a row that vanishes when a probe fails is worse than one
+ // that says it does not know.
+ probes: Record<string, MemoizedProbe | undefined>;
+ rollups: Record<string, LocationRollup | undefined>;
+ registry: RegistryReader;
+ udisksctlAvailable?: boolean;
+ now: number;
+};
+
+const STATUS_LABEL: Record<StorageLocationStatus, string> = {
+ available: "Available",
+ "mounted-elsewhere": "Mounted elsewhere",
+ unmounted: "Not mounted",
+ absent: "Not attached",
+ missing: "Missing",
+};
+
+export const REPOINT_JOB_KIND = "repoint-storage-location";
+
+function identityLine(identity: StorageIdentity): string | null {
+ if (!identity.known) return null;
+ const bits: string[] = [];
+ if (identity.label) bits.push(identity.label);
+ if (identity.fstype) bits.push(identity.fstype);
+ bits.push(`UUID ${identity.uuid}`);
+ bits.push(`at ${identity.mountpoint}`);
+ return bits.join(" · ");
+}
+
+function countsOf(rollup: LocationRollup | undefined): StorageChannelCounts {
+ return {
+ ok: rollup?.ok ?? 0,
+ unreachable: rollup?.unreachable ?? 0,
+ moving: rollup?.moving ?? 0,
+ total: rollup?.total ?? 0,
+ };
+}
+
+// The one running re-point, or null. Reported for the whole page rather than
+// per row because the job record carries no location id: it is enqueued on the
+// shared `relocate` queue key, which the registry caps at concurrency 1, so
+// "one is running" is a fact about the machine and not about a row.
+function runningRepoint(registry: RegistryReader): string | null {
+ const job = registry
+ .list()
+ .find(
+ (j) =>
+ j.kind === REPOINT_JOB_KIND &&
+ (j.status === "running" || j.status === "queued"),
+ );
+ if (!job) return null;
+ return (
+ `A storage re-point is ${job.status} (job ${job.id}). One runs at a ` +
+ `time — wait for it to finish, or cancel it on /jobs.`
+ );
+}
+
+export function buildStorageRows(i: StorageRowsInputs): StorageRowsPayload {
+ const busy = runningRepoint(i.registry);
+ const udisksctlAvailable = i.udisksctlAvailable ?? false;
+ const rows = i.locations.map((loc): StorageRow => {
+ const probe = i.probes[loc.id];
+ const status: StorageLocationStatus = probe?.status ?? "missing";
+ const counts = countsOf(i.rollups[loc.id]);
+ const candidateRoot = probe?.candidateRoot;
+
+ const actions: StorageActionView[] = [
+ {
+ kind: "refresh",
+ label: "Refresh",
+ offered: !busy,
+ ...(busy ? { withheld: busy } : {}),
+ },
+ repointAction(status, candidateRoot, busy),
+ mountAction(status, probe?.identity, loc, udisksctlAvailable, busy),
+ {
+ kind: "edit",
+ label: "Edit",
+ offered: !busy,
+ ...(busy ? { withheld: busy } : {}),
+ },
+ deleteAction(loc, counts, busy),
+ ];
+
+ return {
+ id: loc.id,
+ label: loc.label || loc.id,
+ root: loc.root,
+ isDefault: loc.id === i.defaultLocationId,
+ autoRepoint: loc.autoRepoint,
+ status,
+ statusLabel: STATUS_LABEL[status],
+ identity: probe ? identityLine(probe.identity) : null,
+ channels: counts,
+ channelsText: `${counts.ok} ok / ${counts.unreachable} unreachable / ${counts.moving} moving`,
+ ...(probe?.freeBytes !== undefined ? { freeBytes: probe.freeBytes } : {}),
+ lastProbeAgeMs: probe ? Math.max(0, i.now - probe.probedAt) : 0,
+ ...(probe?.warning ? { warning: probe.warning } : {}),
+ ...(candidateRoot ? { candidateRoot } : {}),
+ busy,
+ actions,
+ };
+ });
+ return { rows, defaultLocationId: i.defaultLocationId, udisksctlAvailable };
+}
+
+// RE-POINT IS OFFERED FOR EXACTLY ONE STATUS. `mounted-elsewhere` means the
+// recorded UUID was found mounted at a different mountpoint, and
+// `candidateRoot` is that mountpoint plus the root's recorded relative path —
+// the only path the job may be pointed at without the operator typing one. Any
+// other status is either "nothing is wrong" or "there is nothing to point at",
+// and both read better as a sentence than as a greyed button with no tooltip.
+function repointAction(
+ status: StorageLocationStatus,
+ candidateRoot: string | undefined,
+ busy: string | null,
+): StorageActionView {
+ if (busy) {
+ return { kind: "repoint", label: "Re-point", offered: false, withheld: busy };
+ }
+ if (status !== "mounted-elsewhere") {
+ return {
+ kind: "repoint",
+ label: "Re-point",
+ offered: false,
+ withheld:
+ status === "available"
+ ? "The root is there — nothing to re-point."
+ : `Re-point needs the volume mounted somewhere else; this location reads ${STATUS_LABEL[status].toLowerCase()}.`,
+ };
+ }
+ if (!candidateRoot) {
+ return {
+ kind: "repoint",
+ label: "Re-point",
+ offered: false,
+ withheld:
+ "The volume is mounted elsewhere but the probe could not work out " +
+ "where the root would be. Edit the location's root by hand.",
+ };
+ }
+ return {
+ kind: "repoint",
+ label: `Re-point to ${candidateRoot}`,
+ offered: true,
+ newRoot: candidateRoot,
+ };
+}
+
+// MOUNT NEEDS BOTH A DISK TO MOUNT AND A BINARY TO MOUNT IT WITH. In a
+// container there is neither — block devices are invisible and udisksctl is not
+// installed — so the withheld reason names the binary rather than implying the
+// disk is at fault.
+function mountAction(
+ status: StorageLocationStatus,
+ identity: StorageIdentity | undefined,
+ loc: StorageLocation,
+ udisksctlAvailable: boolean,
+ busy: string | null,
+): StorageActionView {
+ const uuid = loc.volume?.uuid ?? (identity?.known ? identity.uuid : "");
+ if (busy) {
+ return { kind: "mount", label: "Mount", offered: false, withheld: busy };
+ }
+ if (status !== "unmounted") {
+ return {
+ kind: "mount",
+ label: "Mount",
+ offered: false,
+ withheld:
+ status === "absent"
+ ? "The volume is not attached to this machine."
+ : `Mount is only offered for an attached, unmounted volume; this location reads ${STATUS_LABEL[status].toLowerCase()}.`,
+ };
+ }
+ if (!udisksctlAvailable) {
+ return {
+ kind: "mount",
+ label: "Mount",
+ offered: false,
+ withheld:
+ "udisksctl is not available in this process (set UDISKSCTL_BIN) — " +
+ "mount the volume from the host and press Refresh.",
+ };
+ }
+ return { kind: "mount", label: "Mount", offered: true, uuid };
+}
+
+// DELETE IS REFUSED WHILE ANYBODY LIVES THERE. Deleting the location would not
+// touch a byte — but it would erase the only record of which disk those
+// channels' absolute `dataDir`s belong to, which is precisely the knowledge
+// this page exists to keep. Move the channels off it (or re-point it) first.
+function deleteAction(
+ loc: StorageLocation,
+ counts: StorageChannelCounts,
+ busy: string | null,
+): StorageActionView {
+ if (busy) {
+ return { kind: "delete", label: "Delete", offered: false, withheld: busy };
+ }
+ if (counts.total > 0) {
+ return {
+ kind: "delete",
+ label: "Delete",
+ offered: false,
+ withheld:
+ `${counts.total} channel(s) still have their media under ` +
+ `${loc.root}: ${counts.total === 1 ? "move it" : "move them"} back ` +
+ `in place, or onto another location, first.`,
+ };
+ }
+ return { kind: "delete", label: "Delete", offered: true };
+}
