commit 12c231722b8042a2f0807a1153f1c87f7134287e
parent c82466ca961ef33fa5c2f4ab665311ac589c4258
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 11 Sep 2026 11:39:33 -0400
common: a resumed relocation observes the disk instead of trusting the marker
The marker says how far the last attempt GOT. It cannot say how far it got
through a phase, and a kill lands between two syscalls — so every step past the
copy now dispatches on what `data/` actually is, read with lstat, and the plan's
"a rerun re-verifies and continues from swap/reclaim" is finally true for six
crash points rather than one. There is a test per phase per direction, each
seeding the exact on-disk state of a crash at that point.
Four ways it was not:
Resuming at `swap` did not re-verify and minted a NEW parked name, while the
`data.relocated-*` sweep sat in the `else` a fresh run never takes. A crash
between writeChannelConfig and the reclaim marker therefore orphaned a full
second copy of the channel on `/home` — the disk the whole move exists to free —
permanently. The sweep now runs on every path and takes every sibling.
Resuming `back` at `swap` was idempotent in exactly the case that never happens.
A crash AFTER rename(incoming, data) left `data/` a real directory; the
swallowed unlink then failed on it and the rename ENOENTed on an `incoming` that
was gone, identically on every rerun, forever.
Resuming `back` at `reclaim` was unreachable: the swap clears config.dataDir, so
the entry point read "this channel is not relocated" and threw — while the
marker beside it named the target still holding the media. The marker is the
second source of truth for `back`, and a marker of the OTHER direction is now
refused outright rather than resumed into, in both directions.
And a marker whose run is gone was a dead end with no way out, so
clearRelocationMarker() removes the marker and NOTHING else: no link, no config,
no bytes. Whatever inspect() says afterwards is what the disk was already
saying underneath it.
Also here, all from the same review: the space check is bytes + resumeMarginGB
(landing with nothing to spare puts the destination under the gate's own floor
the moment it arrives, so the next download is refused on the drive it was just
moved to), the root is checked for writability explicitly rather than three
minutes into an rsync, a channel with no data/ is refused by name instead of by
rsync's exit 23, and the `data/` probes use lstat — a dangling link read as
absent through stat, and symlink() then threw EEXIST on a path it had just been
told was not there.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
4 files changed, 547 insertions(+), 36 deletions(-)
diff --git a/common/controller/relocateChannelMedia.test.ts b/common/controller/relocateChannelMedia.test.ts
@@ -7,8 +7,10 @@ import {
readdir,
readFile,
readlink,
+ rename,
rm,
stat,
+ symlink,
utimes,
writeFile,
} from "node:fs/promises";
@@ -385,3 +387,278 @@ async function pathThere(p: string): Promise<boolean> {
return false;
}
}
+
+// ---------------------------------------------------------------------------
+// CRASH RECOVERY: one case per phase per direction.
+//
+// The plan's contract is "a rerun with a matching marker re-verifies and
+// continues from swap/reclaim", and every one of these states is reachable from
+// a kill -9 between two syscalls. What they pin is that a rerun OBSERVES the
+// disk rather than assuming the previous run finished the step the marker names:
+// the marker says how far the last attempt got, it cannot say how far it got
+// THROUGH a phase, and the disk is the only evidence.
+//
+// Each seeds the exact on-disk state of a crash at that point and asserts the
+// rerun reaches a clean `ok` / `in-place` — with the config right, the marker
+// gone and NO `data.relocated-*` or `data.incoming` left holding a second copy
+// on the volume the move exists to free.
+
+async function seedMarker(
+ paths: Paths,
+ slug: string,
+ marker: {
+ target: string;
+ direction: "out" | "back";
+ phase: "copy" | "swap" | "reclaim";
+ },
+): Promise<void> {
+ await writeFile(
+ path.join(paths.channelsDir, slug, ".relocating.json"),
+ JSON.stringify({ ...marker, startedAt: new Date().toISOString() }) + "\n",
+ );
+}
+
+async function setConfigDataDir(
+ paths: Paths,
+ slug: string,
+ value: string | null,
+): Promise<void> {
+ const file = path.join(paths.channelsDir, slug, "config.json");
+ const config = JSON.parse(await readFile(file, "utf8")) as Record<
+ string,
+ unknown
+ >;
+ if (value === null) delete config.dataDir;
+ else config.dataDir = value;
+ await writeFile(file, JSON.stringify(config, null, 2) + "\n");
+}
+
+async function leftoverCopies(channelDir: string): Promise<string[]> {
+ return (await readdir(channelDir))
+ .filter((n) => n.startsWith("data.relocated-") || n === "data.incoming")
+ .sort();
+}
+
+// rsync -a of the tree, so the seeded "already copied" target is byte-for-byte
+// what a completed copy phase would have left — including mtimes, which the
+// re-verify compares.
+async function copyTree(src: string, dest: string): Promise<void> {
+ await mkdir(dest, { recursive: true });
+ for (const entry of await readdir(src, { withFileTypes: true })) {
+ const from = path.join(src, entry.name);
+ const to = path.join(dest, entry.name);
+ if (entry.isDirectory()) {
+ await copyTree(from, to);
+ } else {
+ await writeFile(to, await readFile(from));
+ await utimes(to, MTIME, MTIME);
+ }
+ }
+}
+
+test("out @ swap: crash before the rename — the rerun re-verifies and completes", async () => {
+ await withTmp(async (paths, root) => {
+ const channelDir = await seed(paths, "alpha", {
+ v1: { "audio.m4a": "one".repeat(500) },
+ });
+ const target = relocatedDataDir(root, "alpha");
+ // The copy finished; the process died before `data/` was parked.
+ await copyTree(path.join(channelDir, "data"), target);
+ await seedMarker(paths, "alpha", { target, direction: "out", phase: "swap" });
+
+ const res = await relocateChannelMedia({
+ paths,
+ slug: "alpha",
+ direction: "out",
+ root,
+ onLog: () => {},
+ });
+ assert.equal(res.resumed, true);
+ assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
+ assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, target);
+ assert.deepEqual(await leftoverCopies(channelDir), []);
+ assert.equal(await readRelocationMarker(paths, "alpha"), null);
+ });
+});
+
+test("out @ swap: crash after the config write — the first run's parked copy is still reclaimed", async () => {
+ await withTmp(async (paths, root) => {
+ const channelDir = await seed(paths, "alpha", {
+ v1: { "audio.m4a": "one".repeat(500) },
+ });
+ const dataDir = path.join(channelDir, "data");
+ const target = relocatedDataDir(root, "alpha");
+ await copyTree(dataDir, target);
+ // Everything through writeChannelConfig ran; the reclaim marker never
+ // landed. A full second copy of the channel is parked on the source volume
+ // — the disk the whole move exists to free — and the marker still says
+ // "swap", so the old code minted a SECOND parked name and reclaimed only
+ // that one, orphaning this forever.
+ const parked = path.join(channelDir, "data.relocated-1700000000000");
+ await rename(dataDir, parked);
+ await symlink(target, dataDir);
+ await setConfigDataDir(paths, "alpha", target);
+ await seedMarker(paths, "alpha", { target, direction: "out", phase: "swap" });
+
+ await relocateChannelMedia({
+ paths,
+ slug: "alpha",
+ direction: "out",
+ root,
+ onLog: () => {},
+ });
+ assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
+ assert.deepEqual(await leftoverCopies(channelDir), []);
+ assert.equal(await readRelocationMarker(paths, "alpha"), null);
+ // The media itself is untouched by the sweep.
+ assert.equal(
+ await readFile(path.join(dataDir, "v1", "audio.m4a"), "utf8"),
+ "one".repeat(500),
+ );
+ });
+});
+
+test("out @ reclaim: the rerun sweeps every parked copy and clears the marker", async () => {
+ await withTmp(async (paths, root) => {
+ const channelDir = await seed(paths, "alpha", {
+ v1: { "audio.m4a": "one".repeat(500) },
+ });
+ const dataDir = path.join(channelDir, "data");
+ const target = relocatedDataDir(root, "alpha");
+ await copyTree(dataDir, target);
+ await rename(dataDir, path.join(channelDir, "data.relocated-1700000000000"));
+ await symlink(target, dataDir);
+ await setConfigDataDir(paths, "alpha", target);
+ await seedMarker(paths, "alpha", {
+ target,
+ direction: "out",
+ phase: "reclaim",
+ });
+
+ await relocateChannelMedia({
+ paths,
+ slug: "alpha",
+ direction: "out",
+ root,
+ onLog: () => {},
+ });
+ assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
+ assert.deepEqual(await leftoverCopies(channelDir), []);
+ assert.equal(await readRelocationMarker(paths, "alpha"), null);
+ });
+});
+
+test("back @ swap: crash before the rename — the rerun finishes the swap", async () => {
+ await withTmp(async (paths, root) => {
+ const channelDir = await seed(paths, "alpha", {
+ v1: { "audio.m4a": "one".repeat(500) },
+ });
+ const dataDir = path.join(channelDir, "data");
+ const target = relocatedDataDir(root, "alpha");
+ // A completed relocation, then a move-back whose copy finished.
+ await copyTree(dataDir, target);
+ await rm(dataDir, { recursive: true, force: true });
+ await symlink(target, dataDir);
+ await setConfigDataDir(paths, "alpha", target);
+ await copyTree(target, path.join(channelDir, "data.incoming"));
+ await seedMarker(paths, "alpha", { target, direction: "back", phase: "swap" });
+
+ const res = await relocateChannelMedia({
+ paths,
+ slug: "alpha",
+ direction: "back",
+ onLog: () => {},
+ });
+ assert.equal(res.resumed, true);
+ assert.equal((await inspectChannelMedia(paths, "alpha")).status, "in-place");
+ assert.ok((await lstat(dataDir)).isDirectory());
+ assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, undefined);
+ assert.deepEqual(await leftoverCopies(channelDir), []);
+ assert.equal(await readRelocationMarker(paths, "alpha"), null);
+ });
+});
+
+test("back @ swap: crash AFTER the rename — the rerun does not ENOENT forever", async () => {
+ await withTmp(async (paths, root) => {
+ const channelDir = await seed(paths, "alpha", {
+ v1: { "audio.m4a": "one".repeat(500) },
+ });
+ const dataDir = path.join(channelDir, "data");
+ const target = relocatedDataDir(root, "alpha");
+ await copyTree(dataDir, target);
+ // rename(incoming, data) committed; the process died before the config was
+ // cleared. `data/` is a real directory and `incoming` is gone — so the old
+ // sequence unlinked nothing (swallowed), then renamed a path that no longer
+ // exists, and failed identically on every rerun.
+ await setConfigDataDir(paths, "alpha", target);
+ await seedMarker(paths, "alpha", { target, direction: "back", phase: "swap" });
+
+ await relocateChannelMedia({
+ paths,
+ slug: "alpha",
+ direction: "back",
+ onLog: () => {},
+ });
+ assert.equal((await inspectChannelMedia(paths, "alpha")).status, "in-place");
+ assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, undefined);
+ assert.equal(await pathIsThere(target), false);
+ assert.deepEqual(await leftoverCopies(channelDir), []);
+ assert.equal(await readRelocationMarker(paths, "alpha"), null);
+ });
+});
+
+test("back @ reclaim: the config is already clear, and the rerun still finishes", async () => {
+ await withTmp(async (paths, root) => {
+ const channelDir = await seed(paths, "alpha", {
+ v1: { "audio.m4a": "one".repeat(500) },
+ });
+ const target = relocatedDataDir(root, "alpha");
+ // The swap completed and cleared config.dataDir — which is why this case
+ // was UNREACHABLE: with no dataDir the entry point threw "is not relocated"
+ // and the marker named the only place that knew where the media had been.
+ await copyTree(path.join(channelDir, "data"), target);
+ await seedMarker(paths, "alpha", {
+ target,
+ direction: "back",
+ phase: "reclaim",
+ });
+
+ await relocateChannelMedia({
+ paths,
+ slug: "alpha",
+ direction: "back",
+ onLog: () => {},
+ });
+ assert.equal((await inspectChannelMedia(paths, "alpha")).status, "in-place");
+ assert.equal(await pathIsThere(target), false);
+ assert.deepEqual(await leftoverCopies(channelDir), []);
+ assert.equal(await readRelocationMarker(paths, "alpha"), null);
+ });
+});
+
+test("a marker for the other direction is never resumed into", async () => {
+ await withTmp(async (paths, root) => {
+ await seed(paths, "alpha", { v1: { "audio.m4a": "one" } });
+ const target = relocatedDataDir(root, "alpha");
+ await setConfigDataDir(paths, "alpha", target);
+ await seedMarker(paths, "alpha", { target, direction: "out", phase: "swap" });
+ await assert.rejects(
+ relocateChannelMedia({
+ paths,
+ slug: "alpha",
+ direction: "back",
+ onLog: () => {},
+ }),
+ /move-out .* in flight|interrupted/,
+ );
+ });
+});
+
+async function pathIsThere(p: string): Promise<boolean> {
+ try {
+ await lstat(p);
+ return true;
+ } catch {
+ return false;
+ }
+}
diff --git a/common/controller/relocateChannelMedia.ts b/common/controller/relocateChannelMedia.ts
@@ -1,8 +1,12 @@
import path from "node:path";
import {
+ access,
+ constants as fsConstants,
+ lstat,
mkdir,
readdir,
readFile,
+ readlink,
rename,
rm,
stat,
@@ -14,6 +18,7 @@ import { execa } from "execa";
import type { Paths } from "../lib/paths";
import { isSocialChannel } from "../lib/channelConfig";
import { getFreeBytes } from "../lib/diskSpace";
+import { getSettings } from "../lib/settings";
import { formatBytes } from "../lib/format";
import {
inspectChannelMedia,
@@ -41,6 +46,8 @@ import { readChannelConfig, writeChannelConfig } from "./channels";
// `rsync -a` preserves mtimes, which is what makes this free for the LMDB index:
// it stores ids and mtimes, no paths, so a relocated channel needs no reindex.
+const BYTES_PER_GB = 1024 ** 3;
+
export type RelocateChannelMediaResult = {
slug: string;
direction: RelocationDirection;
@@ -129,6 +136,60 @@ async function isDirectory(p: string): Promise<boolean> {
}
}
+// WHAT `data/` ACTUALLY IS RIGHT NOW. lstat, never stat: a dangling symlink —
+// the exact state a half-finished move or an unmounted drive leaves behind —
+// reads as ABSENT through stat, and the code that then tries to create the link
+// fails with EEXIST on a path it was just told was not there.
+//
+// Every phase past the copy dispatches on this rather than on what the previous
+// phase is supposed to have done, which is what makes a rerun idempotent: the
+// marker says how far the last run GOT, the disk says what is actually there,
+// and only the disk is evidence.
+type DataDirState =
+ | { kind: "missing" }
+ | { kind: "real-dir" }
+ | { kind: "link"; linkTarget: string }
+ | { kind: "other" };
+
+async function dataDirState(p: string): Promise<DataDirState> {
+ let st;
+ try {
+ st = await lstat(p);
+ } catch {
+ return { kind: "missing" };
+ }
+ if (st.isSymbolicLink()) {
+ return { kind: "link", linkTarget: await readlink(p).catch(() => "") };
+ }
+ if (st.isDirectory()) return { kind: "real-dir" };
+ return { kind: "other" };
+}
+
+// Every leftover a crashed run can have parked next to `data/`, in one list.
+// The reclaim phase sweeps ALL of them rather than the one name the run that is
+// finishing happens to hold: a crash between the config write and the reclaim
+// marker leaves a full second copy of the channel on the source volume, and
+// reclaiming only the name this process minted would orphan it forever — on the
+// disk the move exists to free.
+async function parkedSiblings(channelDir: string): Promise<string[]> {
+ const names = await readdir(channelDir).catch(() => [] as string[]);
+ return names
+ .filter((n) => n.startsWith("data.relocated-") || n === "data.incoming")
+ .map((n) => path.join(channelDir, n));
+}
+
+async function sweepParked(
+ channelDir: string,
+ log: (m: string) => void,
+ keep?: string,
+): Promise<void> {
+ for (const p of await parkedSiblings(channelDir)) {
+ if (keep && path.resolve(p) === path.resolve(keep)) continue;
+ log(`Reclaiming ${p}`);
+ await rm(p, { recursive: true, force: true });
+ }
+}
+
async function writeMarker(
paths: Paths,
slug: string,
@@ -291,7 +352,14 @@ export async function relocateChannelMedia(
const existingMarker = await readMarkerRaw(paths, slug);
if (direction === "back") {
- const target = config.dataDir?.trim();
+ // THE MARKER IS THE SECOND SOURCE OF TRUTH FOR THE TARGET, and without it
+ // the resume path was unreachable. moveBack clears config.dataDir as part
+ // of its swap, so a crash after that point left a rerun reading "this
+ // channel is not relocated" from the config and throwing — while a marker
+ // sat next to it naming the very target still holding the media, and
+ // inspect() reported in-transition forever.
+ const resume = existingMarker?.direction === "back" ? existingMarker : null;
+ const target = config.dataDir?.trim() || resume?.target;
if (!target) {
throw new Error(
`Channel "${slug}" is not relocated — its media is already in place`,
@@ -302,6 +370,16 @@ export async function relocateChannelMedia(
`A relocation to ${existingMarker.target} is already in progress for "${slug}"`,
);
}
+ // A marker for the OTHER direction is never resumed into this one. Same
+ // target, opposite intent: an interrupted move-out has a full copy on the
+ // platter and a real dir here, and treating that as a move-back to resume
+ // would swap the wrong way round.
+ if (existingMarker && !resume) {
+ throw new Error(
+ `A move-out to ${existingMarker.target} is in flight or was interrupted ` +
+ `for "${slug}" — finish that before moving back`,
+ );
+ }
return moveBack({
paths,
slug,
@@ -310,8 +388,8 @@ export async function relocateChannelMedia(
target,
log,
signal,
- resumed: Boolean(existingMarker),
- phase: existingMarker?.direction === "back" ? existingMarker.phase : "copy",
+ resumed: Boolean(resume),
+ phase: resume?.phase ?? "copy",
});
}
@@ -334,6 +412,14 @@ export async function relocateChannelMedia(
);
}
+ const resume = existingMarker?.direction === "out" ? existingMarker : null;
+ if (existingMarker && !resume) {
+ throw new Error(
+ `A move-back from ${existingMarker.target} is in flight or was interrupted ` +
+ `for "${slug}" — finish that before moving out again`,
+ );
+ }
+
return moveOut({
paths,
slug,
@@ -344,8 +430,10 @@ export async function relocateChannelMedia(
target,
log,
signal,
- resumed: Boolean(existingMarker),
- phase: existingMarker?.direction === "out" ? existingMarker.phase : "copy",
+ // `resumed` relaxes the "must be in-place" precondition, so it must mean
+ // "this run continues an interrupted move OUT" and nothing looser.
+ resumed: Boolean(resume),
+ phase: resume?.phase ?? "copy",
});
}
@@ -372,6 +460,14 @@ async function moveOut(args: {
`(is the drive mounted?)`,
);
}
+ // Writability, checked explicitly rather than discovered by rsync's exit code
+ // three minutes in. A read-only mount is the ordinary way a platter comes
+ // back after a bad shutdown.
+ try {
+ await access(root, fsConstants.W_OK);
+ } catch {
+ throw new Error(`The destination root ${root} is not writable`);
+ }
// A RESUMING RUN MUST TOLERATE ITS OWN MARKER. `inspectChannelMedia` reports
// "in-transition" for any channel carrying one, which is exactly right for
// every guard and exactly wrong here: the rerun that finishes an interrupted
@@ -386,6 +482,16 @@ async function moveOut(args: {
);
}
+ // A channel that has downloaded nothing has no data/ at all, and rsync exits
+ // 23 on a missing source — an error message about a partial transfer for
+ // something that is not a transfer. Say what is actually the matter.
+ if (args.phase === "copy" && !(await isDirectory(dataDir))) {
+ throw new Error(
+ `Channel "${slug}" has no ${dataDir} to move — nothing has been ` +
+ `downloaded for it yet`,
+ );
+ }
+
const measured = await measureTree(dataDir);
log(
`Relocating ${slug}: ${measured.files} file(s), ${formatBytes(measured.bytes)} ` +
@@ -394,11 +500,19 @@ async function moveOut(args: {
let phase = args.phase;
if (phase === "copy") {
+ // THE BAR IS THE BYTES PLUS THE RESUME MARGIN, not the bytes. Landing the
+ // media with nothing to spare puts the destination volume under the disk
+ // gate's own floor the moment it arrives, so the channel's next download is
+ // refused by the gate on the drive it was just moved to. The margin is the
+ // operator's configured one, so the two numbers cannot drift apart.
+ const marginGB = getSettings().resumeMarginGB;
+ const needed = measured.bytes + marginGB * BYTES_PER_GB;
const free = await getFreeBytes(root);
- if (free < measured.bytes) {
+ if (free < needed) {
throw new Error(
`Not enough space on ${root}: ${formatBytes(free)} free, ` +
- `${formatBytes(measured.bytes)} to move`,
+ `${formatBytes(measured.bytes)} to move plus a ${marginGB} GB resume ` +
+ `margin = ${formatBytes(needed)} required`,
);
}
await mkdir(target, { recursive: true });
@@ -438,17 +552,60 @@ async function moveOut(args: {
startedAt: new Date().toISOString(),
phase: "swap",
});
- // Same device, so the rename is atomic: `data/` is a real dir one instant
- // and the parked copy the next, never half of each.
- const parked = path.join(channelDir, `data.relocated-${Date.now()}`);
- if (await pathExists(dataDir)) {
- await rename(dataDir, parked);
+
+ // EVERY STEP BELOW OBSERVES THE DISK INSTEAD OF ASSUMING THE LAST ONE RAN.
+ // The marker says how far the previous attempt got; it cannot say how far
+ // it got THROUGH a phase, and a crash lands between any two syscalls.
+ const state = await dataDirState(dataDir);
+ const parked = (await parkedSiblings(channelDir)).filter((p) =>
+ path.basename(p).startsWith("data.relocated-"),
+ );
+
+ // Re-verify, because a resumed run did not do the copy in this process and
+ // must not take the interrupted one's word for it. The source to verify
+ // against is whichever copy of it still exists; once the swap has committed
+ // there is none, and there is nothing left to check.
+ const verifySrc =
+ state.kind === "real-dir" ? dataDir : (parked[0] ?? null);
+ if (verifySrc) {
+ log("Verifying the copy…");
+ await verifyCopy({ paths, src: verifySrc, dest: target, log, signal });
+ }
+
+ if (state.kind === "real-dir") {
+ // Same device, so the rename is atomic: `data/` is a real dir one instant
+ // and the parked copy the next, never half of each. A fresh name even
+ // when a parked dir already exists — renaming onto a non-empty directory
+ // is ENOTEMPTY, and the reclaim sweep takes all of them anyway.
+ await rename(dataDir, path.join(channelDir, `data.relocated-${Date.now()}`));
+ } else if (state.kind === "other") {
+ throw new Error(
+ `${dataDir} is neither a directory nor a symlink — refusing to replace it`,
+ );
+ }
+
+ const after = await dataDirState(dataDir);
+ if (
+ after.kind === "link" &&
+ path.resolve(after.linkTarget) !== path.resolve(target)
+ ) {
+ // A link to somewhere else is not this move's work to reinterpret, and
+ // silently repointing it would strand whatever it does point at.
+ throw new Error(
+ `${dataDir} already points at ${after.linkTarget}, not ${target}`,
+ );
}
- await symlink(target, dataDir);
+ if (after.kind === "missing") {
+ await symlink(target, dataDir);
+ }
+
// Written only now, on success: config.dataDir is a record of what is on
- // disk, never an intention.
+ // disk, never an intention. Skipped when it already says so, so a rerun
+ // does not rewrite a file it agrees with.
const fresh = (await readChannelConfig(paths, slug)) ?? args.config;
- await writeChannelConfig(paths, slug, { ...fresh, dataDir: target });
+ if (fresh.dataDir?.trim() !== target) {
+ await writeChannelConfig(paths, slug, { ...fresh, dataDir: target });
+ }
log(`Swapped: ${dataDir} -> ${target}`);
await writeMarker(paths, slug, {
target,
@@ -456,19 +613,15 @@ async function moveOut(args: {
startedAt: new Date().toISOString(),
phase: "reclaim",
});
- if (await pathExists(parked)) {
- await rm(parked, { recursive: true, force: true });
- }
- } else {
- // Resuming at "reclaim": the swap already happened, so any parked copy left
- // behind is the only thing outstanding.
- for (const name of await readdir(channelDir).catch(() => [] as string[])) {
- if (name.startsWith("data.relocated-")) {
- await rm(path.join(channelDir, name), { recursive: true, force: true });
- }
- }
}
+ // RECLAIM RUNS ON EVERY PATH, not only on a resume. A crash between the
+ // config write and the reclaim marker used to orphan a full second copy of
+ // the channel on the source volume — the disk the move exists to free — and
+ // the sweep that would have caught it was in the branch a fresh run never
+ // takes. It sweeps every sibling, not the one name this process minted.
+ await sweepParked(channelDir, log);
+
await clearMarker(paths, slug);
const freeNow = await getFreeBytes(paths.channelsDir);
log(
@@ -497,13 +650,22 @@ async function moveBack(args: {
phase: RelocationPhase;
}): Promise<RelocateChannelMediaResult> {
const { paths, slug, channelDir, dataDir, target, log, signal } = args;
- if (!(await isDirectory(target))) {
+ const incoming = path.join(channelDir, "data.incoming");
+ // THE PRECONDITION BELONGS TO THE COPY, NOT TO THE RERUN. Past the swap the
+ // media is already back in the channel dir and the target may well be gone —
+ // demanding it be reachable there would refuse the very run that finishes
+ // cleaning up after an interrupted move.
+ if (args.phase === "copy" && !(await isDirectory(target))) {
throw new Error(
`The relocated media at ${target} is not reachable (is the drive mounted?)`,
);
}
- const measured = await measureTree(target);
- const incoming = path.join(channelDir, "data.incoming");
+ // Measured off whichever copy still exists, in the order they stop existing.
+ const measured = (await isDirectory(target))
+ ? await measureTree(target)
+ : (await isDirectory(incoming))
+ ? await measureTree(incoming)
+ : await measureTree(dataDir);
log(
`Moving ${slug} back in place: ${measured.files} file(s), ` +
`${formatBytes(measured.bytes)} <- ${target}`,
@@ -552,16 +714,36 @@ async function moveBack(args: {
startedAt: new Date().toISOString(),
phase: "swap",
});
- // unlink, not rm -r: `data` is the LINK here, and removing it recursively
- // would be the one way this whole design eats the media.
- await unlink(dataDir).catch(() => {});
- await rename(incoming, dataDir);
+
+ // OBSERVE, DO NOT ASSUME. The old sequence was `unlink(data)` (swallowing
+ // its error) then `rename(incoming, data)`, which is idempotent in exactly
+ // the case that never happens: a crash AFTER the rename left `data/` a real
+ // directory, the swallowed unlink then failed on it, and the rename ENOENTed
+ // on an `incoming` that no longer existed — forever, on every rerun.
+ const state = await dataDirState(dataDir);
+ if (state.kind === "real-dir") {
+ // The rename already committed. Nothing to swap; the leftovers are the
+ // reclaim's business.
+ log(`${dataDir} is already a real directory — the swap had completed`);
+ } else {
+ if (!(await isDirectory(incoming))) {
+ throw new Error(
+ `Cannot finish moving "${slug}" back: ${dataDir} is not a directory ` +
+ `and there is no verified copy at ${incoming}`,
+ );
+ }
+ // unlink, not rm -r: `data` is the LINK here, and removing it recursively
+ // would be the one way this whole design eats the media.
+ if (state.kind !== "missing") await unlink(dataDir);
+ await rename(incoming, dataDir);
+ log(`Swapped: ${dataDir} is a real directory again`);
+ }
+
const fresh = await readChannelConfig(paths, slug);
- if (fresh) {
+ if (fresh?.dataDir !== undefined) {
const { dataDir: _dropped, ...rest } = fresh;
await writeChannelConfig(paths, slug, rest);
}
- log(`Swapped: ${dataDir} is a real directory again`);
await writeMarker(paths, slug, {
target,
direction: "back",
@@ -576,6 +758,9 @@ async function moveBack(args: {
if ((await readdir(slugRoot).catch(() => ["keep"])).length === 0) {
await rm(slugRoot, { recursive: true, force: true });
}
+ // Any half-copied `data.incoming` (or a parked dir from an earlier move out)
+ // goes with it — the same sweep, for the same reason.
+ await sweepParked(channelDir, log);
await clearMarker(paths, slug);
log(`Done. ${formatBytes(measured.bytes)} back in place.`);
return {
diff --git a/common/lib/channelMedia.test.ts b/common/lib/channelMedia.test.ts
@@ -6,6 +6,7 @@ import path from "node:path";
import {
ChannelMediaUnreachableError,
assertChannelMediaReachable,
+ clearRelocationMarker,
inspectChannelMedia,
readRelocationMarker,
relocatedDataDir,
@@ -250,3 +251,31 @@ test("an unreadable or missing config.json is not a relocation", async () => {
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "in-place");
});
});
+
+test("clearRelocationMarker removes the marker and touches nothing else", async () => {
+ await withTmp(async (paths) => {
+ const target = path.join(paths.channelsDir, "..", "platter", "alpha", "data");
+ await mkdir(target, { recursive: true });
+ const channelDir = await seedChannel(paths, "alpha", { dataDir: target });
+ await symlink(target, path.join(channelDir, "data"));
+ await writeFile(
+ path.join(channelDir, RELOCATION_MARKER_FILENAME),
+ JSON.stringify({ target, direction: "out", phase: "swap", startedAt: "" }),
+ );
+ // A marker outranks everything: the channel is in transition, which is the
+ // dead end this escape hatch exists for when the run that wrote it is gone.
+ assert.equal(
+ (await inspectChannelMedia(paths, "alpha")).status,
+ "in-transition",
+ );
+
+ await clearRelocationMarker(paths, "alpha");
+
+ assert.equal(await readRelocationMarker(paths, "alpha"), null);
+ // The link, the config and the media are exactly as they were — what is
+ // left is the truth the disk was already telling underneath the marker.
+ assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
+ // Idempotent: clearing a marker that is not there is not an error.
+ await clearRelocationMarker(paths, "alpha");
+ });
+});
diff --git a/common/lib/channelMedia.ts b/common/lib/channelMedia.ts
@@ -1,5 +1,5 @@
import path from "node:path";
-import { lstat, readFile, readlink, stat } from "node:fs/promises";
+import { lstat, readFile, readlink, rm, stat } from "node:fs/promises";
import type { Paths } from "./paths";
import type { ChannelConfig } from "./channelConfig";
@@ -144,6 +144,26 @@ export async function readRelocationMarker(
}
}
+// THE OPERATOR'S LAST RESORT, and the only writer in this module.
+//
+// A channel carrying a marker is "in-transition" to every guard, which is
+// correct while a move is running and a dead end once one is not: the runners
+// skip the channel, runManagedFunction refuses its media jobs, and the snapshot
+// will not regenerate. The relocate job itself resumes from a marker and clears
+// it on success, so this is not the normal way out — it is for a marker whose
+// run is gone (a killed process, a container replaced mid-copy) and whose state
+// on disk the operator has looked at.
+//
+// It removes the marker and NOTHING else: no link is touched, no config is
+// rewritten, nothing is deleted. Whatever inspect() says afterwards is the truth
+// the disk was already telling, with the transition claim taken off the top.
+export async function clearRelocationMarker(
+ paths: ChannelMediaPaths,
+ slug: string,
+): Promise<void> {
+ await rm(relocationMarkerPath(paths, slug), { force: true });
+}
+
// The `dataDir` field alone, read straight off config.json. Deliberately NOT
// parseChannelConfig: this runs in guards on hot paths and must not depend on
// the controller that owns the rest of the schema.
