import path from "node:path"; import { access, constants as fsConstants, lstat, mkdir, readdir, realpath, rename, rm, stat, symlink, unlink, } from "node:fs/promises"; import type { Paths } from "../lib/paths"; import { clearDirMarker, copyMirrorVerify, isDirectory, linkOrDirState, makeProgressSink, measureTree, pathExists, readDirMarker, verifyCopy, writeDirMarker, type RelocationProgress, } from "./relocateDir"; import { channelWriters, channelWritersRefusal, type ChannelWriter, type ChannelWritersOptions, } from "./channelWriters"; import { kindNeedsMedia } from "../jobs/jobKinds"; import { isSocialChannel } from "../lib/channelConfig"; import { locationOfDataDir, type StorageLocation, type StorageSettings, } from "../lib/storageLocations"; import { probeLocationMemo, type ProbeOptions, type VolumeBins, } from "../lib/storageVolumes"; import { getFreeBytes } from "../lib/diskSpace"; import { NOT_ANSWERING, isDriveNotAnswering, onDrive, sinceText, stalledLocation, } from "../lib/storageHealth"; import { getSettings, type SiteSettings } from "../lib/settings"; import { formatBytes } from "../lib/format"; import { forgetChannelMedia, inspectChannelMedia, legacyDetail, relocationMarkerPath, type RelocationDirection, type RelocationMarker, type RelocationPhase, } from "../lib/channelMedia"; import { channelMediaLink, relocatedMediaDir, tierChannelMedia, } from "../lib/mediaTier-server"; import { patchChannelConfig, readChannelConfig, writeChannelConfig, } from "./channels"; // MOVE A CHANNEL'S MEDIA TIER TO ANOTHER DRIVE, AND BACK (release 17). // // The unit of a move is `channels//media` — the channel's BIG files // (lib/mediaTier.ts says which), each reached from `data//` by a // RELATIVE link `../../media//`. The text — transcripts, cues, // metadata, every sidecar, `clips/` — never moves: `data/` stays a real // directory on the corpus disk. A move copies `media/` to `//media`, // verifies it, and makes `channels//media` ONE absolute link to it while // `config.json` records the target in `mediaDir`. Not one per-file link changes, // in either direction: they are relative to `media`, whatever `media` is. // // A CLASSIC CHANNEL IS TIERED FIRST. One whose big files are still real files // in `data//` (no `media/` yet) is tiered in place by the move's preflight — // `media/` made a real directory, each file renamed into it and linked, all on // one filesystem, in seconds — and then moved like any other. // // THE SOURCE IS NEVER TOUCHED UNTIL THE COPY IS VERIFIED. An abort, a full // target, a crash, an rsync failure — all of them leave `media/` exactly where // it was, and leave the partial copy resumable. The only destructive step is // the reclaim at the very end, after the link is in place and the config is // written. // // `rsync -a` preserves mtimes, and the LMDB index stats no media file at all // (presence by name, from one readdir of `data//`), so a move needs no // reindex. // // THE RETIRED LAYOUT IS REFUSED. A channel whose whole `data/` was moved by the // mover before release 17 (`data/` a link, `config.dataDir`) is `legacy`: it is // migrated by `archilyzer storage migrate-tier`, never moved by this. const BYTES_PER_GB = 1024 ** 3; export type RelocateChannelMediaResult = { slug: string; direction: RelocationDirection; // Absolute path of the relocated media dir, `//media`. For // "back" this is what was reclaimed, not where the media now lives. target: string; bytes: number; files: number; // How many big files the preflight tiered into `media/` before the copy (a // classic channel's first move; 0 for "back" and for a channel already // tiered). tiered: number; // True when the run picked up an interrupted one from its marker rather than // starting from scratch. resumed: boolean; // True when the verify found the source had changed after the mirror pass // (a directory timestamp, or a file written or removed) and one more mirror // pass settled it — worth saying on the job's done line, because an operator // who has seen this refuse a copy should be told it did not this time. A // second change is a refusal, never this. retried: boolean; // True only when a Reconcile and resume actually reconciled: it found the // move in its copy phase and made the destination copy match the source. A // reconcile of a marker already past the copy phase has nothing to // reconcile and is a plain resume — the done line says "resumed". reconciled: boolean; }; // The progress shape is the shared mover's — re-exported because callers // (the editor's relocate job) reach for it through this module, which is the // one they already import. export type { RelocationProgress } from "./relocateDir"; type RelocateOpts = { paths: Paths; slug: string; direction: RelocationDirection; // Required for "out"; ignored for "back", which reads the target from config. root?: string; onLog?: (line: string) => void; // Called on every rsync redraw of the COPY phase (never the verify, which // transfers nothing). Optional: a bin script that only wants the log passes // nothing and pays only the parse. onProgress?: (p: RelocationProgress) => void; signal?: AbortSignal; // THE SETTINGS READ, INJECTABLE — the same seam `relocateSavedVideos` already // takes for the same reason, and the reason is not style. // // The space check demands `bytes + resumeMarginGB` free on the destination, // and `resumeMarginGB` defaults to 2 GB. Every test in this file builds its // corpus and its "platter" under `os.tmpdir()`, which on a Linux box is a // TMPFS sized from RAM — 1.8 GB free here — so eight of them failed with // "Not enough space ... 2.9 KB to move plus a 2 GB resume margin", on a // machine, in a suite, that had nothing to do with disk space. A test that // fails because of how much RAM the machine happens to have left is not // testing the mover. // // Production passes nothing and gets `getSettings()`, unchanged. io?: { read: () => SiteSettings }; // RECONCILE AND RESUME: finish an interrupted or refused move by making the // destination copy match the source — the mirror pass, the verify, then the // swap and the reclaim — with no copy pass first. Refused without a marker // to finish. The panel's remediation for a move whose verify failed; the // operator never deletes a file on the destination by hand. reconcile?: boolean; // WHO IS WRITING INTO THE CHANNEL, injectable for the tests. Production asks // the live job registry and the live lane runners (channelWriters.ts), // leaving out this job itself. writers?: (slug: string) => ChannelWriter[]; }; // The move's own kind is not a writer to refuse over: it is the asker, running // on this channel's slug, and the relocation queue runs one move at a time. const RELOCATE_KIND = "relocate-channel-media"; // MEDIA WRITERS ONLY (release 17 ruling): a move carries `media/` and nothing // else, so a digest, a normalize or any other reader/writer of the TEXT may run // while it does. Kept: the jobs whose kind opens or writes a big file // (`kindNeedsMedia`), the lane units of every lane but the digest one — and a // MOVE of this channel's media itself, whose kind is not `needsMedia` (it must // not be refused by the media guard it is the reason for) but which is, of // everything, the thing writing into `media/`. In the order `channelWriters` // gives (running jobs, queued jobs, lane units), so the refusal names the same // writer first. // // The Storage panel's actions, the preview and the job's own first step all // ask this; a rename or a delete of the channel asks every writer. export function channelMediaWriters( slug: string, opts: Omit = {}, ): ChannelWriter[] { return channelWriters(slug, opts).filter((w) => w.source === "job" ? kindNeedsMedia(w.kind) || w.kind === RELOCATE_KIND : w.lane !== "digest", ); } function liveWriters(slug: string): ChannelWriter[] { return channelMediaWriters(slug, { ignoreKinds: [RELOCATE_KIND] }); } // THE RETIRED WHOLE-DIRECTORY LAYOUT, refused by every entry point of the mover // with the sentence that names the way out. Answered from the corpus disk alone // (`data/` a link, or `config.dataDir` recorded) — never a call to the far // drive. async function legacyRefusal( channelDir: string, slug: string, config: { dataDir?: string } | null, ): Promise { let dataIsLink = false; try { dataIsLink = (await lstat(path.join(channelDir, "data"))).isSymbolicLink(); } catch { /* no data/ yet */ } if (!dataIsLink && !config?.dataDir?.trim()) return null; return `Channel "${slug}" cannot be moved: ${legacyDetail(slug)}.`; } // A TIER MIGRATION'S MARKER is not this mover's to resume, replace or clear: // the migration rebuilds `data/` itself, with the editor stopped, and resumes // from its marker's phase. ONE sentence, for the job, the preview, and the // Storage panel's Resume, Reconcile and Clear marker. export function tierMigrationRefusal( slug: string, marker: RelocationMarker | null, ): string | null { if (marker?.scope !== "tier-migration") return null; return ( `Channel "${slug}" has a media-tier migration in flight or interrupted ` + `(phase "${marker.phase}") — finish it with archilyzer storage ` + `migrate-tier ${slug}.` ); } // A MOVE NEVER STARTS OVER A WRITER, and never waits silently for one either: it // refuses, naming the job (release 16 slice RM). Asked by the preview, by the // job's first step, and again the moment the marker is written — after which // the lanes skip the channel and a media job refuses to start on it, so the // third answer is the one that cannot go stale. function assertNoWriters( slug: string, writers: (slug: string) => ChannelWriter[], ): void { const refusal = channelWritersRefusal(slug, writers(slug)); if (refusal) throw new Error(refusal); } export type RelocationPreview = { slug: string; target: string; bytesToMove: number; files: number; freeOnRoot: number; freeOnSource: number; // The move is pointless (and rename-unsafe assumptions change) when the root // is the volume the corpus is already on. sameDevice: boolean; // A resumable partial copy from an earlier attempt is already at the target. existingPartial: boolean; // Big files the preview tiered into `media/` first (a classic channel: its // audio and raw live chat renamed into `channels//media//` and // linked, on the corpus disk). Idempotent: 0 on a second preview. tieredFirst: number; }; // Resolved through every symlink of its DEEPEST EXISTING ANCESTOR, the rest // joined on lexically. A destination root that does not exist yet is refused // elsewhere; a root that exists and is a symlink back into the corpus is // exactly what this is for. The ancestor walk matters since release 17: the // target `//media` usually does not exist yet, and a `` level // that links back into the channel dir must still resolve there. async function realOrResolved(p: string): Promise { const abs = path.resolve(p); try { return await realpath(abs); } catch { const parent = path.dirname(abs); if (parent === abs) return abs; return path.join(await realOrResolved(parent), path.basename(abs)); } } // `child` IS `parent`, or lives under it. function isWithin(parent: string, child: string): boolean { const rel = path.relative(parent, child); return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel)); } // WHY A ROOT INSIDE THE CORPUS IS NOT MERELY POINTLESS BUT DESTRUCTIVE. // // Take `root = /channels`. Then `relocatedMediaDir(root, slug)` // is `//media` — the SOURCE. `rsync -a src/ src/` succeeds, // verifyCopy compares the tree with itself and passes, the swap renames `media/` // to `media.relocated-` (which moves the "target" it just verified), creates // a symlink pointing at a path that no longer exists, and the reclaim sweep then // `rm -r`s the parked directory — the only copy of the media. Nothing in the // happy path can notice, because every check it runs is comparing the tree with // itself. The panel's own help text ("an absolute directory that already // exists") describes `transcripts/channels` almost word for word. // // So containment is checked, in one place, and called from all three: the // preview the operator reads, the action that enqueues, and the job that moves. // A root inside the corpus is refused whether it frees anything or not — the // same-device hint stays a hint, and this is a refusal. export async function relocationRootProblem(opts: { paths: Paths; slug: string; root: string; // Test seam only. Defaults to the live settings, because the whole point of // asking here is that the preview, the action and the job give one answer. storage?: StorageSettings; probeOpts?: ProbeOptions; }): Promise { const root = opts.root.trim(); if (!root) return "No destination root given"; if (!path.isAbsolute(root)) { return `The destination root must be an absolute path (got "${root}")`; } const channelDir = path.join(opts.paths.channelsDir, opts.slug); const [realRoot, realCorpus, realChannel] = await Promise.all([ realOrResolved(root), realOrResolved(opts.paths.transcriptsDir), realOrResolved(channelDir), ]); if (isWithin(realCorpus, realRoot)) { return ( `The destination root ${root} is inside the corpus at ` + `${opts.paths.transcriptsDir}. A relocation there would copy the channel ` + `onto itself and then reclaim the only copy — pick a directory on the ` + `other drive.` ); } // Belt and braces for a root that is outside the corpus but whose `` // level is a link back into it: the target is resolved separately, because // realpath of the root cannot see through a link one level down. const realTarget = await realOrResolved(relocatedMediaDir(realRoot, opts.slug)); if (isWithin(realChannel, realTarget) || isWithin(realTarget, realChannel)) { return ( `The destination ${relocatedMediaDir(root, opts.slug)} resolves inside ` + `the channel directory ${channelDir} — the media would be copied onto ` + `itself and then reclaimed.` ); } if (isWithin(realCorpus, realTarget)) { return ( `The destination ${relocatedMediaDir(root, opts.slug)} resolves inside ` + `the corpus at ${opts.paths.transcriptsDir}. Pick a directory on the ` + `other drive.` ); } // LAST, because containment is the destructive answer and presence is the // merely-wrong one. Same call the job makes immediately before its mkdir, so // the preview the operator reads and the run that moves the bytes cannot // disagree about whether the drive is there. return relocationRootPresenceProblem( root, opts.storage ?? getSettings().storage, opts.paths, opts.probeOpts, ); } // A MOVE MUST NEVER MATERIALISE A MOUNT, and `mkdir -p` is exactly what does. // // Every absolute-path `mkdir(…, {recursive: true})` in this file and in // relocateSavedVideos.ts creates whatever is missing above it. Point a move at // `/mnt/platter/archilyzer-media` with the platter unplugged and the mkdir // cheerfully builds that path ON THE ROOT FILESYSTEM, rsync fills it, the // channel's symlink is rewritten to it, and the operator has silently moved a // channel onto the disk the move existed to free — with the real media still on // the unmounted platter, and `/mnt/platter` now non-empty so the platter can no // longer mount there. // // So: before any of those mkdirs, and from `relocationRootProblem` so the // PREVIEW the operator reads gives the same answer the action does. // // Two checks, and the second is the one that catches the case above: // // 1. `stat(root)` must be a directory. The move creates `/` and // `//media`, never the root itself — a root is a fact about the // machine, not something a move gets to invent. // 2. When the root belongs to a LOCATION that has learned a `volume.uuid`, // the probe must answer `available` with a KNOWN identity whose uuid // matches. An empty mountpoint directory sitting on the root filesystem // passes check 1 and fails this one, because findmnt -T reports the root // filesystem's uuid, which is not the platter's. // // FAILS OPEN ON UNKNOWN IDENTITY, deliberately. `probeLocation`'s identity is // unknown when findmnt is missing or wedged (a container has no block devices // at all), and refusing every move on a machine that cannot answer the question // would break relocation for exactly the deployments that most need it. Only a // KNOWN MISMATCH refuses. // // A root nobody named as a location is STAT-ONLY: there is no recorded identity // to compare against, so there is nothing to compare. That is the documented // gap and the nudge out of it — an unmounted fstab mountpoint directory is // precisely what a location protects you from, so add it on /storage. function locationForRoot( root: string, locations: StorageLocation[], ): StorageLocation | null { // `root + "/x"` rather than `root`: `locationOfDataDir` is deliberately // STRICT ("under", not "equal to"), because a channel's mediaDir is always // `//media` and equality there only ever means a misconfiguration. // Here equality is the ordinary case — the destination root IS the location // root — so the question is asked about a path one level inside it. return locationOfDataDir(path.join(root, "x"), locations); } export async function relocationRootPresenceProblem( root: string, storage: StorageSettings, bins: VolumeBins, probeOpts?: ProbeOptions, ): Promise { const r = root.trim(); if (!r) return "No destination root given"; const named = locationForRoot(r, storage.locations); const where = named ? ` (location "${named.id}")` : ""; // A destination whose drive is not answering is refused WITHOUT the stat // below: that stat would wait on the drive, and a move onto it would too. const stall = named ? stalledLocation(named) : null; if (stall) { return ( `The destination root ${r}${where}: ${NOT_ANSWERING} ` + `(${sinceText(stall.since)}). Wait for it to answer, or check the drive.` ); } // Through the watchdog: a stat that has not answered within the budget // (`storage.health.budgetMs`, 3 s by default) marks the location stalled // and refuses the same way. let isDir = false; try { isDir = (await onDrive(named ?? r, () => stat(r))).isDirectory(); } catch (err) { if (isDriveNotAnswering(err)) { return ( `The destination root ${r}${where}: ${NOT_ANSWERING} ` + `(${err.health ? sinceText(err.health.since) : "a stat of it did not answer"}). ` + `Wait for it to answer, or check the drive.` ); } isDir = false; } if (!isDir) { // THE SAME SENTENCE the pre-existing existence check uses, deliberately: // two refusals for one condition that read differently is two bugs to // report. This one fires first and adds where the root was supposed to be. return ( `The destination root ${r} does not exist or is not a directory${where}. ` + `A move creates /, never the root itself — ` + (named ? `mount the drive or re-point the location first.` : `create it first.`) ); } if (!named?.volume?.uuid) return null; // MEMOIZED, because this is called once per channel by the bulk move and by // the re-point preflight — seventy-one rows times three subprocesses is the // thing the memo exists to stop. Ten seconds; `refresh` is what the operator // presses after plugging a disk in. const probe = await probeLocationMemo(named, bins, probeOpts); if (probe.status !== "available") { return ( `The destination root ${r} is ${probe.status}${where}. ` + `Mount it or re-point the location first.` ); } if (probe.identity.known && probe.identity.uuid !== named.volume.uuid) { return ( `The destination root ${r}${where} is on volume ` + `${probe.identity.uuid}, not the ${named.volume.uuid} this location was ` + `last seen on — the drive is not mounted there. Mount it or re-point ` + `the location first.` ); } return null; } // The inverse of `relocatedMediaDir`: `//media` -> ``. The // suffix is fixed (mediaTier-server.ts says so, and deleteChannel recognises a // target by it), so this is two dirnames and not a guess. export function rootOfRelocatedMediaDir(target: string, slug: string): string { const parent = path.dirname(target); return path.basename(parent) === slug ? path.dirname(parent) : parent; } export async function assertRelocationRootPresent( root: string, storage: StorageSettings, bins: VolumeBins, probeOpts?: ProbeOptions, ): Promise { const problem = await relocationRootPresenceProblem( root, storage, bins, probeOpts, ); if (problem) throw new Error(problem); } // Every leftover a crashed run can have parked next to `media/`, in one list. // The reclaim phase sweeps ALL of them rather than the one name the run that is // finishing happens to hold: a crash between the config write and the reclaim // marker leaves a full second copy of the channel on the source volume, and // reclaiming only the name this process minted would orphan it forever — on the // disk the move exists to free. async function parkedSiblings(channelDir: string): Promise { const names = await readdir(channelDir).catch(() => [] as string[]); return names .filter((n) => n.startsWith("media.relocated-") || n === "media.incoming") .map((n) => path.join(channelDir, n)); } async function sweepParked( channelDir: string, log: (m: string) => void, ): Promise { for (const p of await parkedSiblings(channelDir)) { log(`Reclaiming ${p}`); await rm(p, { recursive: true, force: true }); } } // The channel's marker, at `channels//.relocating.json`. The FILE is the // contract — see relocateDir.ts — and these three are the channel's name for it. // // Each write and the clear also drop the channel from `inspectChannelMedia`'s // five-second page memo. Every phase change (copy → swap → reclaim) writes the // marker AFTER the link and the config it changes, so a page asks the disk // again the moment the move has done something it would see. async function writeMarker( paths: Paths, slug: string, marker: RelocationMarker, ): Promise { await writeDirMarker(relocationMarkerPath(paths, slug), marker); forgetChannelMedia(slug); } async function clearMarker(paths: Paths, slug: string): Promise { await clearDirMarker(relocationMarkerPath(paths, slug)); forgetChannelMedia(slug); } async function readMarkerRaw( paths: Paths, slug: string, ): Promise { return readDirMarker(relocationMarkerPath(paths, slug)); } // What the operator sees before committing to a move. Cheap enough to run on a // form keystroke debounce: one tree walk of the channel's `media/` plus two // statfs calls — and, the first time for a classic channel, the tiering (the // same renames the job's preflight would make, on the corpus disk; idempotent). export async function previewRelocation({ paths, slug, root, writers = (s) => channelMediaWriters(s), }: { paths: Paths; slug: string; root: string; // Test seam; the live registry and lanes otherwise. A running move of this // channel counts here — the preview is not that move. writers?: (slug: string) => ChannelWriter[]; }): Promise { // The preview REFUSES rather than reporting numbers for a root the job will // reject: its whole job is to answer "is this root usable" before the // operator commits, and a plausible pair of figures for a destructive root is // the worst possible answer. const problem = await relocationRootProblem({ paths, slug, root }); if (problem) throw new Error(problem); const channelDir = path.join(paths.channelsDir, slug); const config = await readChannelConfig(paths, slug); const migrating = tierMigrationRefusal(slug, await readMarkerRaw(paths, slug)); if (migrating) throw new Error(migrating); const legacy = await legacyRefusal(channelDir, slug, config); if (legacy) throw new Error(legacy); // The same refusal the job's first step gives, before the operator commits. assertNoWriters(slug, writers); // EXISTENCE, HERE AS WELL AS IN THE JOB. getFreeBytes walks up to the nearest // existing ancestor, so a typo'd root statfs's its parent and previews with // perfectly plausible numbers — and the move is then refused by the job, after // the operator has already read a confirmation. if (!(await isDirectory(root))) { throw new Error( `The destination root ${root} does not exist or is not a directory ` + `(is the drive mounted?)`, ); } // TIER FIRST, as the job will: the figures below are then the ones the job // moves. Only an in-place channel (a relocated one's tier is on the far // drive, and a preview copies nothing there), only with nothing in flight (a // marker means a move has already tiered it, and the hook writes nothing // under one anyway), and only one with something downloaded (no `data/` → no // `media/` invented). let tieredFirst = 0; const marker = await readMarkerRaw(paths, slug); if ( !marker && !config?.mediaDir?.trim() && (await isDirectory(path.join(channelDir, "data"))) ) { tieredFirst = ( await tierChannelMedia(paths, slug, { createMediaDir: true }) ).tiered; } const source = channelMediaLink(paths, slug); const target = relocatedMediaDir(root, slug); const [measured, freeOnRoot, freeOnSource, existingPartial] = await Promise.all([ measureTree(source), getFreeBytes(root), getFreeBytes(paths.channelsDir), pathExists(target), ]); let sameDevice = false; try { const [a, b] = await Promise.all([stat(paths.channelsDir), stat(root)]); sameDevice = a.dev === b.dev; } catch { /* an unmounted or absent root is not "same device" */ } return { slug, target, bytesToMove: measured.bytes, files: measured.files, freeOnRoot, freeOnSource, sameDevice, existingPartial, tieredFirst, }; } export async function relocateChannelMedia( opts: RelocateOpts, ): Promise { const { paths, slug, direction, signal } = opts; const log = opts.onLog ?? ((m: string) => console.log(m)); const io = opts.io ?? { read: getSettings }; const onProgress = opts.onProgress; const config = await readChannelConfig(paths, slug); if (!config) throw new Error(`Channel "${slug}" not found`); if (isSocialChannel(config)) { throw new Error( `Channel "${slug}" is a social channel — it has no downloaded media to relocate`, ); } // THE JOB'S FIRST STEP: nothing may be writing into the channel. The action // that enqueued this asked too, but a move can wait a long time in the // relocation queue, and on 2026-09-30 a transcription started in exactly // that wait. const writers = opts.writers ?? liveWriters; assertNoWriters(slug, writers); const channelDir = path.join(paths.channelsDir, slug); const mediaLink = channelMediaLink(paths, slug); const existingMarker = await readMarkerRaw(paths, slug); const migrating = tierMigrationRefusal(slug, existingMarker); if (migrating) throw new Error(migrating); const legacy = await legacyRefusal(channelDir, slug, config); if (legacy) throw new Error(legacy); if (opts.reconcile && !existingMarker) { throw new Error( `Channel "${slug}" has no relocation marker — there is no interrupted ` + `move to reconcile.`, ); } const reconcile = opts.reconcile === true; if (direction === "back") { // THE MARKER IS THE SECOND SOURCE OF TRUTH FOR THE TARGET, and without it // the resume path was unreachable. moveBack clears config.mediaDir as part // of its swap, so a crash after that point left a rerun reading "this // channel is not relocated" from the config and throwing — while a marker // sat next to it naming the very target still holding the media, and // inspect() reported in-transition forever. const resume = existingMarker?.direction === "back" ? existingMarker : null; const target = config.mediaDir?.trim() || resume?.target; if (!target) { throw new Error( `Channel "${slug}" is not relocated — its media is already in place`, ); } if (existingMarker && existingMarker.target !== target) { throw new Error( `A relocation to ${existingMarker.target} is already in progress for "${slug}"`, ); } // A marker for the OTHER direction is never resumed into this one. Same // target, opposite intent: an interrupted move-out has a full copy on the // platter and a real dir here, and treating that as a move-back to resume // would swap the wrong way round. if (existingMarker && !resume) { throw new Error( `A move-out to ${existingMarker.target} is in flight or was interrupted ` + `for "${slug}" — finish that before moving back`, ); } return moveBack({ paths, slug, io, config, channelDir, mediaLink, target, log, onProgress, signal, resumed: Boolean(resume), phase: resume?.phase ?? "copy", writers, reconcile, }); } const root = opts.root?.trim() ?? ""; // Blank, relative, and inside-the-corpus, in one list — see // relocationRootProblem. The action and the preview ask the same question // earlier so the operator does not find out from a job log, but this is the // one that is load-bearing. const rootProblem = await relocationRootProblem({ paths, slug, root, storage: io.read().storage, }); if (rootProblem) throw new Error(rootProblem); const target = relocatedMediaDir(root, slug); if (existingMarker && existingMarker.target !== target) { throw new Error( `A relocation to ${existingMarker.target} is already in progress for "${slug}" — ` + `finish or clear it before moving to ${target}`, ); } if (config.mediaDir?.trim() && config.mediaDir.trim() !== target) { throw new Error( `Channel "${slug}" is already relocated to ${config.mediaDir.trim()}. ` + `Move it back in place first.`, ); } const resume = existingMarker?.direction === "out" ? existingMarker : null; if (existingMarker && !resume) { throw new Error( `A move-back from ${existingMarker.target} is in flight or was interrupted ` + `for "${slug}" — finish that before moving out again`, ); } return moveOut({ paths, slug, io, config, channelDir, mediaLink, root, target, log, onProgress, signal, // `resumed` relaxes the "must be in-place" precondition, so it must mean // "this run continues an interrupted move OUT" and nothing looser. resumed: Boolean(resume), phase: resume?.phase ?? "copy", writers, reconcile, }); } // THE THIRD ASK, right after the copy phase's marker is written. From that // write on, the lanes skip the channel (their next inspect reads the marker) // and a media job refuses to start on it, so a writer seen now was already // running when the marker landed — one that started between the first step and // here. The refusal puts the channel back as it was: a marker this run created // is removed (nothing has been copied under it), one it resumed is left. async function assertNoWritersUnderMarker(args: { paths: Paths; slug: string; resumed: boolean; writers: (slug: string) => ChannelWriter[]; }): Promise { const refusal = channelWritersRefusal(args.slug, args.writers(args.slug)); if (!refusal) return; if (!args.resumed) await clearMarker(args.paths, args.slug); throw new Error(refusal); } async function moveOut(args: { paths: Paths; slug: string; io: { read: () => SiteSettings }; config: NonNullable>>; channelDir: string; mediaLink: string; root: string; target: string; log: (m: string) => void; onProgress?: (p: RelocationProgress) => void; signal?: AbortSignal; resumed: boolean; phase: RelocationPhase; writers: (slug: string) => ChannelWriter[]; reconcile: boolean; }): Promise { const { paths, slug, channelDir, mediaLink, root, target, log, signal } = args; const dataDir = path.join(channelDir, "data"); // PREFLIGHT. Everything that can refuse does so here, before a single byte is // written and before the marker exists. if (!(await isDirectory(root))) { throw new Error( `The destination root ${root} does not exist or is not a directory ` + `(is the drive mounted?)`, ); } // Writability, checked explicitly rather than discovered by rsync's exit code // three minutes in. A read-only mount is the ordinary way a platter comes // back after a bad shutdown. try { await access(root, fsConstants.W_OK); } catch { throw new Error(`The destination root ${root} is not writable`); } // A RESUMING RUN MUST TOLERATE ITS OWN MARKER. `inspectChannelMedia` reports // "in-transition" for any channel carrying one, which is exactly right for // every guard and exactly wrong here: the rerun that finishes an interrupted // move is the one caller allowed to see it. A marker for a DIFFERENT target // was already refused above. const location = await inspectChannelMedia(paths, slug, args.config, { fresh: true, }); if (!args.resumed && location.status !== "in-place") { throw new Error( `Channel "${slug}" is not in a movable state: ${ location.detail ?? location.status }`, ); } // A channel that has downloaded nothing has no data/ at all. Say so, rather // than invent a `media/` for it and move nothing. if (args.phase === "copy" && !(await isDirectory(dataDir))) { throw new Error( `Channel "${slug}" has no ${dataDir} to move — nothing has been ` + `downloaded for it yet`, ); } // TIER FIRST (release 17): after the writers check (the job's first step, // above) and before anything is measured. A classic channel's big files are // renamed into a new real `media/` and linked — the same filesystem, so a // rename each — and a channel tiered already counts them as "already". A // resumed run tiers nothing: its marker stands, and the hook writes nothing // under one (a file a writer finished since stays real, on the corpus disk, // until the next sweep tiers it onto the far side). let tiered = 0; if (args.phase === "copy") { // THE DESTINATION'S IDENTITY FIRST (review N5): a bare mountpoint, or a // drive whose learned volume does not match, is refused before the // channel is touched at all — the tiering below is harmless, but "refused // before anything" should be true here too. Asked again immediately before // the mkdir below, for a job that sat in the queue. await assertRelocationRootPresent(root, args.io.read().storage, paths); const counts = await tierChannelMedia(paths, slug, { createMediaDir: true, onLog: (line) => log(line.replace(/\n$/, "")), }); tiered = counts.tiered; if (tiered > 0) log(`Tiered ${tiered} file(s) into ${mediaLink} first`); if (!(await isDirectory(mediaLink))) { throw new Error( `Channel "${slug}" has no ${mediaLink} to move — it could not be ` + `made a directory`, ); } } const measured = await measureTree(mediaLink); // Sticky across both verify points below: a retry at either one is the fact // the caller wants reported, and neither overwrites the other's answer. let verifyRetried = false; let reconciled = false; log( `Relocating ${slug}: ${measured.files} file(s), ${formatBytes(measured.bytes)} ` + `-> ${target}`, ); let phase = args.phase; if (phase === "copy") { // THE BAR IS THE BYTES PLUS THE RESUME MARGIN, not the bytes. Landing the // media with nothing to spare puts the destination volume under the disk // gate's own floor the moment it arrives, so the channel's next download is // refused by the gate on the drive it was just moved to. The margin is the // operator's configured one, so the two numbers cannot drift apart — and it // is ZERO when the gate is switched off, because the margin exists to clear // a bar that then does not exist. Demanding headroom for a gate nobody // armed would refuse a move on a disk with room for it. const settings = args.io.read(); const marginGB = settings.minFreeDiskGB > 0 ? settings.resumeMarginGB : 0; const needed = measured.bytes + marginGB * BYTES_PER_GB; const free = await getFreeBytes(root); if (free < needed) { throw new Error( `Not enough space on ${root}: ${formatBytes(free)} free, ` + `${formatBytes(measured.bytes)} to move` + (marginGB > 0 ? ` plus a ${marginGB} GB resume margin = ${formatBytes(needed)} required` : " required"), ); } // THE LAST THING BEFORE THE MKDIR THAT WOULD INVENT THE MOUNTPOINT. // Re-asked here and not only at enqueue time: a job can sit in the queue // for hours behind other work, and the drive that was mounted when the // operator clicked may not be mounted when the copy starts. await assertRelocationRootPresent(root, settings.storage, paths); await mkdir(target, { recursive: true }); // WHAT A PREVIOUS ATTEMPT ALREADY LANDED, so the bar is about the TREE and // not about this process's share of it. rsync counts only what it sends: a // copy resumed at 60 % would otherwise climb 0 → 40 % and stop, having // moved every remaining byte. One walk of the target, which for a fresh // move is an empty directory and costs a readdir. const already = (await measureTree(target)).bytes; await writeMarker(paths, slug, { target, direction: "out", startedAt: new Date().toISOString(), phase: "copy", scope: "media", }); await assertNoWritersUnderMarker({ paths, slug, resumed: args.resumed, writers: args.writers, }); // Copy (--partial keeps an aborted transfer resumable; -a preserves mtimes, // which is what makes the LMDB index a no-op afterwards), mirror toward // the target, verify — relocateDir.ts's copyMirrorVerify, the same for a // fresh move, a resume and a reconcile. verifyRetried ||= ( await copyMirrorVerify({ rsyncBin: paths.rsyncBin, src: mediaLink, dest: target, // The channel's media, which --delete may never reach. live: mediaLink, log, progress: makeProgressSink({ totalBytes: measured.bytes, alreadyBytes: already, log, onProgress: args.onProgress, }), signal, reconcile: args.reconcile, cancelled: () => new Error( `Cancelled. ${mediaLink} is untouched and the partial copy at ${target} ` + `is resumable — rerun to continue.`, ), }) ).retried; reconciled = args.reconcile; phase = "swap"; } if (phase === "swap") { await writeMarker(paths, slug, { target, direction: "out", startedAt: new Date().toISOString(), phase: "swap", scope: "media", }); // EVERY STEP BELOW OBSERVES THE DISK INSTEAD OF ASSUMING THE LAST ONE RAN. // The marker says how far the previous attempt got; it cannot say how far // it got THROUGH a phase, and a crash lands between any two syscalls. const state = await linkOrDirState(mediaLink); const parked = (await parkedSiblings(channelDir)).filter((p) => path.basename(p).startsWith("media.relocated-"), ); // Re-verify, because a resumed run did not do the copy in this process and // must not take the interrupted one's word for it. The source to verify // against is whichever copy of it still exists; once the swap has committed // there is none, and there is nothing left to check. // // MIRRORED ONLY FROM THE LIVE MEDIA. While `media/` is still the real // directory it is the source, and a difference gets the mirror pass a copy // phase would give it. A parked `media.relocated-*` is not live any more — // the link already points at the target — so the target is never mirrored // FROM it: that verify is the strict one, as it always was. const verifySrc = state.kind === "real-dir" ? mediaLink : (parked[0] ?? null); if (verifySrc) { log("Verifying the copy…"); verifyRetried ||= ( await verifyCopy({ rsyncBin: paths.rsyncBin, src: verifySrc, dest: target, log, signal, ...(verifySrc === mediaLink ? { mirror: { live: mediaLink } } : {}), }) ).retried; } if (state.kind === "real-dir") { // Same device, so the rename is atomic: `media/` is a real dir one // instant and the parked copy the next, never half of each. A fresh name // even when a parked dir already exists — renaming onto a non-empty // directory is ENOTEMPTY, and the reclaim sweep takes all of them anyway. // Between this rename and the symlink below every per-file link in // `data//` dangles for one syscall; the marker stands, so every media // guard reads the channel as in transition and nothing opens one. await rename(mediaLink, path.join(channelDir, `media.relocated-${Date.now()}`)); } else if (state.kind === "other") { throw new Error( `${mediaLink} is neither a directory nor a symlink — refusing to replace it`, ); } const after = await linkOrDirState(mediaLink); if ( after.kind === "link" && path.resolve(after.linkTarget) !== path.resolve(target) ) { // A link to somewhere else is not this move's work to reinterpret, and // silently repointing it would strand whatever it does point at. throw new Error( `${mediaLink} already points at ${after.linkTarget}, not ${target}`, ); } if (after.kind === "missing") { await symlink(target, mediaLink); } // Written only now, on success: config.mediaDir is a record of what is on // disk, never an intention. Skipped when it already says so, so a rerun // does not rewrite a file it agrees with. // No readable config.json (it vanished mid-move, before the read or // between the read and the patch): the job's own copy is the best record // there is, and the swap has already happened — so write that, never // nothing. const fresh = await readChannelConfig(paths, slug); const patched = fresh && fresh.mediaDir?.trim() !== target ? await patchChannelConfig(paths, slug, { mediaDir: target }) : fresh; if (!patched) { await writeChannelConfig(paths, slug, { ...args.config, mediaDir: target }); } log(`Swapped: ${mediaLink} -> ${target}`); await writeMarker(paths, slug, { target, direction: "out", startedAt: new Date().toISOString(), phase: "reclaim", scope: "media", }); } // RECLAIM RUNS ON EVERY PATH, not only on a resume. A crash between the // config write and the reclaim marker used to orphan a full second copy of // the channel on the source volume — the disk the move exists to free — and // the sweep that would have caught it was in the branch a fresh run never // takes. It sweeps every sibling, not the one name this process minted. await sweepParked(channelDir, log); await clearMarker(paths, slug); const freeNow = await getFreeBytes(paths.channelsDir); log( `Done. ${formatBytes(measured.bytes)} now on ${root}; ` + `${formatBytes(freeNow)} free on the source volume.`, ); return { slug, direction: "out", target, bytes: measured.bytes, files: measured.files, tiered, resumed: args.resumed, retried: verifyRetried, reconciled, }; } async function moveBack(args: { paths: Paths; slug: string; io: { read: () => SiteSettings }; config: NonNullable>>; channelDir: string; mediaLink: string; target: string; log: (m: string) => void; onProgress?: (p: RelocationProgress) => void; signal?: AbortSignal; resumed: boolean; phase: RelocationPhase; writers: (slug: string) => ChannelWriter[]; reconcile: boolean; }): Promise { const { paths, slug, channelDir, mediaLink, target, log, signal } = args; const incoming = path.join(channelDir, "media.incoming"); // THE MIRROR OF moveOut's "must be in-place", and it is not symmetry for its // own sake: move back is the only direction that ENDS by deleting the target. // // `relocated` is true for `inconsistent` and `unreachable` as well as `ok` — // config.mediaDir is set in all three — so without this the UI offers Move // back for a channel whose config records a target while `media/` is a REAL // directory. That state is not hypothetical: it is what `rsync --copy-links` // of a channel produces, which WORKTREES.md documents as the way to carry // media into a shard. The run then copies the target to `media.incoming`, // verifies it, finds `media/` already a real dir, logs "the swap had // completed", clears the config, `rm -r`s the target and finally sweeps // `media.incoming` — three copies in, zero out. // // `in-transition` is allowed because a marker is what a resume carries, and a // rerun is the caller this precondition must not refuse. const location = await inspectChannelMedia(paths, slug, args.config, { fresh: true, }); if ( !args.resumed && location.status !== "ok" && location.status !== "in-transition" ) { throw new Error( `Channel "${slug}" is not in a movable state: ${ location.detail ?? location.status }. Nothing has been touched.`, ); } // THE PRECONDITION BELONGS TO THE COPY, NOT TO THE RERUN. Past the swap the // media is already back in the channel dir and the target may well be gone — // demanding it be reachable there would refuse the very run that finishes // cleaning up after an interrupted move. if (args.phase === "copy" && !(await isDirectory(target))) { throw new Error( `The relocated media at ${target} is not reachable (is the drive mounted?)`, ); } // Measured off whichever copy still exists, in the order they stop existing. let verifyRetried = false; let reconciled = false; const measured = (await isDirectory(target)) ? await measureTree(target) : (await isDirectory(incoming)) ? await measureTree(incoming) : await measureTree(mediaLink); log( `Moving ${slug} back in place: ${measured.files} file(s), ` + `${formatBytes(measured.bytes)} <- ${target}`, ); let phase = args.phase; if (phase === "copy") { // THE SAME BAR MOVE-OUT USES, and for the same reason: landing the media // with nothing to spare puts the corpus volume under the disk gate's floor // the moment it arrives. And a resumed move-back must only be charged for // what is still MISSING — the bytes already sitting in `media.incoming` are // not about to be written twice, and counting them refused reruns on a disk // that had room for the remainder. const settings = args.io.read(); const marginGB = settings.minFreeDiskGB > 0 ? settings.resumeMarginGB : 0; const already = (await isDirectory(incoming)) ? (await measureTree(incoming)).bytes : 0; const needed = Math.max(0, measured.bytes - already) + marginGB * BYTES_PER_GB; const free = await getFreeBytes(paths.channelsDir); if (free < needed) { throw new Error( `Not enough space on the corpus volume: ${formatBytes(free)} free, ` + `${formatBytes(Math.max(0, measured.bytes - already))} still to move back` + (marginGB > 0 ? ` plus a ${marginGB} GB resume margin = ${formatBytes(needed)} required` : " required"), ); } // THE GUARD GOES ON THE LOCATION ROOT, NOT ON `incoming`. `incoming` is // `channels//media.incoming`, a corpus directory a move-back is // entitled to create. What must be present is the SOURCE side: the // `isDirectory(target)` precondition above covers existence, and this // covers IDENTITY — an empty mountpoint directory with the platter // unplugged is a directory, and copying it back would report a successful // move of zero bytes and then delete the target. await assertRelocationRootPresent( rootOfRelocatedMediaDir(target, slug), settings.storage, paths, ); await mkdir(incoming, { recursive: true }); await writeMarker(paths, slug, { target, direction: "back", startedAt: new Date().toISOString(), phase: "copy", scope: "media", }); await assertNoWritersUnderMarker({ paths, slug, resumed: args.resumed, writers: args.writers, }); // The copy under construction is `media.incoming`; the target on the other // drive is the source, and is never the target of the mirror's --delete. // `live` is `media` — the link, which resolves to that target — so a call // with source and destination swapped is refused before rsync runs. verifyRetried ||= ( await copyMirrorVerify({ rsyncBin: paths.rsyncBin, src: target, dest: incoming, live: mediaLink, log, progress: makeProgressSink({ totalBytes: measured.bytes, // The same figure the space check above is priced in — what is // already in `media.incoming` from an interrupted run. alreadyBytes: already, log, onProgress: args.onProgress, }), signal, reconcile: args.reconcile, cancelled: () => new Error( `Cancelled. ${target} is untouched and ${incoming} is resumable — ` + `rerun to continue.`, ), }) ).retried; reconciled = args.reconcile; phase = "swap"; } if (phase === "swap") { await writeMarker(paths, slug, { target, direction: "back", startedAt: new Date().toISOString(), phase: "swap", scope: "media", }); // OBSERVE, DO NOT ASSUME. The old sequence was `unlink(data)` (swallowing // its error) then `rename(incoming, data)`, which is idempotent in exactly // the case that never happens: a crash AFTER the rename left `data/` a real // directory, the swallowed unlink then failed on it, and the rename ENOENTed // on an `incoming` that no longer existed — forever, on every rerun. The // same holds for `media` since release 17. const state = await linkOrDirState(mediaLink); if (state.kind === "real-dir") { // The rename already committed. Nothing to swap; the leftovers are the // reclaim's business. log(`${mediaLink} is already a real directory — the swap had completed`); } else if (state.kind === "other") { // A regular file (or a socket, or a fifo) where `media/` should be is not // a link to replace and not a directory to keep. moveOut refuses the same // shape at its own swap; refusing here too is what keeps `unlink` below // meaning "remove the symlink" and nothing else. throw new Error( `${mediaLink} is neither a directory nor a symlink — refusing to replace it`, ); } else { if (!(await isDirectory(incoming))) { throw new Error( `Cannot finish moving "${slug}" back: ${mediaLink} is not a directory ` + `and there is no verified copy at ${incoming}`, ); } // unlink, not rm -r: `media` is the LINK here, and removing it // recursively would be the one way this whole design eats the media. // Every per-file link in `data//` is relative to `media`, so the // rename below makes each of them resolve on the corpus disk — not one // of them is rewritten (no "untier"). if (state.kind !== "missing") await unlink(mediaLink); await rename(incoming, mediaLink); log(`Swapped: ${mediaLink} is a real directory again`); } const fresh = await readChannelConfig(paths, slug); if (fresh?.mediaDir !== undefined) { await patchChannelConfig(paths, slug, {}, { unset: ["mediaDir"] }); } await writeMarker(paths, slug, { target, direction: "back", startedAt: new Date().toISOString(), phase: "reclaim", scope: "media", }); } await rm(target, { recursive: true, force: true }); // Leave / behind only if something else is in it. const slugRoot = path.dirname(target); if ((await readdir(slugRoot).catch(() => ["keep"])).length === 0) { await rm(slugRoot, { recursive: true, force: true }); } // Any half-copied `media.incoming` (or a parked dir from an earlier move out) // goes with it — the same sweep, for the same reason. await sweepParked(channelDir, log); await clearMarker(paths, slug); log(`Done. ${formatBytes(measured.bytes)} back in place.`); return { slug, direction: "back", target, bytes: measured.bytes, files: measured.files, tiered: 0, resumed: args.resumed, retried: verifyRetried, reconciled, }; }