commit 519cfcedac86a3fcc61af4fd4f8e5e0a7810d948
parent de5b77d53c3e70b344864cb33ffbe84e01256605
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 2 Oct 2026 00:39:27 -0400
common: the mover over media/ — a move carries channels/<slug>/media, never data/
The mover tiers a classic channel first (tierChannelMedia, createMediaDir),
copies media/ to <root>/<slug>/media, links channels/<slug>/media and records
config.mediaDir; the marker carries scope "media"; parked names are
media.relocated-<ts> / media.incoming; a move back leaves media/ a real
directory on the corpus disk and every per-file link untouched. A legacy
channel and a tier-migration marker are refused naming migrate-tier. The
preview tiers an in-place classic channel and reports how many files.
The re-point, the rename, deleteChannel, the storage watch and the location
rollups key on mediaDir (a legacy channel by its retired dataDir); /storage's
location rows hold the media tier, the internal row every channel's text and
clips plus the in-place media, and a legacy channel counts unreachable with
"(n to migrate)". The 28 tests T1 skipped are rebased and run.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
25 files changed, 1233 insertions(+), 600 deletions(-)
diff --git a/SETTINGS.md b/SETTINGS.md
@@ -547,7 +547,7 @@ Where a channel's downloaded media goes when it is relocated off the corpus disk
|---|---|---|
| `locations` | `[]` | The named storage locations a channel's media may be relocated to — one entry per root, each with an id, label, root, `autoRepoint` and the learned volume identity. Order is display order. Managed on /storage. |
| `defaultLocationId` | `""` | The location prefilled as the destination of a move. "" = no default. |
-| `savedVideosLocationId` | absent | WHERE THE SAVED-VIDEO STORE IS, by location id. "" = in place, under the corpus at `paths.savedVideosDir`.<br><br>A RECORD OF WHAT IS ON DISK, never an intention — the same contract as a channel's `config.dataDir`. It is written by the move, on success, after the copy has verified and the symlink is in place; nothing else writes it, and a reader that disagrees with the disk trusts the disk. Optional so an older settings.json parses (and an older binary that drops it leaves a store that still works, because the symlink is what every reader follows). |
+| `savedVideosLocationId` | absent | WHERE THE SAVED-VIDEO STORE IS, by location id. "" = in place, under the corpus at `paths.savedVideosDir`.<br><br>A RECORD OF WHAT IS ON DISK, never an intention — the same contract as a channel's `config.mediaDir`. It is written by the move, on success, after the copy has verified and the symlink is in place; nothing else writes it, and a reader that disagrees with the disk trusts the disk. Optional so an older settings.json parses (and an older binary that drops it leaves a store that still works, because the symlink is what every reader follows). |
| `health` | absent | THE DRIVE-HEALTH TIMINGS: how long a read may take before a drive counts as not answering, how often the health pass looks, how long its look may take, how many clean looks clear a stall, and how many reads may be on one drive at once. Edited on /storage (Drive health timing). Absent = every default, and only a value that differs from its default is written, so an untuned install follows a default changed later. See `storage.health` below. |
#### `storage.locations[]`
diff --git a/common/controller/channels.test.ts b/common/controller/channels.test.ts
@@ -17,6 +17,7 @@ import {
relocatedDataDir,
RELOCATION_MARKER_FILENAME,
} from "../lib/channelMedia";
+import { relocatedMediaDir } from "../lib/mediaTier-server";
import {
channelConfigPath,
channelExists,
@@ -70,11 +71,11 @@ test("deleting an in-place channel removes its directory", async () => {
test("deleting a relocated channel reclaims the media on the other drive", async () => {
await withPaths(async (paths, mediaRoot) => {
- const target = relocatedDataDir(mediaRoot, "alpha");
- await writeChannelConfig(paths, "alpha", { ...config, dataDir: target });
+ const target = relocatedMediaDir(mediaRoot, "alpha");
+ await writeChannelConfig(paths, "alpha", { ...config, mediaDir: target });
await mkdir(path.join(target, "v1"), { recursive: true });
await writeFile(path.join(target, "v1", "audio.m4a"), "BYTES");
- await symlink(target, path.join(paths.channelsDir, "alpha", "data"));
+ await symlink(target, path.join(paths.channelsDir, "alpha", "media"));
await deleteChannel(paths, "alpha");
@@ -89,11 +90,11 @@ test("deleting a relocated channel reclaims the media on the other drive", async
test("a non-empty <root>/<slug> survives the delete", async () => {
await withPaths(async (paths, mediaRoot) => {
- const target = relocatedDataDir(mediaRoot, "alpha");
- await writeChannelConfig(paths, "alpha", { ...config, dataDir: target });
+ const target = relocatedMediaDir(mediaRoot, "alpha");
+ await writeChannelConfig(paths, "alpha", { ...config, mediaDir: target });
await mkdir(target, { recursive: true });
- await symlink(target, path.join(paths.channelsDir, "alpha", "data"));
- // Something of the operator's, alongside the data dir we own.
+ await symlink(target, path.join(paths.channelsDir, "alpha", "media"));
+ // Something of the operator's, alongside the media dir we own.
await writeFile(path.join(mediaRoot, "alpha", "NOTES.txt"), "mine");
await deleteChannel(paths, "alpha");
@@ -102,9 +103,26 @@ test("a non-empty <root>/<slug> survives the delete", async () => {
});
});
-test("deleting is refused while a relocation is in flight", async () => {
+// THE RETIRED LAYOUT (release 17): a channel never migrated off the whole-
+// directory move still carries `dataDir`; deleting it reclaims that tree too.
+test("deleting a legacy channel reclaims its retired data dir", async () => {
await withPaths(async (paths, mediaRoot) => {
const target = relocatedDataDir(mediaRoot, "alpha");
+ await writeChannelConfig(paths, "alpha", { ...config, dataDir: target });
+ await mkdir(path.join(target, "v1"), { recursive: true });
+ await writeFile(path.join(target, "v1", "transcript.json"), "{}");
+ await symlink(target, path.join(paths.channelsDir, "alpha", "data"));
+
+ await deleteChannel(paths, "alpha");
+ assert.equal(await channelExists(paths, "alpha"), false);
+ assert.equal(await there(target), false);
+ assert.equal(await there(path.join(mediaRoot, "alpha")), false);
+ });
+});
+
+test("deleting is refused while a relocation is in flight", async () => {
+ await withPaths(async (paths, mediaRoot) => {
+ const target = relocatedMediaDir(mediaRoot, "alpha");
await writeChannelConfig(paths, "alpha", config);
await mkdir(path.join(paths.channelsDir, "alpha", "data"), {
recursive: true,
@@ -131,9 +149,9 @@ test("an unmounted target does not make the channel undeletable", async () => {
// `force: true` makes a missing target a no-op, so the channel goes and its
// media stays on the unmounted platter. Deliberate: refusing here would
// make a channel whose drive is gone for good impossible to remove.
- const target = relocatedDataDir(mediaRoot, "alpha");
- await writeChannelConfig(paths, "alpha", { ...config, dataDir: target });
- await symlink(target, path.join(paths.channelsDir, "alpha", "data"));
+ const target = relocatedMediaDir(mediaRoot, "alpha");
+ await writeChannelConfig(paths, "alpha", { ...config, mediaDir: target });
+ await symlink(target, path.join(paths.channelsDir, "alpha", "media"));
await deleteChannel(paths, "alpha");
assert.equal(await channelExists(paths, "alpha"), false);
});
diff --git a/common/controller/channels.ts b/common/controller/channels.ts
@@ -591,10 +591,15 @@ export async function deleteChannel(
);
}
const dir = path.join(paths.channelsDir, slug);
- const target = (await readChannelConfig(paths, slug))?.dataDir?.trim();
- if (target) {
+ const config = await readChannelConfig(paths, slug);
+ // The media tier's target (`mediaDir`, release 17) and, on a channel never
+ // migrated off the retired whole-directory layout, its `dataDir` — each on
+ // another drive, each removed before the channel dir. An in-place `media/`
+ // is a real directory inside `dir` and goes with it.
+ for (const target of [config?.mediaDir?.trim(), config?.dataDir?.trim()]) {
+ if (!target) continue;
await rm(target, { recursive: true, force: true });
- // <root>/<slug> is ours by construction (relocatedDataDir fixes the
+ // <root>/<slug> is ours by construction (relocatedMediaDir fixes the
// suffix), so take it too — but only when nothing else landed in it.
const slugRoot = path.dirname(target);
if (path.basename(slugRoot) === slug) {
diff --git a/common/controller/relocateChannelMedia.test.ts b/common/controller/relocateChannelMedia.test.ts
@@ -23,8 +23,12 @@ import type { SiteSettings } from "../lib/settings";
import {
inspectChannelMedia,
readRelocationMarker,
- relocatedDataDir,
} from "../lib/channelMedia";
+import {
+ relocatedMediaDir,
+ tierChannelMedia,
+ tierLinkTarget,
+} from "../lib/mediaTier-server";
import type { ChannelWriter } from "./channelWriters";
import {
assertRelocationRootPresent,
@@ -42,16 +46,17 @@ import {
} from "../lib/storageVolumes";
import { readChannelConfig } from "./channels";
-// RELEASE 17 SLICE T1 made a channel whose `data/` is a link (or whose config
-// carries `dataDir`) `legacy`; these cases still build that retired layout and
-// expect it to read `ok`. Slice T2 rebases them on `media/` and un-skips them.
-const T1_SKIP = "release 17 T2 rebases the mover on media/";
-
// Run with:
// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/relocateChannelMedia.test.ts
//
// These exercise the REAL rsync binary (paths.rsyncBin -> "rsync"), like
// controller/backupSavedVideos.test.ts. Everything happens inside one mkdtemp.
+//
+// RELEASE 17: the unit of a move is `channels/<slug>/media` (the media tier),
+// never `data/`. A seeded channel is CLASSIC — its big files real in
+// `data/<id>/` — and the move's preflight tiers it; the crash cases build the
+// tiered state first (`tierInPlace`) and then the exact on-disk state of the
+// crash.
const MTIME = new Date("2021-03-04T05:06:07.000Z");
@@ -132,24 +137,36 @@ async function seed(
const file = path.join(videoDir, name);
await writeFile(file, content);
// A known mtime, so "rsync -a preserved it" is an assertion and not a
- // coincidence. The LMDB index keys on mtimes, so this is what makes a
- // relocation free of a reindex.
+ // coincidence.
await utimes(file, MTIME, MTIME);
}
}
return channelDir;
}
-test("out: copies, links, records the target, keeps mtimes and reclaims the source", { skip: T1_SKIP }, async () => {
+// A classic channel tiered in place: `media/` a real directory on the corpus
+// disk, every big file in it and a relative link left in `data/<id>/` — what a
+// move's preflight does first, and the state every crash case starts from.
+async function tierInPlace(paths: Paths, slug: string): Promise<string> {
+ await tierChannelMedia(paths, slug, { createMediaDir: true });
+ return path.join(paths.channelsDir, slug, "media");
+}
+
+test("out: copies, links, records the target, keeps mtimes and reclaims the source", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
- v1: { "audio.m4a": "one".repeat(500), "transcript.json": "{}" },
- v2: { "audio.m4a": "two".repeat(500) },
+ v1: {
+ "audio.m4a": "one".repeat(500),
+ "transcript.json": "{}",
+ "transcript.live_chat.json": "[]",
+ },
+ v2: { "audio.m4a": "two".repeat(500), "source-media.mp4": "container" },
});
- const target = relocatedDataDir(root, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ const mediaLink = path.join(channelDir, "media");
const res = await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -158,36 +175,50 @@ test("out: copies, links, records the target, keeps mtimes and reclaims the sour
});
assert.equal(res.direction, "out");
assert.equal(res.target, target);
+ // The two audio files and the raw live chat — the media tier. The text
+ // and the persisted container are not in it.
assert.equal(res.files, 3);
+ assert.equal(res.tiered, 3, "the classic channel was tiered first");
assert.equal(res.resumed, false);
- // data/ is an absolute symlink at the target...
- const link = await lstat(path.join(channelDir, "data"));
- assert.ok(link.isSymbolicLink());
- assert.equal(await readlink(path.join(channelDir, "data")), target);
-
- // ...so the on-disk contract channelDir/data/<id>/<file> still resolves.
- assert.equal(
- await readFile(
- path.join(channelDir, "data", "v1", "transcript.json"),
- "utf8",
- ),
- "{}",
+ // `media` is ONE absolute symlink at the target...
+ assert.ok((await lstat(mediaLink)).isSymbolicLink());
+ assert.equal(await readlink(mediaLink), target);
+ // ...`data/` stays a real directory on the corpus disk...
+ assert.ok((await lstat(path.join(channelDir, "data"))).isDirectory());
+ assert.ok(!(await lstat(path.join(channelDir, "data"))).isSymbolicLink());
+ // ...each big file is a RELATIVE link into `media/` that resolves through it.
+ const audio = path.join(channelDir, "data", "v1", "audio.m4a");
+ assert.ok((await lstat(audio)).isSymbolicLink());
+ assert.equal(await readlink(audio), tierLinkTarget("v1", "audio.m4a"));
+ assert.equal(await readFile(audio, "utf8"), "one".repeat(500));
+ assert.ok(
+ (await lstat(
+ path.join(channelDir, "data", "v1", "transcript.live_chat.json"),
+ )).isSymbolicLink(),
);
+ // ...and the text, and what is not tierable, never left.
+ for (const real of [["v1", "transcript.json"], ["v2", "source-media.mp4"]]) {
+ const st = await lstat(path.join(channelDir, "data", ...real));
+ assert.ok(st.isFile() && !st.isSymbolicLink(), real.join("/"));
+ }
+ await assert.rejects(() => stat(path.join(target, "v1", "transcript.json")));
assert.equal(
(await stat(path.join(target, "v1", "audio.m4a"))).mtime.getTime(),
MTIME.getTime(),
);
// The record is in config.json, written only now, on success.
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, target);
+ const config = await readChannelConfig(paths, "alpha");
+ assert.equal(config?.mediaDir, target);
+ assert.equal(config?.dataDir, undefined);
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
// The parked copy is gone and the marker with it — this is the step that
// actually frees the source volume.
const left = await readdir(channelDir);
assert.deepEqual(
- left.filter((n) => n.startsWith("data.")),
+ left.filter((n) => n.startsWith("media.") || n.startsWith("data.")),
[],
);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
@@ -197,12 +228,30 @@ test("out: copies, links, records the target, keeps mtimes and reclaims the sour
});
});
-test("abort from an onLog hook leaves the source intact, and the rerun completes", { skip: T1_SKIP }, async () => {
+test("a channel already tiered in place moves without tiering anything", async () => {
+ await withTmp(async (paths, root) => {
+ await seed(paths, "alpha", { v1: { "audio.m4a": "one" } });
+ await tierInPlace(paths, "alpha");
+ const res = await relocateChannelMedia({
+ io: TEST_IO,
+ paths,
+ slug: "alpha",
+ direction: "out",
+ root,
+ onLog: () => {},
+ });
+ assert.equal(res.tiered, 0);
+ assert.equal(res.files, 1);
+ assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
+ });
+});
+
+test("abort from an onLog hook leaves the source intact, and the rerun completes", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
- v1: { "audio.m4a": "x".repeat(20000) },
+ v1: { "audio.m4a": "x".repeat(20000), "transcript.json": "{}" },
});
- const target = relocatedDataDir(root, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
// Aborting from the log hook is how a Cancel button reaches this code: the
// job's onLog and its AbortSignal belong to the same run. Firing on the
@@ -224,29 +273,35 @@ test("abort from an onLog hook leaves the source intact, and the rerun completes
/Cancelled/,
);
- // The source is a REAL directory still, with its file.
- assert.ok((await lstat(path.join(channelDir, "data"))).isDirectory());
+ // The source is a REAL `media/` still, with its file, reached through the
+ // link in `data/`.
+ assert.ok((await lstat(path.join(channelDir, "media"))).isDirectory());
+ assert.ok(!(await lstat(path.join(channelDir, "media"))).isSymbolicLink());
assert.equal(
(await readFile(path.join(channelDir, "data", "v1", "audio.m4a"), "utf8"))
.length,
20000,
);
- // Nothing was recorded, and the marker says where it was going.
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, undefined);
+ // Nothing was recorded, and the marker says where it was going — and that
+ // it moves the media only.
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, undefined);
const marker = await readRelocationMarker(paths, "alpha");
assert.equal(marker?.target, target);
assert.equal(marker?.direction, "out");
assert.equal(marker?.phase, "copy");
- // And every guard now reads the channel as in transition.
- assert.equal(
- (await inspectChannelMedia(paths, "alpha")).status,
- "in-transition",
- );
+ assert.equal(marker?.scope, "media");
+ // Every media guard now reads the channel as in transition; its text stays
+ // readable.
+ const during = await inspectChannelMedia(paths, "alpha", undefined, {
+ fresh: true,
+ });
+ assert.equal(during.status, "in-transition");
+ assert.equal(during.text.readable, true);
// The rerun finishes it. It is the ONE caller allowed to look past its own
// marker.
const res = await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -255,7 +310,7 @@ test("abort from an onLog hook leaves the source intact, and the rerun completes
});
assert.equal(res.resumed, true);
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, target);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, target);
});
});
@@ -269,7 +324,7 @@ test("a stray file on the destination is removed by the mirror pass", async () =
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one" },
});
- const target = relocatedDataDir(root, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
await mkdir(path.join(target, "v9"), { recursive: true });
await writeFile(path.join(target, "v9", "stray.m4a"), "not ours");
@@ -287,7 +342,7 @@ test("a stray file on the destination is removed by the mirror pass", async () =
await readFile(path.join(target, "v1", "audio.m4a"), "utf8"),
"one",
);
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, target);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, target);
assert.deepEqual(await leftoverCopies(channelDir), []);
});
});
@@ -315,7 +370,7 @@ test("a verify failure keeps the source and does not write the config", async ()
if (m.startsWith("$ ") && m.includes("--dry-run")) {
n++;
writeFileSync(
- path.join(channelDir, "data", "v1", `chunk-${n}.json`),
+ path.join(channelDir, "media", "v1", `chunk-${n}.json`),
"{}",
);
}
@@ -324,24 +379,26 @@ test("a verify failure keeps the source and does not write the config", async ()
/Verification failed: after a second mirror pass .*1 missing on the destination \(v1\/chunk-2\.json\)/,
);
- assert.ok((await lstat(path.join(channelDir, "data"))).isDirectory());
+ assert.ok((await lstat(path.join(channelDir, "media"))).isDirectory());
+ assert.ok(!(await lstat(path.join(channelDir, "media"))).isSymbolicLink());
assert.equal(
await readFile(path.join(channelDir, "data", "v1", "audio.m4a"), "utf8"),
"one",
);
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, undefined);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, undefined);
assert.equal((await readRelocationMarker(paths, "alpha"))?.phase, "copy");
});
});
-test("back: restores a real directory, clears the config and reclaims the target", { skip: T1_SKIP }, async () => {
+test("back: restores a real directory, clears the config and reclaims the target", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one", "transcript.json": "{}" },
});
- const target = relocatedDataDir(root, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ const mediaLink = path.join(channelDir, "media");
await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -350,36 +407,48 @@ test("back: restores a real directory, clears the config and reclaims the target
});
const res = await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "back",
onLog: () => {},
});
assert.equal(res.direction, "back");
- assert.equal(res.files, 2);
+ assert.equal(res.files, 1);
+ assert.equal(res.tiered, 0);
- const back = await lstat(path.join(channelDir, "data"));
+ // `media/` is a REAL directory on the corpus disk again, and the per-file
+ // link was never touched: it resolves there now (no "untier").
+ const back = await lstat(mediaLink);
assert.ok(back.isDirectory());
assert.ok(!back.isSymbolicLink());
- assert.equal(
- await readFile(path.join(channelDir, "data", "v1", "audio.m4a"), "utf8"),
- "one",
- );
- assert.equal(
- (await stat(path.join(channelDir, "data", "v1", "audio.m4a"))).mtime.getTime(),
- MTIME.getTime(),
- );
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, undefined);
+ const audio = path.join(channelDir, "data", "v1", "audio.m4a");
+ assert.ok((await lstat(audio)).isSymbolicLink());
+ assert.equal(await readlink(audio), tierLinkTarget("v1", "audio.m4a"));
+ assert.equal(await readFile(audio, "utf8"), "one");
+ assert.equal((await stat(audio)).mtime.getTime(), MTIME.getTime());
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, undefined);
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "in-place");
// The target is reclaimed, and so is <root>/<slug> once it is empty.
assert.equal(await pathThere(target), false);
assert.equal(await pathThere(path.dirname(target)), false);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
assert.equal(
- (await readdir(channelDir)).includes("data.incoming"),
+ (await readdir(channelDir)).includes("media.incoming"),
false,
);
+
+ // And the next move out has nothing to tier.
+ const again = await relocateChannelMedia({
+ io: TEST_IO,
+ paths,
+ slug: "alpha",
+ direction: "out",
+ root,
+ onLog: () => {},
+ });
+ assert.equal(again.tiered, 0);
+ assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
});
});
@@ -404,7 +473,7 @@ test("a relocated channel is refused a second move without a move back", async (
await withTmp(async (paths, root, dir) => {
await seed(paths, "alpha", { v1: { "audio.m4a": "one" } });
await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -445,6 +514,11 @@ test("an unmounted root is refused before anything is written", async () => {
);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
assert.ok((await lstat(path.join(channelDir, "data"))).isDirectory());
+ // Not even tiered: the root is refused before the preflight runs.
+ assert.equal(await pathIsThere(path.join(channelDir, "media")), false);
+ assert.ok(
+ (await lstat(path.join(channelDir, "data", "v1", "audio.m4a"))).isFile(),
+ );
});
});
@@ -466,22 +540,128 @@ test("a social channel has no media to relocate", async () => {
});
});
-test("preview measures the tree and both volumes without moving anything", async () => {
+test("preview tiers a classic channel first, measures media/ and both volumes, and moves nothing", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "abcde", "transcript.json": "{}" },
v2: { "audio.m4a": "fgh" },
});
const preview = await previewRelocation({ paths, slug: "alpha", root });
- assert.equal(preview.target, relocatedDataDir(root, "alpha"));
- assert.equal(preview.files, 3);
- assert.equal(preview.bytesToMove, 5 + 2 + 3);
+ assert.equal(preview.target, relocatedMediaDir(root, "alpha"));
+ // The media tier only: two audio files, not the transcript.
+ assert.equal(preview.files, 2);
+ assert.equal(preview.bytesToMove, 5 + 3);
+ assert.equal(preview.tieredFirst, 2);
assert.equal(preview.existingPartial, false);
assert.ok(preview.freeOnRoot > 0);
assert.ok(preview.freeOnSource > 0);
// Both dirs are under one tmpdir, so this really is the same volume.
assert.equal(preview.sameDevice, true);
+ // Tiered in place — `media/` a real directory, the files linked — and
+ // nothing moved off the corpus disk.
assert.ok((await lstat(path.join(channelDir, "data"))).isDirectory());
+ assert.ok(!(await lstat(path.join(channelDir, "media"))).isSymbolicLink());
+ assert.ok(
+ (await lstat(path.join(channelDir, "data", "v1", "audio.m4a"))).isSymbolicLink(),
+ );
+ assert.equal(await pathThere(path.join(root, "alpha")), false);
+ assert.equal(await readRelocationMarker(paths, "alpha"), null);
+ assert.equal(
+ (await inspectChannelMedia(paths, "alpha", undefined, { fresh: true })).status,
+ "in-place",
+ );
+
+ // Idempotent: a second preview tiers nothing and says the same figures.
+ const again = await previewRelocation({ paths, slug: "alpha", root });
+ assert.equal(again.tieredFirst, 0);
+ assert.equal(again.files, 2);
+ assert.equal(again.bytesToMove, 8);
+ });
+});
+
+// THE RETIRED LAYOUT (release 17): a channel whose whole `data/` was moved by
+// the old mover is `legacy`, and the mover refuses it — out, back and the
+// preview — with the sentence that names the migration. Nothing is touched.
+test("a legacy channel is refused by the job, the move back and the preview, naming migrate-tier", async () => {
+ await withTmp(async (paths, root, dir) => {
+ const channelDir = await seed(paths, "alpha", { v1: { "audio.m4a": "one" } });
+ const oldTarget = path.join(root, "alpha", "data");
+ await mkdir(path.dirname(oldTarget), { recursive: true });
+ await rename(path.join(channelDir, "data"), oldTarget);
+ await symlink(oldTarget, path.join(channelDir, "data"));
+ await setConfigField(paths, "alpha", "dataDir", oldTarget);
+ assert.equal(
+ (await inspectChannelMedia(paths, "alpha", undefined, { fresh: true })).status,
+ "legacy",
+ );
+ const other = path.join(dir, "platter2");
+ await mkdir(other, { recursive: true });
+
+ const refused = /cannot be moved: its media layout is the retired whole-directory one — run archilyzer storage migrate-tier alpha/;
+ await assert.rejects(
+ () =>
+ relocateChannelMedia({
+ io: TEST_IO,
+ paths,
+ slug: "alpha",
+ direction: "out",
+ root: other,
+ onLog: () => {},
+ }),
+ refused,
+ );
+ await assert.rejects(
+ () =>
+ relocateChannelMedia({
+ io: TEST_IO,
+ paths,
+ slug: "alpha",
+ direction: "back",
+ onLog: () => {},
+ }),
+ refused,
+ );
+ await assert.rejects(
+ () => previewRelocation({ paths, slug: "alpha", root: other }),
+ refused,
+ );
+ // Untouched: no marker, no `media`, the old link and its bytes as they were.
+ assert.equal(await readRelocationMarker(paths, "alpha"), null);
+ assert.equal(await pathIsThere(path.join(channelDir, "media")), false);
+ assert.equal(await readlink(path.join(channelDir, "data")), oldTarget);
+ assert.equal(
+ await readFile(path.join(oldTarget, "v1", "audio.m4a"), "utf8"),
+ "one",
+ );
+ assert.equal(await pathThere(path.join(other, "alpha")), false);
+ });
+});
+
+test("a tier migration's marker is never resumed or replaced by the mover", async () => {
+ await withTmp(async (paths, root) => {
+ await seed(paths, "alpha", { v1: { "audio.m4a": "one" } });
+ const target = relocatedMediaDir(root, "alpha");
+ await seedMarker(paths, "alpha", {
+ target,
+ direction: "out",
+ phase: "copy",
+ scope: "tier-migration",
+ });
+ for (const direction of ["out", "back"] as const) {
+ await assert.rejects(
+ () =>
+ relocateChannelMedia({
+ io: TEST_IO,
+ paths,
+ slug: "alpha",
+ direction,
+ root,
+ onLog: () => {},
+ }),
+ /media-tier migration in flight or interrupted .* archilyzer storage migrate-tier alpha/,
+ );
+ }
+ assert.equal((await readRelocationMarker(paths, "alpha"))?.scope, "tier-migration");
});
});
@@ -506,8 +686,8 @@ async function pathThere(p: string): Promise<boolean> {
//
// Each seeds the exact on-disk state of a crash at that point and asserts the
// rerun reaches a clean `ok` / `in-place` — with the config right, the marker
-// gone and NO `data.relocated-*` or `data.incoming` left holding a second copy
-// on the volume the move exists to free.
+// gone and NO `media.relocated-*` or `media.incoming` left holding a second
+// copy on the volume the move exists to free.
async function seedMarker(
paths: Paths,
@@ -516,6 +696,7 @@ async function seedMarker(
target: string;
direction: "out" | "back";
phase: "copy" | "swap" | "reclaim";
+ scope?: "media" | "tier-migration";
},
): Promise<void> {
await writeFile(
@@ -524,9 +705,10 @@ async function seedMarker(
);
}
-async function setConfigDataDir(
+async function setConfigField(
paths: Paths,
slug: string,
+ key: "mediaDir" | "dataDir",
value: string | null,
): Promise<void> {
const file = path.join(paths.channelsDir, slug, "config.json");
@@ -534,20 +716,28 @@ async function setConfigDataDir(
string,
unknown
>;
- if (value === null) delete config.dataDir;
- else config.dataDir = value;
+ if (value === null) delete config[key];
+ else config[key] = value;
await writeFile(file, JSON.stringify(config, null, 2) + "\n");
}
+async function setConfigMediaDir(
+ paths: Paths,
+ slug: string,
+ value: string | null,
+): Promise<void> {
+ await setConfigField(paths, slug, "mediaDir", value);
+}
+
async function leftoverCopies(channelDir: string): Promise<string[]> {
return (await readdir(channelDir))
- .filter((n) => n.startsWith("data.relocated-") || n === "data.incoming")
+ .filter((n) => n.startsWith("media.relocated-") || n === "media.incoming")
.sort();
}
// rsync -a of the tree, so the seeded "already copied" target is byte-for-byte
// what a completed copy phase would have left — including mtimes, which the
-// re-verify compares.
+// re-verify compares. Over `media/` only: it holds no links.
async function copyTree(src: string, dest: string): Promise<void> {
await mkdir(dest, { recursive: true });
for (const entry of await readdir(src, { withFileTypes: true })) {
@@ -562,18 +752,35 @@ async function copyTree(src: string, dest: string): Promise<void> {
}
}
-test("out @ swap: crash before the rename — the rerun re-verifies and completes", { skip: T1_SKIP }, async () => {
+// A COMPLETED MOVE OUT, built by hand: tiered, `media/` copied to the target
+// and replaced by the absolute link, `mediaDir` recorded.
+async function seedRelocated(
+ paths: Paths,
+ slug: string,
+ root: string,
+): Promise<{ mediaLink: string; target: string }> {
+ const mediaLink = await tierInPlace(paths, slug);
+ const target = relocatedMediaDir(root, slug);
+ await copyTree(mediaLink, target);
+ await rm(mediaLink, { recursive: true, force: true });
+ await symlink(target, mediaLink);
+ await setConfigMediaDir(paths, slug, target);
+ return { mediaLink, target };
+}
+
+test("out @ swap: crash before the rename — the rerun re-verifies and completes", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
});
- const target = relocatedDataDir(root, "alpha");
- // The copy finished; the process died before `data/` was parked.
- await copyTree(path.join(channelDir, "data"), target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ // The copy finished; the process died before `media/` was parked.
+ await copyTree(mediaLink, target);
await seedMarker(paths, "alpha", { target, direction: "out", phase: "swap" });
const res = await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -582,33 +789,33 @@ test("out @ swap: crash before the rename — the rerun re-verifies and complete
});
assert.equal(res.resumed, true);
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, target);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, target);
assert.deepEqual(await leftoverCopies(channelDir), []);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
});
});
-test("out @ swap: crash after the config write — the first run's parked copy is still reclaimed", { skip: T1_SKIP }, async () => {
+test("out @ swap: crash after the config write — the first run's parked copy is still reclaimed", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
- await copyTree(dataDir, target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
// Everything through writeChannelConfig ran; the reclaim marker never
- // landed. A full second copy of the channel is parked on the source volume
- // — the disk the whole move exists to free — and the marker still says
+ // landed. A full second copy of the media is parked on the source volume —
+ // the disk the whole move exists to free — and the marker still says
// "swap", so the old code minted a SECOND parked name and reclaimed only
// that one, orphaning this forever.
- const parked = path.join(channelDir, "data.relocated-1700000000000");
- await rename(dataDir, parked);
- await symlink(target, dataDir);
- await setConfigDataDir(paths, "alpha", target);
+ const parked = path.join(channelDir, "media.relocated-1700000000000");
+ await rename(mediaLink, parked);
+ await symlink(target, mediaLink);
+ await setConfigMediaDir(paths, "alpha", target);
await seedMarker(paths, "alpha", { target, direction: "out", phase: "swap" });
await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -618,25 +825,25 @@ test("out @ swap: crash after the config write — the first run's parked copy i
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
assert.deepEqual(await leftoverCopies(channelDir), []);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
- // The media itself is untouched by the sweep.
+ // The media itself is untouched by the sweep, and reached from `data/`.
assert.equal(
- await readFile(path.join(dataDir, "v1", "audio.m4a"), "utf8"),
+ await readFile(path.join(channelDir, "data", "v1", "audio.m4a"), "utf8"),
"one".repeat(500),
);
});
});
-test("out @ reclaim: the rerun sweeps every parked copy and clears the marker", { skip: T1_SKIP }, async () => {
+test("out @ reclaim: the rerun sweeps every parked copy and clears the marker", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
- await copyTree(dataDir, target);
- await rename(dataDir, path.join(channelDir, "data.relocated-1700000000000"));
- await symlink(target, dataDir);
- await setConfigDataDir(paths, "alpha", target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
+ await rename(mediaLink, path.join(channelDir, "media.relocated-1700000000000"));
+ await symlink(target, mediaLink);
+ await setConfigMediaDir(paths, "alpha", target);
await seedMarker(paths, "alpha", {
target,
direction: "out",
@@ -644,7 +851,7 @@ test("out @ reclaim: the rerun sweeps every parked copy and clears the marker",
});
await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -661,45 +868,41 @@ test("out @ reclaim: the rerun sweeps every parked copy and clears the marker",
// {swap(before), swap(after), reclaim} all had cases, and back @ copy had none —
// the only one where the rerun has to finish an rsync rather than a rename, and
// the only one where the space check is asked to credit a PARTIAL copy.
-test("back @ copy: a half-copied data.incoming is resumed, not restarted", async () => {
+test("back @ copy: a half-copied media.incoming is resumed, not restarted", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500), "transcript.json": "{}" },
v2: { "audio.m4a": "two".repeat(500) },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
- await copyTree(dataDir, target);
- await rm(dataDir, { recursive: true, force: true });
- await symlink(target, dataDir);
- await setConfigDataDir(paths, "alpha", target);
+ const { mediaLink, target } = await seedRelocated(paths, "alpha", root);
// The copy got one video in and died. The marker says `copy`, so the rerun
// resumes the rsync — and the surviving file keeps its mtime, which is what
// says rsync skipped it rather than re-sending it.
- const incoming = path.join(channelDir, "data.incoming");
+ const incoming = path.join(channelDir, "media.incoming");
await copyTree(path.join(target, "v1"), path.join(incoming, "v1"));
await seedMarker(paths, "alpha", { target, direction: "back", phase: "copy" });
const res = await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "back",
onLog: () => {},
});
assert.equal(res.resumed, true);
- assert.equal(res.files, 3);
+ assert.equal(res.files, 2);
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "in-place");
- assert.ok((await lstat(dataDir)).isDirectory());
+ assert.ok((await lstat(mediaLink)).isDirectory());
+ assert.ok(!(await lstat(mediaLink)).isSymbolicLink());
assert.equal(
- await readFile(path.join(dataDir, "v2", "audio.m4a"), "utf8"),
+ await readFile(path.join(channelDir, "data", "v2", "audio.m4a"), "utf8"),
"two".repeat(500),
);
assert.equal(
- (await stat(path.join(dataDir, "v1", "audio.m4a"))).mtime.getTime(),
+ (await stat(path.join(mediaLink, "v1", "audio.m4a"))).mtime.getTime(),
MTIME.getTime(),
);
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, undefined);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, undefined);
assert.equal(await pathThere(target), false);
assert.deepEqual(await leftoverCopies(channelDir), []);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
@@ -711,18 +914,13 @@ test("back @ swap: crash before the rename — the rerun finishes the swap", asy
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
// A completed relocation, then a move-back whose copy finished.
- await copyTree(dataDir, target);
- await rm(dataDir, { recursive: true, force: true });
- await symlink(target, dataDir);
- await setConfigDataDir(paths, "alpha", target);
- await copyTree(target, path.join(channelDir, "data.incoming"));
+ const { mediaLink, target } = await seedRelocated(paths, "alpha", root);
+ await copyTree(target, path.join(channelDir, "media.incoming"));
await seedMarker(paths, "alpha", { target, direction: "back", phase: "swap" });
const res = await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "back",
@@ -730,8 +928,9 @@ test("back @ swap: crash before the rename — the rerun finishes the swap", asy
});
assert.equal(res.resumed, true);
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "in-place");
- assert.ok((await lstat(dataDir)).isDirectory());
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, undefined);
+ assert.ok((await lstat(mediaLink)).isDirectory());
+ assert.ok(!(await lstat(mediaLink)).isSymbolicLink());
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, undefined);
assert.deepEqual(await leftoverCopies(channelDir), []);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
});
@@ -742,25 +941,25 @@ test("back @ swap: crash AFTER the rename — the rerun does not ENOENT forever"
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
- await copyTree(dataDir, target);
- // rename(incoming, data) committed; the process died before the config was
- // cleared. `data/` is a real directory and `incoming` is gone — so the old
- // sequence unlinked nothing (swallowed), then renamed a path that no longer
- // exists, and failed identically on every rerun.
- await setConfigDataDir(paths, "alpha", target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
+ // rename(incoming, media) committed; the process died before the config
+ // was cleared. `media/` is a real directory and `incoming` is gone — so the
+ // old sequence unlinked nothing (swallowed), then renamed a path that no
+ // longer exists, and failed identically on every rerun.
+ await setConfigMediaDir(paths, "alpha", target);
await seedMarker(paths, "alpha", { target, direction: "back", phase: "swap" });
await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "back",
onLog: () => {},
});
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "in-place");
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, undefined);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, undefined);
assert.equal(await pathIsThere(target), false);
assert.deepEqual(await leftoverCopies(channelDir), []);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
@@ -772,11 +971,13 @@ test("back @ reclaim: the config is already clear, and the rerun still finishes"
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
});
- const target = relocatedDataDir(root, "alpha");
- // The swap completed and cleared config.dataDir — which is why this case
- // was UNREACHABLE: with no dataDir the entry point threw "is not relocated"
- // and the marker named the only place that knew where the media had been.
- await copyTree(path.join(channelDir, "data"), target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ // The swap completed and cleared config.mediaDir — which is why this case
+ // was UNREACHABLE: with no mediaDir the entry point threw "is not
+ // relocated" and the marker named the only place that knew where the media
+ // had been.
+ await copyTree(mediaLink, target);
await seedMarker(paths, "alpha", {
target,
direction: "back",
@@ -784,7 +985,7 @@ test("back @ reclaim: the config is already clear, and the rerun still finishes"
});
await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "back",
@@ -800,12 +1001,12 @@ test("back @ reclaim: the config is already clear, and the rerun still finishes"
test("a marker for the other direction is never resumed into", async () => {
await withTmp(async (paths, root) => {
await seed(paths, "alpha", { v1: { "audio.m4a": "one" } });
- const target = relocatedDataDir(root, "alpha");
- await setConfigDataDir(paths, "alpha", target);
+ const target = relocatedMediaDir(root, "alpha");
+ await setConfigMediaDir(paths, "alpha", target);
await seedMarker(paths, "alpha", { target, direction: "out", phase: "swap" });
await assert.rejects(
relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "back",
@@ -830,31 +1031,32 @@ async function pathIsThere(p: string): Promise<boolean> {
// UI and both ended with rm -r on the only copy, so both get their own cases.
// ---------------------------------------------------------------------------
-// `inconsistent` — config.json records a target while `data/` is a REAL
+// `inconsistent` — config.json records a target while `media/` is a REAL
// directory — is what `rsync --copy-links` of a channel produces, which
// WORKTREES.md documents as the way to carry media into a shard. inspect()
// reports relocated:true for it, so the panel offered "Move back in place", and
-// the run copied the target to data.incoming, verified it, found data/ already a
-// real dir, logged "the swap had completed", cleared the config, rm -r'd the
-// target and then swept data.incoming. Three copies in, zero out.
-test("back: an inconsistent channel is refused, and the target keeps its bytes", { skip: T1_SKIP }, async () => {
+// the run copied the target to the incoming dir, verified it, found the live
+// dir already real, logged "the swap had completed", cleared the config, rm -r'd
+// the target and then swept the incoming copy. Three copies in, zero out.
+test("back: an inconsistent channel is refused, and the target keeps its bytes", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one" },
});
- const target = relocatedDataDir(root, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ const mediaLink = path.join(channelDir, "media");
await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
root,
onLog: () => {},
});
- // Turn the link back into a real directory WITHOUT clearing config.dataDir —
- // the --copy-links shape, reproduced.
- await rm(path.join(channelDir, "data"));
- await copyTree(target, path.join(channelDir, "data"));
+ // Turn the link back into a real directory WITHOUT clearing
+ // config.mediaDir — the --copy-links shape, reproduced.
+ await rm(mediaLink);
+ await copyTree(target, mediaLink);
assert.equal(
(await inspectChannelMedia(paths, "alpha")).status,
"inconsistent",
@@ -882,21 +1084,21 @@ test("back: an inconsistent channel is refused, and the target keeps its bytes",
await readFile(path.join(channelDir, "data", "v1", "audio.m4a"), "utf8"),
"one",
);
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, target);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, target);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
- assert.equal((await readdir(channelDir)).includes("data.incoming"), false);
+ assert.equal((await readdir(channelDir)).includes("media.incoming"), false);
});
});
// An `unreachable` channel (the drive is not mounted) is refused for the same
// reason: nothing can vouch for what the target holds, and the run ends by
// deleting it.
-test("back: an unreachable channel is refused", { skip: T1_SKIP }, async () => {
+test("back: an unreachable channel is refused", async () => {
await withTmp(async (paths, root) => {
await seed(paths, "alpha", { v1: { "audio.m4a": "one" } });
- const target = relocatedDataDir(root, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -905,10 +1107,10 @@ test("back: an unreachable channel is refused", { skip: T1_SKIP }, async () => {
});
// The platter goes away. The link dangles; config still names the target.
await rm(path.dirname(target), { recursive: true, force: true });
- assert.equal(
- (await inspectChannelMedia(paths, "alpha")).status,
- "unreachable",
- );
+ const away = await inspectChannelMedia(paths, "alpha");
+ assert.equal(away.status, "unreachable");
+ // The text does not care.
+ assert.equal(away.text.readable, true);
await assert.rejects(
() =>
relocateChannelMedia({
@@ -925,7 +1127,7 @@ test("back: an unreachable channel is refused", { skip: T1_SKIP }, async () => {
// THE SELF-RELOCATION. root = <transcriptsDir>/channels makes the target the
// source: rsync src/ src/ succeeds, verifyCopy compares the tree with itself,
-// the swap parks `data/` (which IS the verified "target") and links to a path
+// the swap parks `media/` (which IS the verified "target") and links to a path
// that no longer exists, and the reclaim sweep deletes the only copy. The
// panel's own help text describes transcripts/channels almost word for word.
test("a root inside the corpus is refused by the job and by the preview", async () => {
@@ -944,7 +1146,7 @@ test("a root inside the corpus is refused by the job and by the preview", async
await assert.rejects(
() =>
relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -960,16 +1162,22 @@ test("a root inside the corpus is refused by the job and by the preview", async
`preview accepted ${bad}`,
);
}
- // Untouched: still a real directory, no config, no marker, no parked copy.
+ // Untouched: still a real file in a real directory (not even tiered), no
+ // config, no marker, no parked copy.
assert.ok((await lstat(path.join(channelDir, "data"))).isDirectory());
+ assert.ok(
+ (await lstat(path.join(channelDir, "data", "v1", "audio.m4a"))).isFile(),
+ );
assert.equal(
await readFile(path.join(channelDir, "data", "v1", "audio.m4a"), "utf8"),
"one",
);
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, undefined);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, undefined);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
assert.equal(
- (await readdir(channelDir)).filter((n) => n.startsWith("data.")).length,
+ (await readdir(channelDir)).filter(
+ (n) => n.startsWith("media") || n.startsWith("data."),
+ ).length,
0,
);
});
@@ -1015,24 +1223,24 @@ test("a relative root is refused by the preview, not only by the job", async ()
// THE OMNIMIRROR REFUSAL (2026-09-13), in a tmpdir.
//
// A 131 GB copy landed byte-complete and the verify refused it. The whole of the
-// drift was one directory timestamp: a sidecar written into `data/v4p31nz/`
-// while rsync was already past that directory bumped the SOURCE directory's
-// mtime and left the target's behind. `.d..t` is rsync's itemization for
-// exactly that — a directory, and only its time differs — and it means nothing
-// about the bytes.
+// drift was one directory timestamp: a sidecar written into a video dir while
+// rsync was already past that directory bumped the SOURCE directory's mtime and
+// left the target's behind. `.d..t` is rsync's itemization for exactly that — a
+// directory, and only its time differs — and it means nothing about the bytes.
//
// Both cases run at phase `swap`, where the re-verify runs on its own with no
// `rsync -a` ahead of it. That is the shape that can see the difference: in the
// copy phase the transfer itself would have set the timestamps.
-test("out @ swap: a directory timestamp is settled by one more pass, not refused", { skip: T1_SKIP }, async () => {
+test("out @ swap: a directory timestamp is settled by one more pass, not refused", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500), "transcript.json": "{}" },
v2: { "audio.m4a": "two".repeat(500) },
});
- const target = relocatedDataDir(root, "alpha");
- await copyTree(path.join(channelDir, "data"), target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
// The echo of the sidecar: the target's copy of v1/ carries a different
// mtime from the source's. Every file underneath is identical.
await utimes(path.join(target, "v1"), MTIME, MTIME);
@@ -1040,7 +1248,7 @@ test("out @ swap: a directory timestamp is settled by one more pass, not refused
const lines: string[] = [];
const res = await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -1056,31 +1264,35 @@ test("out @ swap: a directory timestamp is settled by one more pass, not refused
);
// And the move completed: link, config, no leftovers, no marker.
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, target);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, target);
assert.deepEqual(await leftoverCopies(channelDir), []);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
assert.equal(
- await readFile(path.join(target, "v1", "transcript.json"), "utf8"),
+ await readFile(path.join(channelDir, "data", "v1", "audio.m4a"), "utf8"),
+ "one".repeat(500),
+ );
+ assert.equal(
+ await readFile(path.join(channelDir, "data", "v1", "transcript.json"), "utf8"),
"{}",
);
});
});
// CONTENT drift at the swap's re-verify, arriving the way the timestamp did —
-// a sidecar written into the source after the copy. Until release 16 slice RM
-// any file line refused; now the re-verify runs in mirror mode while `data/` is
-// still the live directory, and one change gets one more mirror pass, exactly
-// as in the copy phase. A second change is still a refusal ("a verify failure
-// keeps the source …" above).
-test("out @ swap: a file the target is missing is mirrored by one more pass", { skip: T1_SKIP }, async () => {
+// a file written into the source after the copy. Until release 16 slice RM
+// any file line refused; now the re-verify runs in mirror mode while `media/`
+// is still the live directory, and one change gets one more mirror pass,
+// exactly as in the copy phase. A second change is still a refusal ("a verify
+// failure keeps the source …" above).
+test("out @ swap: a file the target is missing is mirrored by one more pass", async () => {
await withTmp(async (paths, root) => {
- const channelDir = await seed(paths, "alpha", {
+ await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
- await copyTree(dataDir, target);
- await writeFile(path.join(dataDir, "v1", "diarization.json"), "{}");
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
+ await writeFile(path.join(mediaLink, "v1", "transcript.live_chat.json"), "[]");
await seedMarker(paths, "alpha", { target, direction: "out", phase: "swap" });
const res = await relocateChannelMedia({
@@ -1093,16 +1305,16 @@ test("out @ swap: a file the target is missing is mirrored by one more pass", {
});
assert.equal(res.retried, true);
assert.equal(
- await readFile(path.join(target, "v1", "diarization.json"), "utf8"),
- "{}",
+ await readFile(path.join(target, "v1", "transcript.live_chat.json"), "utf8"),
+ "[]",
);
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, target);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, target);
});
});
// A PARKED COPY IS NEVER MIRRORED FROM. Once the rename has committed, the
-// link is what readers follow and `data.relocated-*` is not live media any
+// link is what readers follow and `media.relocated-*` is not live media any
// more: the re-verify against it is the strict one, as it always was — a
// difference refuses, and nothing on the target is deleted to match a stale
// copy.
@@ -1111,12 +1323,12 @@ test("out @ swap: after the rename, the re-verify against the parked copy is str
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
- await copyTree(dataDir, target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
// A file on the target the parked copy lacks.
await writeFile(path.join(target, "v1", "newer.json"), "{}");
- await rename(dataDir, path.join(channelDir, "data.relocated-1"));
+ await rename(mediaLink, path.join(channelDir, "media.relocated-1"));
await seedMarker(paths, "alpha", { target, direction: "out", phase: "swap" });
await assert.rejects(
@@ -1155,8 +1367,8 @@ const TRANSCRIBING: ChannelWriter = {
// THE REFUSAL OVER A RUNNING JOB, the job's first step. The registry is
// injected: what is pinned is that the move asks, refuses naming the writer,
-// and has touched nothing — no marker, no target directory, the source as it
-// was.
+// and has touched nothing — no marker, no target directory, no tiering, the
+// source as it was.
test("a move refuses to start over a running job, naming it, and touches nothing", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
@@ -1178,6 +1390,10 @@ test("a move refuses to start over a running job, naming it, and touches nothing
assert.equal(await readRelocationMarker(paths, "alpha"), null);
await assert.rejects(() => stat(path.join(root, "alpha")));
assert.ok((await lstat(path.join(channelDir, "data"))).isDirectory());
+ assert.equal(await pathIsThere(path.join(channelDir, "media")), false);
+ assert.ok(
+ (await lstat(path.join(channelDir, "data", "v1", "audio.m4a"))).isFile(),
+ );
});
});
@@ -1230,16 +1446,21 @@ test("a writer seen once the marker is written refuses, and a fresh move's marke
);
assert.equal(asked, 2);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
+ // Tiered in place (the preflight ran) and nothing else: an in-place channel.
assert.equal(
(await inspectChannelMedia(paths, "alpha", undefined, { fresh: true })).status,
"in-place",
);
assert.deepEqual(
- await readdir(path.join(relocatedDataDir(root, "alpha"))),
+ await readdir(path.join(relocatedMediaDir(root, "alpha"))),
[],
"nothing was copied",
);
assert.ok((await lstat(path.join(channelDir, "data"))).isDirectory());
+ assert.equal(
+ await readFile(path.join(channelDir, "data", "v1", "audio.m4a"), "utf8"),
+ "one",
+ );
});
});
@@ -1247,15 +1468,15 @@ test("a writer seen once the marker is written refuses, and a fresh move's marke
// on the destination that has since gone from the source. The resume used to
// copy everything else and refuse on the counts (1755 against 1750), and no
// rerun could settle it; the mirror pass now does.
-test("a resume with a stale extra dir on the destination completes", { skip: T1_SKIP }, async () => {
+test("a resume with a stale extra dir on the destination completes", async () => {
await withTmp(async (paths, root) => {
- const channelDir = await seed(paths, "alpha", {
+ await seed(paths, "alpha", {
v50t5yt: { "audio.mp3": "a".repeat(64), "transcript.json": "{}" },
v51fpcd: { "audio.mp3": "b".repeat(64) },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
- await copyTree(dataDir, target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
const scratch = path.join(target, "v50t5yt", ".audio.mp3.parakeet");
await mkdir(scratch, { recursive: true });
for (const f of ["meta", "win-0000", "win-0001", "win-0002", "win-0003"]) {
@@ -1272,7 +1493,7 @@ test("a resume with a stale extra dir on the destination completes", { skip: T1_
onLog: () => {},
});
assert.equal(res.resumed, true);
- assert.equal(res.files, 3);
+ assert.equal(res.files, 2);
await assert.rejects(() => stat(scratch));
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
assert.equal(await readRelocationMarker(paths, "alpha"), null);
@@ -1292,10 +1513,10 @@ test("back: a stale extra on the copy coming home is removed on resume", async (
root,
onLog: () => {},
});
- const target = relocatedDataDir(root, "alpha");
- // An interrupted move back: `data.incoming` holds the copy, plus a file
+ const target = relocatedMediaDir(root, "alpha");
+ // An interrupted move back: `media.incoming` holds the copy, plus a file
// the target (the source of this direction) no longer has.
- const incoming = path.join(channelDir, "data.incoming");
+ const incoming = path.join(channelDir, "media.incoming");
await copyTree(target, incoming);
await writeFile(path.join(incoming, "v1", "gone.json"), "{}");
await seedMarker(paths, "alpha", { target, direction: "back", phase: "copy" });
@@ -1308,9 +1529,10 @@ test("back: a stale extra on the copy coming home is removed on resume", async (
onLog: () => {},
});
assert.equal(res.resumed, true);
- const dataDir = path.join(channelDir, "data");
- assert.ok((await lstat(dataDir)).isDirectory());
- assert.deepEqual(await readdir(path.join(dataDir, "v1")), ["audio.m4a"]);
+ const mediaLink = path.join(channelDir, "media");
+ assert.ok((await lstat(mediaLink)).isDirectory());
+ assert.ok(!(await lstat(mediaLink)).isSymbolicLink());
+ assert.deepEqual(await readdir(path.join(mediaLink, "v1")), ["audio.m4a"]);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
});
});
@@ -1318,16 +1540,19 @@ test("back: a stale extra on the copy coming home is removed on resume", async (
// RECONCILE AND RESUME — the remediation (the ruling's last bullet). An extra
// file and a changed one on the destination: the job says what it found, by
// kind, makes the copy match the source and finishes the move.
-test("reconcile: an extra and a changed file on the destination are settled, and the move completes", { skip: T1_SKIP }, async () => {
+test("reconcile: an extra and a changed file on the destination are settled, and the move completes", async () => {
await withTmp(async (paths, root) => {
- const channelDir = await seed(paths, "alpha", {
- v1: { "audio.m4a": "one".repeat(100), "transcript.json": '{"v":2}' },
+ await seed(paths, "alpha", {
+ v1: {
+ "audio.m4a": "one".repeat(100),
+ "transcript.live_chat.json": '{"v":2}',
+ },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
- await copyTree(dataDir, target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
await writeFile(path.join(target, "v1", "stale.json"), "{}");
- await writeFile(path.join(target, "v1", "transcript.json"), '{"v":1}');
+ await writeFile(path.join(target, "v1", "transcript.live_chat.json"), '{"v":1}');
await seedMarker(paths, "alpha", { target, direction: "out", phase: "copy" });
const lines: string[] = [];
@@ -1344,13 +1569,13 @@ test("reconcile: an extra and a changed file on the destination are settled, and
assert.equal(res.reconciled, true);
const said = lines.find((l) => l.startsWith("Reconciling:")) ?? "";
assert.match(said, /1 extra on the destination \(v1\/stale\.json\)/);
- assert.match(said, /1 changed \(v1\/transcript\.json\)/);
+ assert.match(said, /1 changed \(v1\/transcript\.live_chat\.json\)/);
assert.deepEqual(
(await readdir(path.join(target, "v1"))).sort(),
- ["audio.m4a", "transcript.json"],
+ ["audio.m4a", "transcript.live_chat.json"],
);
assert.equal(
- await readFile(path.join(target, "v1", "transcript.json"), "utf8"),
+ await readFile(path.join(target, "v1", "transcript.live_chat.json"), "utf8"),
'{"v":2}',
);
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
@@ -1360,13 +1585,14 @@ test("reconcile: an extra and a changed file on the destination are settled, and
// A marker past the copy phase has nothing to reconcile: the run is a plain
// resume, and says so (the review's L3).
-test("reconcile: a marker past the copy phase resumes, and does not claim a reconcile", { skip: T1_SKIP }, async () => {
+test("reconcile: a marker past the copy phase resumes, and does not claim a reconcile", async () => {
await withTmp(async (paths, root) => {
- const channelDir = await seed(paths, "alpha", {
+ await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(100) },
});
- const target = relocatedDataDir(root, "alpha");
- await copyTree(path.join(channelDir, "data"), target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
await seedMarker(paths, "alpha", { target, direction: "out", phase: "swap" });
const lines: string[] = [];
const res = await relocateChannelMedia({
diff --git a/common/controller/relocateChannelMedia.ts b/common/controller/relocateChannelMedia.ts
@@ -2,6 +2,7 @@ import path from "node:path";
import {
access,
constants as fsConstants,
+ lstat,
mkdir,
readdir,
realpath,
@@ -54,44 +55,67 @@ import { formatBytes } from "../lib/format";
import {
forgetChannelMedia,
inspectChannelMedia,
- relocatedDataDir,
+ legacyDetail,
relocationMarkerPath,
type RelocationDirection,
type RelocationMarker,
type RelocationPhase,
} from "../lib/channelMedia";
import {
+ channelMediaLink,
+ relocatedMediaDir,
+ tierChannelMedia,
+} from "../lib/mediaTier-server";
+import {
patchChannelConfig,
readChannelConfig,
writeChannelConfig,
} from "./channels";
-// MOVE A CHANNEL'S MEDIA TO ANOTHER DRIVE, AND BACK.
+// MOVE A CHANNEL'S MEDIA TIER TO ANOTHER DRIVE, AND BACK (release 17).
+//
+// The unit of a move is `channels/<slug>/media` — the channel's BIG files
+// (lib/mediaTier.ts says which), each reached from `data/<id>/<name>` by a
+// RELATIVE link `../../media/<id>/<name>`. The text — transcripts, cues,
+// metadata, every sidecar, `clips/` — never moves: `data/` stays a real
+// directory on the corpus disk. A move copies `media/` to `<root>/<slug>/media`,
+// verifies it, and makes `channels/<slug>/media` ONE absolute link to it while
+// `config.json` records the target in `mediaDir`. Not one per-file link changes,
+// in either direction: they are relative to `media`, whatever `media` is.
//
-// The mechanism is a symlink (see common/lib/channelMedia.ts for why): the media
-// is copied to `<root>/<slug>/data`, verified, and `channels/<slug>/data` becomes
-// an absolute link to it while `config.json` records the target in `dataDir`.
-// Nothing that reads a channel changes, because the on-disk contract
-// `channelDir/data/<id>/…` is preserved exactly.
+// A CLASSIC CHANNEL IS TIERED FIRST. One whose big files are still real files
+// in `data/<id>/` (no `media/` yet) is tiered in place by the move's preflight —
+// `media/` made a real directory, each file renamed into it and linked, all on
+// one filesystem, in seconds — and then moved like any other.
//
// THE SOURCE IS NEVER TOUCHED UNTIL THE COPY IS VERIFIED. An abort, a full
-// target, a crash, an rsync failure — all of them leave `data/` exactly where it
-// was, and leave the partial copy resumable. The only destructive step is the
-// reclaim at the very end, after the link is in place and the config is written.
+// target, a crash, an rsync failure — all of them leave `media/` exactly where
+// it was, and leave the partial copy resumable. The only destructive step is
+// the reclaim at the very end, after the link is in place and the config is
+// written.
//
-// `rsync -a` preserves mtimes, which is what makes this free for the LMDB index:
-// it stores ids and mtimes, no paths, so a relocated channel needs no reindex.
+// `rsync -a` preserves mtimes, and the LMDB index stats no media file at all
+// (presence by name, from one readdir of `data/<id>/`), so a move needs no
+// reindex.
+//
+// THE RETIRED LAYOUT IS REFUSED. A channel whose whole `data/` was moved by the
+// mover before release 17 (`data/` a link, `config.dataDir`) is `legacy`: it is
+// migrated by `archilyzer storage migrate-tier`, never moved by this.
const BYTES_PER_GB = 1024 ** 3;
export type RelocateChannelMediaResult = {
slug: string;
direction: RelocationDirection;
- // Absolute path of the relocated data dir. For "back" this is what was
- // reclaimed, not where the media now lives.
+ // Absolute path of the relocated media dir, `<root>/<slug>/media`. For
+ // "back" this is what was reclaimed, not where the media now lives.
target: string;
bytes: number;
files: number;
+ // How many big files the preflight tiered into `media/` before the copy (a
+ // classic channel's first move; 0 for "back" and for a channel already
+ // tiered).
+ tiered: number;
// True when the run picked up an interrupted one from its marker rather than
// starting from scratch.
resumed: boolean;
@@ -155,8 +179,48 @@ type RelocateOpts = {
// on this channel's slug, and the relocation queue runs one move at a time.
const RELOCATE_KIND = "relocate-channel-media";
+// MEDIA WRITERS ONLY (release 17 ruling): a move carries `media/` and nothing
+// else, so a digest, a normalize or any other reader/writer of the TEXT may run
+// while it does — `mediaOnly` keeps the jobs whose kind opens or writes a big
+// file and the lane units of every lane but the digest one.
function liveWriters(slug: string): ChannelWriter[] {
- return channelWriters(slug, { ignoreKinds: [RELOCATE_KIND] });
+ return channelWriters(slug, {
+ ignoreKinds: [RELOCATE_KIND],
+ mediaOnly: true,
+ });
+}
+
+// THE RETIRED WHOLE-DIRECTORY LAYOUT, refused by every entry point of the mover
+// with the sentence that names the way out. Answered from the corpus disk alone
+// (`data/` a link, or `config.dataDir` recorded) — never a call to the far
+// drive.
+async function legacyRefusal(
+ channelDir: string,
+ slug: string,
+ config: { dataDir?: string } | null,
+): Promise<string | null> {
+ let dataIsLink = false;
+ try {
+ dataIsLink = (await lstat(path.join(channelDir, "data"))).isSymbolicLink();
+ } catch {
+ /* no data/ yet */
+ }
+ if (!dataIsLink && !config?.dataDir?.trim()) return null;
+ return `Channel "${slug}" cannot be moved: ${legacyDetail(slug)}.`;
+}
+
+// A TIER MIGRATION'S MARKER is not this mover's to resume or replace: the
+// migration rebuilds `data/` itself, with the editor stopped.
+function tierMigrationRefusal(
+ slug: string,
+ marker: RelocationMarker | null,
+): string | null {
+ if (marker?.scope !== "tier-migration") return null;
+ return (
+ `Channel "${slug}" has a media-tier migration in flight or interrupted ` +
+ `(phase "${marker.phase}") — finish it with archilyzer storage ` +
+ `migrate-tier ${slug}.`
+ );
}
// A MOVE NEVER STARTS OVER A WRITER, and never waits silently for one either: it
@@ -184,13 +248,27 @@ export type RelocationPreview = {
sameDevice: boolean;
// A resumable partial copy from an earlier attempt is already at the target.
existingPartial: boolean;
+ // Big files the preview tiered into `media/` first (a classic channel: its
+ // audio and raw live chat renamed into `channels/<slug>/media/<id>/` and
+ // linked, on the corpus disk). Idempotent: 0 on a second preview.
+ tieredFirst: number;
};
-// Resolved through every symlink when the path exists, lexically when it does
-// not. A destination root that does not exist yet is refused elsewhere; a root
-// that exists and is a symlink back into the corpus is exactly what this is for.
+// Resolved through every symlink of its DEEPEST EXISTING ANCESTOR, the rest
+// joined on lexically. A destination root that does not exist yet is refused
+// elsewhere; a root that exists and is a symlink back into the corpus is
+// exactly what this is for. The ancestor walk matters since release 17: the
+// target `<root>/<slug>/media` usually does not exist yet, and a `<slug>` level
+// that links back into the channel dir must still resolve there.
async function realOrResolved(p: string): Promise<string> {
- return await realpath(p).catch(() => path.resolve(p));
+ const abs = path.resolve(p);
+ try {
+ return await realpath(abs);
+ } catch {
+ const parent = path.dirname(abs);
+ if (parent === abs) return abs;
+ return path.join(await realOrResolved(parent), path.basename(abs));
+ }
}
// `child` IS `parent`, or lives under it.
@@ -201,10 +279,10 @@ function isWithin(parent: string, child: string): boolean {
// WHY A ROOT INSIDE THE CORPUS IS NOT MERELY POINTLESS BUT DESTRUCTIVE.
//
-// Take `root = <transcriptsDir>/channels`. Then `relocatedDataDir(root, slug)`
-// is `<channels>/<slug>/data` — the SOURCE. `rsync -a src/ src/` succeeds,
-// verifyCopy compares the tree with itself and passes, the swap renames `data/`
-// to `data.relocated-<ts>` (which moves the "target" it just verified), creates
+// Take `root = <transcriptsDir>/channels`. Then `relocatedMediaDir(root, slug)`
+// is `<channels>/<slug>/media` — the SOURCE. `rsync -a src/ src/` succeeds,
+// verifyCopy compares the tree with itself and passes, the swap renames `media/`
+// to `media.relocated-<ts>` (which moves the "target" it just verified), creates
// a symlink pointing at a path that no longer exists, and the reclaim sweep then
// `rm -r`s the parked directory — the only copy of the media. Nothing in the
// happy path can notice, because every check it runs is comparing the tree with
@@ -246,17 +324,17 @@ export async function relocationRootProblem(opts: {
// Belt and braces for a root that is outside the corpus but whose `<slug>`
// level is a link back into it: the target is resolved separately, because
// realpath of the root cannot see through a link one level down.
- const realTarget = await realOrResolved(relocatedDataDir(realRoot, opts.slug));
+ const realTarget = await realOrResolved(relocatedMediaDir(realRoot, opts.slug));
if (isWithin(realChannel, realTarget) || isWithin(realTarget, realChannel)) {
return (
- `The destination ${relocatedDataDir(root, opts.slug)} resolves inside ` +
+ `The destination ${relocatedMediaDir(root, opts.slug)} resolves inside ` +
`the channel directory ${channelDir} — the media would be copied onto ` +
`itself and then reclaimed.`
);
}
if (isWithin(realCorpus, realTarget)) {
return (
- `The destination ${relocatedDataDir(root, opts.slug)} resolves inside ` +
+ `The destination ${relocatedMediaDir(root, opts.slug)} resolves inside ` +
`the corpus at ${opts.paths.transcriptsDir}. Pick a directory on the ` +
`other drive.`
);
@@ -290,7 +368,7 @@ export async function relocationRootProblem(opts: {
// Two checks, and the second is the one that catches the case above:
//
// 1. `stat(root)` must be a directory. The move creates `<root>/<slug>` and
-// `<root>/<slug>/data`, never the root itself — a root is a fact about the
+// `<root>/<slug>/media`, never the root itself — a root is a fact about the
// machine, not something a move gets to invent.
// 2. When the root belongs to a LOCATION that has learned a `volume.uuid`,
// the probe must answer `available` with a KNOWN identity whose uuid
@@ -313,8 +391,8 @@ function locationForRoot(
locations: StorageLocation[],
): StorageLocation | null {
// `root + "/x"` rather than `root`: `locationOfDataDir` is deliberately
- // STRICT ("under", not "equal to"), because a channel's dataDir is always
- // `<root>/<slug>/data` and equality there only ever means a misconfiguration.
+ // STRICT ("under", not "equal to"), because a channel's mediaDir is always
+ // `<root>/<slug>/media` and equality there only ever means a misconfiguration.
// Here equality is the ordinary case — the destination root IS the location
// root — so the question is asked about a path one level inside it.
return locationOfDataDir(path.join(root, "x"), locations);
@@ -394,10 +472,10 @@ export async function relocationRootPresenceProblem(
return null;
}
-// The inverse of `relocatedDataDir`: `<root>/<slug>/data` -> `<root>`. The
-// suffix is fixed (channelMedia.ts says so, and deleteChannel recognises a
+// The inverse of `relocatedMediaDir`: `<root>/<slug>/media` -> `<root>`. The
+// suffix is fixed (mediaTier-server.ts says so, and deleteChannel recognises a
// target by it), so this is two dirnames and not a guess.
-export function rootOfRelocatedDataDir(target: string, slug: string): string {
+export function rootOfRelocatedMediaDir(target: string, slug: string): string {
const parent = path.dirname(target);
return path.basename(parent) === slug ? path.dirname(parent) : parent;
}
@@ -417,7 +495,7 @@ export async function assertRelocationRootPresent(
if (problem) throw new Error(problem);
}
-// Every leftover a crashed run can have parked next to `data/`, in one list.
+// Every leftover a crashed run can have parked next to `media/`, in one list.
// The reclaim phase sweeps ALL of them rather than the one name the run that is
// finishing happens to hold: a crash between the config write and the reclaim
// marker leaves a full second copy of the channel on the source volume, and
@@ -426,7 +504,7 @@ export async function assertRelocationRootPresent(
async function parkedSiblings(channelDir: string): Promise<string[]> {
const names = await readdir(channelDir).catch(() => [] as string[]);
return names
- .filter((n) => n.startsWith("data.relocated-") || n === "data.incoming")
+ .filter((n) => n.startsWith("media.relocated-") || n === "media.incoming")
.map((n) => path.join(channelDir, n));
}
@@ -469,12 +547,14 @@ async function readMarkerRaw(
}
// What the operator sees before committing to a move. Cheap enough to run on a
-// form keystroke debounce: one tree walk of the channel plus two statfs calls.
+// form keystroke debounce: one tree walk of the channel's `media/` plus two
+// statfs calls — and, the first time for a classic channel, the tiering (the
+// same renames the job's preflight would make, on the corpus disk; idempotent).
export async function previewRelocation({
paths,
slug,
root,
- writers = (s) => channelWriters(s),
+ writers = (s) => channelWriters(s, { mediaOnly: true }),
}: {
paths: Paths;
slug: string;
@@ -489,6 +569,10 @@ export async function previewRelocation({
// the worst possible answer.
const problem = await relocationRootProblem({ paths, slug, root });
if (problem) throw new Error(problem);
+ const channelDir = path.join(paths.channelsDir, slug);
+ const config = await readChannelConfig(paths, slug);
+ const legacy = await legacyRefusal(channelDir, slug, config);
+ if (legacy) throw new Error(legacy);
// The same refusal the job's first step gives, before the operator commits.
assertNoWriters(slug, writers);
// EXISTENCE, HERE AS WELL AS IN THE JOB. getFreeBytes walks up to the nearest
@@ -501,8 +585,25 @@ export async function previewRelocation({
`(is the drive mounted?)`,
);
}
- const source = path.join(paths.channelsDir, slug, "data");
- const target = relocatedDataDir(root, slug);
+ // TIER FIRST, as the job will: the figures below are then the ones the job
+ // moves. Only an in-place channel (a relocated one's tier is on the far
+ // drive, and a preview copies nothing there), only with nothing in flight (a
+ // marker means a move has already tiered it, and the hook writes nothing
+ // under one anyway), and only one with something downloaded (no `data/` → no
+ // `media/` invented).
+ let tieredFirst = 0;
+ const marker = await readMarkerRaw(paths, slug);
+ if (
+ !marker &&
+ !config?.mediaDir?.trim() &&
+ (await isDirectory(path.join(channelDir, "data")))
+ ) {
+ tieredFirst = (
+ await tierChannelMedia(paths, slug, { createMediaDir: true })
+ ).tiered;
+ }
+ const source = channelMediaLink(paths, slug);
+ const target = relocatedMediaDir(root, slug);
const [measured, freeOnRoot, freeOnSource, existingPartial] = await Promise.all([
measureTree(source),
getFreeBytes(root),
@@ -511,7 +612,7 @@ export async function previewRelocation({
]);
let sameDevice = false;
try {
- const [a, b] = await Promise.all([stat(source), stat(root)]);
+ const [a, b] = await Promise.all([stat(paths.channelsDir), stat(root)]);
sameDevice = a.dev === b.dev;
} catch {
/* an unmounted or absent root is not "same device" */
@@ -525,6 +626,7 @@ export async function previewRelocation({
freeOnSource,
sameDevice,
existingPartial,
+ tieredFirst,
};
}
@@ -551,8 +653,12 @@ export async function relocateChannelMedia(
assertNoWriters(slug, writers);
const channelDir = path.join(paths.channelsDir, slug);
- const dataDir = path.join(channelDir, "data");
+ const mediaLink = channelMediaLink(paths, slug);
const existingMarker = await readMarkerRaw(paths, slug);
+ const migrating = tierMigrationRefusal(slug, existingMarker);
+ if (migrating) throw new Error(migrating);
+ const legacy = await legacyRefusal(channelDir, slug, config);
+ if (legacy) throw new Error(legacy);
if (opts.reconcile && !existingMarker) {
throw new Error(
`Channel "${slug}" has no relocation marker — there is no interrupted ` +
@@ -563,13 +669,13 @@ export async function relocateChannelMedia(
if (direction === "back") {
// THE MARKER IS THE SECOND SOURCE OF TRUTH FOR THE TARGET, and without it
- // the resume path was unreachable. moveBack clears config.dataDir as part
+ // the resume path was unreachable. moveBack clears config.mediaDir as part
// of its swap, so a crash after that point left a rerun reading "this
// channel is not relocated" from the config and throwing — while a marker
// sat next to it naming the very target still holding the media, and
// inspect() reported in-transition forever.
const resume = existingMarker?.direction === "back" ? existingMarker : null;
- const target = config.dataDir?.trim() || resume?.target;
+ const target = config.mediaDir?.trim() || resume?.target;
if (!target) {
throw new Error(
`Channel "${slug}" is not relocated — its media is already in place`,
@@ -596,7 +702,7 @@ export async function relocateChannelMedia(
io,
config,
channelDir,
- dataDir,
+ mediaLink,
target,
log,
onProgress,
@@ -620,16 +726,16 @@ export async function relocateChannelMedia(
storage: io.read().storage,
});
if (rootProblem) throw new Error(rootProblem);
- const target = relocatedDataDir(root, slug);
+ const target = relocatedMediaDir(root, slug);
if (existingMarker && existingMarker.target !== target) {
throw new Error(
`A relocation to ${existingMarker.target} is already in progress for "${slug}" — ` +
`finish or clear it before moving to ${target}`,
);
}
- if (config.dataDir?.trim() && config.dataDir.trim() !== target) {
+ if (config.mediaDir?.trim() && config.mediaDir.trim() !== target) {
throw new Error(
- `Channel "${slug}" is already relocated to ${config.dataDir.trim()}. ` +
+ `Channel "${slug}" is already relocated to ${config.mediaDir.trim()}. ` +
`Move it back in place first.`,
);
}
@@ -648,7 +754,7 @@ export async function relocateChannelMedia(
io,
config,
channelDir,
- dataDir,
+ mediaLink,
root,
target,
log,
@@ -687,7 +793,7 @@ async function moveOut(args: {
io: { read: () => SiteSettings };
config: NonNullable<Awaited<ReturnType<typeof readChannelConfig>>>;
channelDir: string;
- dataDir: string;
+ mediaLink: string;
root: string;
target: string;
log: (m: string) => void;
@@ -698,7 +804,8 @@ async function moveOut(args: {
writers: (slug: string) => ChannelWriter[];
reconcile: boolean;
}): Promise<RelocateChannelMediaResult> {
- const { paths, slug, channelDir, dataDir, root, target, log, signal } = args;
+ const { paths, slug, channelDir, mediaLink, root, target, log, signal } = args;
+ const dataDir = path.join(channelDir, "data");
// PREFLIGHT. Everything that can refuse does so here, before a single byte is
// written and before the marker exists.
@@ -732,9 +839,8 @@ async function moveOut(args: {
);
}
- // A channel that has downloaded nothing has no data/ at all, and rsync exits
- // 23 on a missing source — an error message about a partial transfer for
- // something that is not a transfer. Say what is actually the matter.
+ // A channel that has downloaded nothing has no data/ at all. Say so, rather
+ // than invent a `media/` for it and move nothing.
if (args.phase === "copy" && !(await isDirectory(dataDir))) {
throw new Error(
`Channel "${slug}" has no ${dataDir} to move — nothing has been ` +
@@ -742,7 +848,30 @@ async function moveOut(args: {
);
}
- const measured = await measureTree(dataDir);
+ // TIER FIRST (release 17): after the writers check (the job's first step,
+ // above) and before anything is measured. A classic channel's big files are
+ // renamed into a new real `media/` and linked — the same filesystem, so a
+ // rename each — and a channel tiered already counts them as "already". A
+ // resumed run tiers nothing: its marker stands, and the hook writes nothing
+ // under one (a file a writer finished since stays real, on the corpus disk,
+ // until the next sweep tiers it onto the far side).
+ let tiered = 0;
+ if (args.phase === "copy") {
+ const counts = await tierChannelMedia(paths, slug, {
+ createMediaDir: true,
+ onLog: (line) => log(line.replace(/\n$/, "")),
+ });
+ tiered = counts.tiered;
+ if (tiered > 0) log(`Tiered ${tiered} file(s) into ${mediaLink} first`);
+ if (!(await isDirectory(mediaLink))) {
+ throw new Error(
+ `Channel "${slug}" has no ${mediaLink} to move — it could not be ` +
+ `made a directory`,
+ );
+ }
+ }
+
+ const measured = await measureTree(mediaLink);
// Sticky across both verify points below: a retry at either one is the fact
// the caller wants reported, and neither overwrites the other's answer.
let verifyRetried = false;
@@ -793,6 +922,7 @@ async function moveOut(args: {
direction: "out",
startedAt: new Date().toISOString(),
phase: "copy",
+ scope: "media",
});
await assertNoWritersUnderMarker({
paths,
@@ -807,10 +937,10 @@ async function moveOut(args: {
verifyRetried ||= (
await copyMirrorVerify({
rsyncBin: paths.rsyncBin,
- src: dataDir,
+ src: mediaLink,
dest: target,
// The channel's media, which --delete may never reach.
- live: dataDir,
+ live: mediaLink,
log,
progress: makeProgressSink({
totalBytes: measured.bytes,
@@ -822,7 +952,7 @@ async function moveOut(args: {
reconcile: args.reconcile,
cancelled: () =>
new Error(
- `Cancelled. ${dataDir} is untouched and the partial copy at ${target} ` +
+ `Cancelled. ${mediaLink} is untouched and the partial copy at ${target} ` +
`is resumable — rerun to continue.`,
),
})
@@ -837,14 +967,15 @@ async function moveOut(args: {
direction: "out",
startedAt: new Date().toISOString(),
phase: "swap",
+ scope: "media",
});
// EVERY STEP BELOW OBSERVES THE DISK INSTEAD OF ASSUMING THE LAST ONE RAN.
// The marker says how far the previous attempt got; it cannot say how far
// it got THROUGH a phase, and a crash lands between any two syscalls.
- const state = await linkOrDirState(dataDir);
+ const state = await linkOrDirState(mediaLink);
const parked = (await parkedSiblings(channelDir)).filter((p) =>
- path.basename(p).startsWith("data.relocated-"),
+ path.basename(p).startsWith("media.relocated-"),
);
// Re-verify, because a resumed run did not do the copy in this process and
@@ -852,13 +983,13 @@ async function moveOut(args: {
// against is whichever copy of it still exists; once the swap has committed
// there is none, and there is nothing left to check.
//
- // MIRRORED ONLY FROM THE LIVE MEDIA. While `data/` is still the real
+ // MIRRORED ONLY FROM THE LIVE MEDIA. While `media/` is still the real
// directory it is the source, and a difference gets the mirror pass a copy
- // phase would give it. A parked `data.relocated-*` is not live any more —
+ // phase would give it. A parked `media.relocated-*` is not live any more —
// the link already points at the target — so the target is never mirrored
// FROM it: that verify is the strict one, as it always was.
const verifySrc =
- state.kind === "real-dir" ? dataDir : (parked[0] ?? null);
+ state.kind === "real-dir" ? mediaLink : (parked[0] ?? null);
if (verifySrc) {
log("Verifying the copy…");
verifyRetried ||= (
@@ -868,26 +999,29 @@ async function moveOut(args: {
dest: target,
log,
signal,
- ...(verifySrc === dataDir
- ? { mirror: { live: dataDir } }
+ ...(verifySrc === mediaLink
+ ? { mirror: { live: mediaLink } }
: {}),
})
).retried;
}
if (state.kind === "real-dir") {
- // Same device, so the rename is atomic: `data/` is a real dir one instant
- // and the parked copy the next, never half of each. A fresh name even
- // when a parked dir already exists — renaming onto a non-empty directory
- // is ENOTEMPTY, and the reclaim sweep takes all of them anyway.
- await rename(dataDir, path.join(channelDir, `data.relocated-${Date.now()}`));
+ // Same device, so the rename is atomic: `media/` is a real dir one
+ // instant and the parked copy the next, never half of each. A fresh name
+ // even when a parked dir already exists — renaming onto a non-empty
+ // directory is ENOTEMPTY, and the reclaim sweep takes all of them anyway.
+ // Between this rename and the symlink below every per-file link in
+ // `data/<id>/` dangles for one syscall; the marker stands, so every media
+ // guard reads the channel as in transition and nothing opens one.
+ await rename(mediaLink, path.join(channelDir, `media.relocated-${Date.now()}`));
} else if (state.kind === "other") {
throw new Error(
- `${dataDir} is neither a directory nor a symlink — refusing to replace it`,
+ `${mediaLink} is neither a directory nor a symlink — refusing to replace it`,
);
}
- const after = await linkOrDirState(dataDir);
+ const after = await linkOrDirState(mediaLink);
if (
after.kind === "link" &&
path.resolve(after.linkTarget) !== path.resolve(target)
@@ -895,14 +1029,14 @@ async function moveOut(args: {
// A link to somewhere else is not this move's work to reinterpret, and
// silently repointing it would strand whatever it does point at.
throw new Error(
- `${dataDir} already points at ${after.linkTarget}, not ${target}`,
+ `${mediaLink} already points at ${after.linkTarget}, not ${target}`,
);
}
if (after.kind === "missing") {
- await symlink(target, dataDir);
+ await symlink(target, mediaLink);
}
- // Written only now, on success: config.dataDir is a record of what is on
+ // Written only now, on success: config.mediaDir is a record of what is on
// disk, never an intention. Skipped when it already says so, so a rerun
// does not rewrite a file it agrees with.
// No readable config.json (it vanished mid-move, before the read or
@@ -911,18 +1045,19 @@ async function moveOut(args: {
// nothing.
const fresh = await readChannelConfig(paths, slug);
const patched =
- fresh && fresh.dataDir?.trim() !== target
- ? await patchChannelConfig(paths, slug, { dataDir: target })
+ fresh && fresh.mediaDir?.trim() !== target
+ ? await patchChannelConfig(paths, slug, { mediaDir: target })
: fresh;
if (!patched) {
- await writeChannelConfig(paths, slug, { ...args.config, dataDir: target });
+ await writeChannelConfig(paths, slug, { ...args.config, mediaDir: target });
}
- log(`Swapped: ${dataDir} -> ${target}`);
+ log(`Swapped: ${mediaLink} -> ${target}`);
await writeMarker(paths, slug, {
target,
direction: "out",
startedAt: new Date().toISOString(),
phase: "reclaim",
+ scope: "media",
});
}
@@ -945,6 +1080,7 @@ async function moveOut(args: {
target,
bytes: measured.bytes,
files: measured.files,
+ tiered,
resumed: args.resumed,
retried: verifyRetried,
reconciled,
@@ -957,7 +1093,7 @@ async function moveBack(args: {
io: { read: () => SiteSettings };
config: NonNullable<Awaited<ReturnType<typeof readChannelConfig>>>;
channelDir: string;
- dataDir: string;
+ mediaLink: string;
target: string;
log: (m: string) => void;
onProgress?: (p: RelocationProgress) => void;
@@ -967,21 +1103,21 @@ async function moveBack(args: {
writers: (slug: string) => ChannelWriter[];
reconcile: boolean;
}): Promise<RelocateChannelMediaResult> {
- const { paths, slug, channelDir, dataDir, target, log, signal } = args;
- const incoming = path.join(channelDir, "data.incoming");
+ const { paths, slug, channelDir, mediaLink, target, log, signal } = args;
+ const incoming = path.join(channelDir, "media.incoming");
// THE MIRROR OF moveOut's "must be in-place", and it is not symmetry for its
// own sake: move back is the only direction that ENDS by deleting the target.
//
// `relocated` is true for `inconsistent` and `unreachable` as well as `ok` —
- // config.dataDir is set in all three — so without this the UI offers Move back
- // for a channel whose config records a target while `data/` is a REAL
+ // config.mediaDir is set in all three — so without this the UI offers Move
+ // back for a channel whose config records a target while `media/` is a REAL
// directory. That state is not hypothetical: it is what `rsync --copy-links`
// of a channel produces, which WORKTREES.md documents as the way to carry
- // media into a shard. The run then copies the target to `data.incoming`,
- // verifies it, finds `data/` already a real dir, logs "the swap had
+ // media into a shard. The run then copies the target to `media.incoming`,
+ // verifies it, finds `media/` already a real dir, logs "the swap had
// completed", clears the config, `rm -r`s the target and finally sweeps
- // `data.incoming` — three copies in, zero out.
+ // `media.incoming` — three copies in, zero out.
//
// `in-transition` is allowed because a marker is what a resume carries, and a
// rerun is the caller this precondition must not refuse.
@@ -1015,7 +1151,7 @@ async function moveBack(args: {
? await measureTree(target)
: (await isDirectory(incoming))
? await measureTree(incoming)
- : await measureTree(dataDir);
+ : await measureTree(mediaLink);
log(
`Moving ${slug} back in place: ${measured.files} file(s), ` +
`${formatBytes(measured.bytes)} <- ${target}`,
@@ -1026,7 +1162,7 @@ async function moveBack(args: {
// THE SAME BAR MOVE-OUT USES, and for the same reason: landing the media
// with nothing to spare puts the corpus volume under the disk gate's floor
// the moment it arrives. And a resumed move-back must only be charged for
- // what is still MISSING — the bytes already sitting in `data.incoming` are
+ // what is still MISSING — the bytes already sitting in `media.incoming` are
// not about to be written twice, and counting them refused reruns on a disk
// that had room for the remainder.
const settings = args.io.read();
@@ -1047,14 +1183,14 @@ async function moveBack(args: {
);
}
// THE GUARD GOES ON THE LOCATION ROOT, NOT ON `incoming`. `incoming` is
- // `channels/<slug>/data.incoming`, a corpus directory a move-back is
+ // `channels/<slug>/media.incoming`, a corpus directory a move-back is
// entitled to create. What must be present is the SOURCE side: the
// `isDirectory(target)` precondition above covers existence, and this
// covers IDENTITY — an empty mountpoint directory with the platter
// unplugged is a directory, and copying it back would report a successful
// move of zero bytes and then delete the target.
await assertRelocationRootPresent(
- rootOfRelocatedDataDir(target, slug),
+ rootOfRelocatedMediaDir(target, slug),
settings.storage,
paths,
);
@@ -1064,6 +1200,7 @@ async function moveBack(args: {
direction: "back",
startedAt: new Date().toISOString(),
phase: "copy",
+ scope: "media",
});
await assertNoWritersUnderMarker({
paths,
@@ -1071,21 +1208,21 @@ async function moveBack(args: {
resumed: args.resumed,
writers: args.writers,
});
- // The copy under construction is `data.incoming`; the target on the other
+ // The copy under construction is `media.incoming`; the target on the other
// drive is the source, and is never the target of the mirror's --delete.
- // `live` is `data/` — the link, which resolves to that target — so a call
+ // `live` is `media` — the link, which resolves to that target — so a call
// with source and destination swapped is refused before rsync runs.
verifyRetried ||= (
await copyMirrorVerify({
rsyncBin: paths.rsyncBin,
src: target,
dest: incoming,
- live: dataDir,
+ live: mediaLink,
log,
progress: makeProgressSink({
totalBytes: measured.bytes,
// The same figure the space check above is priced in — what is
- // already in `data.incoming` from an interrupted run.
+ // already in `media.incoming` from an interrupted run.
alreadyBytes: already,
log,
onProgress: args.onProgress,
@@ -1109,49 +1246,55 @@ async function moveBack(args: {
direction: "back",
startedAt: new Date().toISOString(),
phase: "swap",
+ scope: "media",
});
// OBSERVE, DO NOT ASSUME. The old sequence was `unlink(data)` (swallowing
// its error) then `rename(incoming, data)`, which is idempotent in exactly
// the case that never happens: a crash AFTER the rename left `data/` a real
// directory, the swallowed unlink then failed on it, and the rename ENOENTed
- // on an `incoming` that no longer existed — forever, on every rerun.
- const state = await linkOrDirState(dataDir);
+ // on an `incoming` that no longer existed — forever, on every rerun. The
+ // same holds for `media` since release 17.
+ const state = await linkOrDirState(mediaLink);
if (state.kind === "real-dir") {
// The rename already committed. Nothing to swap; the leftovers are the
// reclaim's business.
- log(`${dataDir} is already a real directory — the swap had completed`);
+ log(`${mediaLink} is already a real directory — the swap had completed`);
} else if (state.kind === "other") {
- // A regular file (or a socket, or a fifo) where `data/` should be is not
+ // A regular file (or a socket, or a fifo) where `media/` should be is not
// a link to replace and not a directory to keep. moveOut refuses the same
// shape at its own swap; refusing here too is what keeps `unlink` below
// meaning "remove the symlink" and nothing else.
throw new Error(
- `${dataDir} is neither a directory nor a symlink — refusing to replace it`,
+ `${mediaLink} is neither a directory nor a symlink — refusing to replace it`,
);
} else {
if (!(await isDirectory(incoming))) {
throw new Error(
- `Cannot finish moving "${slug}" back: ${dataDir} is not a directory ` +
+ `Cannot finish moving "${slug}" back: ${mediaLink} is not a directory ` +
`and there is no verified copy at ${incoming}`,
);
}
- // unlink, not rm -r: `data` is the LINK here, and removing it recursively
- // would be the one way this whole design eats the media.
- if (state.kind !== "missing") await unlink(dataDir);
- await rename(incoming, dataDir);
- log(`Swapped: ${dataDir} is a real directory again`);
+ // unlink, not rm -r: `media` is the LINK here, and removing it
+ // recursively would be the one way this whole design eats the media.
+ // Every per-file link in `data/<id>/` is relative to `media`, so the
+ // rename below makes each of them resolve on the corpus disk — not one
+ // of them is rewritten (no "untier").
+ if (state.kind !== "missing") await unlink(mediaLink);
+ await rename(incoming, mediaLink);
+ log(`Swapped: ${mediaLink} is a real directory again`);
}
const fresh = await readChannelConfig(paths, slug);
- if (fresh?.dataDir !== undefined) {
- await patchChannelConfig(paths, slug, {}, { unset: ["dataDir"] });
+ if (fresh?.mediaDir !== undefined) {
+ await patchChannelConfig(paths, slug, {}, { unset: ["mediaDir"] });
}
await writeMarker(paths, slug, {
target,
direction: "back",
startedAt: new Date().toISOString(),
phase: "reclaim",
+ scope: "media",
});
}
@@ -1161,7 +1304,7 @@ async function moveBack(args: {
if ((await readdir(slugRoot).catch(() => ["keep"])).length === 0) {
await rm(slugRoot, { recursive: true, force: true });
}
- // Any half-copied `data.incoming` (or a parked dir from an earlier move out)
+ // Any half-copied `media.incoming` (or a parked dir from an earlier move out)
// goes with it — the same sweep, for the same reason.
await sweepParked(channelDir, log);
await clearMarker(paths, slug);
@@ -1172,6 +1315,7 @@ async function moveBack(args: {
target,
bytes: measured.bytes,
files: measured.files,
+ tiered: 0,
resumed: args.resumed,
retried: verifyRetried,
reconciled,
diff --git a/common/controller/relocateDir.ts b/common/controller/relocateDir.ts
@@ -27,8 +27,8 @@ import type {
//
// This is `relocateChannelMedia.ts`'s core, lifted out unchanged so a SECOND
// thing can be moved by it: the saved-video store. What stayed behind in that
-// file is everything specific to a channel — `config.dataDir`, the parked
-// `data.relocated-*` siblings, the social-channel refusal, `inspectChannelMedia`
+// file is everything specific to a channel — `config.mediaDir`, the parked
+// `media.relocated-*` siblings, the social-channel refusal, `inspectChannelMedia`
// — because none of it generalises and pretending it did would be the worse
// abstraction.
//
@@ -160,6 +160,12 @@ export async function readDirMarker(
r.phase === "swap" || r.phase === "reclaim"
? r.phase
: ("copy" as RelocationPhase),
+ // Optional (release 17): what a channel's marker is moving. Kept so a
+ // resume rewrites what it read, and so the mover can tell a tier
+ // migration's marker from its own.
+ ...(r.scope === "media" || r.scope === "tier-migration"
+ ? { scope: r.scope }
+ : {}),
};
} catch {
return null;
@@ -458,11 +464,11 @@ export class MirrorDirectionError extends Error {
// THE ONE RULE `--delete` LIVES UNDER, asserted before rsync is spawned: the
// destination is not the live media. `live` is supplied by the caller from
-// what it KNOWS is live — a channel's `channels/<slug>/data` (resolved through
+// what it KNOWS is live — a channel's `channels/<slug>/media` (resolved through
// its link, so on the way back it is the relocated target) or the saved-video
// store — and never derived from `src` or `dest`, so a call with the two
-// swapped is caught: out, the destination would be `data/` itself; back, it
-// would be the target `data/` points at. Refused when the destination's real
+// swapped is caught: out, the destination would be `media/` itself; back, it
+// would be the target `media` points at. Refused when the destination's real
// path is the live media's, contains it, or sits inside it; and, belt and
// braces, when the source and the destination contain one another.
export async function assertMirrorDirection(opts: {
diff --git a/common/controller/renameChannel.test.ts b/common/controller/renameChannel.test.ts
@@ -33,15 +33,9 @@ import {
import { renameChannel } from "./renameChannel";
import {
inspectChannelMedia,
- relocatedDataDir,
RELOCATION_MARKER_FILENAME,
} from "../lib/channelMedia";
-
-// RELEASE 17 SLICE T1 made a channel whose `data/` is a link (or whose config
-// carries `dataDir`) `legacy`; these cases still build that retired layout and
-// expect it to read `ok`. Slice T2 rebases them on `media/` and un-skips them.
-const T1_SKIP = "release 17 T2 rebases the mover on media/";
-
+import { relocatedMediaDir, tierLinkTarget } from "../lib/mediaTier-server";
// Run with:
// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/renameChannel.test.ts
@@ -156,29 +150,38 @@ test("renameChannel rejects invalid, same, and existing targets", async () => {
// --- relocated media ------------------------------------------------------
-test("rename re-points a convention-shaped relocated media dir", { skip: T1_SKIP }, async () => {
+test("rename re-points a convention-shaped relocated media dir", async () => {
await withPaths(async (paths) => {
const dir = path.dirname(paths.channelsDir);
const mediaRoot = path.join(dir, "platter");
- const target = relocatedDataDir(mediaRoot, "old");
- await writeChannelConfig(paths, "old", { ...config, dataDir: target });
+ const target = relocatedMediaDir(mediaRoot, "old");
+ await writeChannelConfig(paths, "old", { ...config, mediaDir: target });
await mkdir(path.join(target, "vid1"), { recursive: true });
await writeFile(path.join(target, "vid1", "audio.m4a"), "BYTES");
- await symlink(target, path.join(paths.channelsDir, "old", "data"));
+ await symlink(target, path.join(paths.channelsDir, "old", "media"));
+ // The tiered file's RELATIVE link in the real `data/` on the corpus disk.
+ const videoDir = path.join(paths.channelsDir, "old", "data", "vid1");
+ await mkdir(videoDir, { recursive: true });
+ await symlink(tierLinkTarget("vid1", "audio.m4a"), path.join(videoDir, "audio.m4a"));
const result = await renameChannel(paths, "old", "new", {
...config,
- dataDir: target,
+ mediaDir: target,
});
assert.deepEqual(result.warnings, []);
- const newTarget = relocatedDataDir(mediaRoot, "new");
- assert.equal((await readChannelConfig(paths, "new"))?.dataDir, newTarget);
+ const newTarget = relocatedMediaDir(mediaRoot, "new");
+ assert.equal((await readChannelConfig(paths, "new"))?.mediaDir, newTarget);
assert.equal(
- await readlink(path.join(paths.channelsDir, "new", "data")),
+ await readlink(path.join(paths.channelsDir, "new", "media")),
newTarget,
);
- // The media reads through the new link at the old on-disk contract path.
+ // The per-file link is untouched — relative, it moved with the channel —
+ // and the media reads through it at the on-disk contract path.
+ assert.equal(
+ await readlink(path.join(paths.channelsDir, "new", "data", "vid1", "audio.m4a")),
+ tierLinkTarget("vid1", "audio.m4a"),
+ );
assert.equal(
await readFile(
path.join(paths.channelsDir, "new", "data", "vid1", "audio.m4a"),
@@ -198,19 +201,19 @@ test("rename re-points a convention-shaped relocated media dir", { skip: T1_SKIP
test("a media dir that does not follow the convention is left alone", async () => {
await withPaths(async (paths) => {
const dir = path.dirname(paths.channelsDir);
- // <root>/<something-else>/data — the link is absolute and still works after
- // the rename, so moving a directory whose name is not ours would be worse
- // than leaving it.
- const target = path.join(dir, "platter", "handpicked", "data");
- await writeChannelConfig(paths, "old", { ...config, dataDir: target });
+ // <root>/<something-else>/media — the link is absolute and still works
+ // after the rename, so moving a directory whose name is not ours would be
+ // worse than leaving it.
+ const target = path.join(dir, "platter", "handpicked", "media");
+ await writeChannelConfig(paths, "old", { ...config, mediaDir: target });
await mkdir(path.join(target, "vid1"), { recursive: true });
- await symlink(target, path.join(paths.channelsDir, "old", "data"));
+ await symlink(target, path.join(paths.channelsDir, "old", "media"));
- await renameChannel(paths, "old", "new", { ...config, dataDir: target });
+ await renameChannel(paths, "old", "new", { ...config, mediaDir: target });
- assert.equal((await readChannelConfig(paths, "new"))?.dataDir, target);
+ assert.equal((await readChannelConfig(paths, "new"))?.mediaDir, target);
assert.equal(
- await readlink(path.join(paths.channelsDir, "new", "data")),
+ await readlink(path.join(paths.channelsDir, "new", "media")),
target,
);
await stat(path.join(target, "vid1"));
@@ -221,15 +224,15 @@ test("rename rolls the channel dir back when the media move fails", async () =>
await withPaths(async (paths) => {
const dir = path.dirname(paths.channelsDir);
const mediaRoot = path.join(dir, "platter");
- const target = relocatedDataDir(mediaRoot, "old");
- await writeChannelConfig(paths, "old", { ...config, dataDir: target });
+ const target = relocatedMediaDir(mediaRoot, "old");
+ await writeChannelConfig(paths, "old", { ...config, mediaDir: target });
await mkdir(path.join(target, "vid1"), { recursive: true });
- await symlink(target, path.join(paths.channelsDir, "old", "data"));
+ await symlink(target, path.join(paths.channelsDir, "old", "media"));
// Something is already sitting at <root>/new.
await mkdir(path.join(mediaRoot, "new"), { recursive: true });
await assert.rejects(
- () => renameChannel(paths, "old", "new", { ...config, dataDir: target }),
+ () => renameChannel(paths, "old", "new", { ...config, mediaDir: target }),
/already exists/,
);
// Nothing half-renamed: the channel is still "old", still linked, still
@@ -237,7 +240,7 @@ test("rename rolls the channel dir back when the media move fails", async () =>
assert.equal(await channelExists(paths, "old"), true);
assert.equal(await channelExists(paths, "new"), false);
assert.equal(
- await readlink(path.join(paths.channelsDir, "old", "data")),
+ await readlink(path.join(paths.channelsDir, "old", "media")),
target,
);
await stat(path.join(target, "vid1"));
@@ -250,7 +253,7 @@ test("rename is refused while a relocation is in flight", async () => {
await writeFile(
path.join(paths.channelsDir, "old", RELOCATION_MARKER_FILENAME),
JSON.stringify({
- target: "/mnt/platter/old/data",
+ target: "/mnt/platter/old/media",
direction: "out",
startedAt: new Date().toISOString(),
phase: "copy",
diff --git a/common/controller/renameChannel.ts b/common/controller/renameChannel.ts
@@ -9,10 +9,8 @@ import {
writeChannelConfig,
} from "./channels";
import { savedVideoRoot } from "../lib/savedVideo";
-import {
- readRelocationMarker,
- relocatedDataDir,
-} from "../lib/channelMedia";
+import { readRelocationMarker } from "../lib/channelMedia";
+import { MEDIA_LINK_NAME, relocatedMediaDir } from "../lib/mediaTier-server";
import { rewriteSavedVideoDir } from "../lib/savedVideo-server";
import { getSite, listSiteIds, writeSite } from "../lib/site";
import {
@@ -25,7 +23,9 @@ import {
// every other store that keys by slug and would otherwise be orphaned:
// - the saved-video store dir + each saved-video.json pointer's absolute `dir`
// - a relocated media dir on another drive, when it follows the
-// <root>/<slug>/data convention, plus the symlink and config.dataDir
+// <root>/<slug>/media convention, plus the `media` symlink and
+// config.mediaDir (the per-file links in `data/<id>/` are RELATIVE to
+// `media`, so they move with the channel dir and need nothing)
// - site.json memberships across all sites
// - the sync scheduler's per-channel backoff state
//
@@ -122,20 +122,21 @@ export async function renameChannel(
}
}
- // 3. Move the relocated media dir when it follows the <root>/<slug>/data
- // convention, and re-point the symlink at it. The link is ABSOLUTE, so a
+ // 3. Move the relocated media dir when it follows the <root>/<slug>/media
+ // convention, and re-point the `media` symlink at it. The link is ABSOLUTE, so a
// target that does NOT follow the convention is deliberately left alone —
// it still works, and moving someone else's directory because its name
// happened to match would be worse than leaving it. Rolls the channel-dir
// (and store) move back on failure, same shape as step 2.
- const relocated = config.dataDir?.trim();
+ const relocated = config.mediaDir?.trim();
const conventional =
- relocated && relocated === relocatedDataDir(path.dirname(path.dirname(relocated)), oldSlug)
+ relocated && relocated === relocatedMediaDir(path.dirname(path.dirname(relocated)), oldSlug)
? relocated
: null;
if (conventional) {
const mediaRoot = path.dirname(path.dirname(conventional));
- const newTarget = relocatedDataDir(mediaRoot, newSlug);
+ const newTarget = relocatedMediaDir(mediaRoot, newSlug);
+ const link = path.join(newChannelDir, MEDIA_LINK_NAME);
// A ROLLBACK MUST ONLY UNDO WHAT ACTUALLY RAN. The pre-check below fails
// BECAUSE something unrelated already occupies <root>/<newSlug> — and the
// old catch then renamed that stranger to <root>/<oldSlug>, destroying a
@@ -145,7 +146,7 @@ export async function renameChannel(
let movedMedia = false;
// THE UNLINK IS A STEP TOO. It was untracked, so a throw from the symlink
// below (EACCES on a read-only channel dir, ENOSPC) rolled the media
- // directory back while `data/` stayed DELETED — config still naming the old
+ // directory back while the link stayed DELETED — config still naming the old
// target, nothing on disk pointing at it: `inconsistent`, which is now a
// state move-back refuses. The catch replays only what ran, and removing the
// link ran.
@@ -157,33 +158,32 @@ export async function renameChannel(
}
await rename(path.dirname(conventional), path.dirname(newTarget));
movedMedia = true;
- await unlink(path.join(newChannelDir, "data"))
+ await unlink(link)
.then(() => {
unlinked = true;
})
.catch(() => {});
- await symlink(newTarget, path.join(newChannelDir, "data"));
+ await symlink(newTarget, link);
relinked = true;
// Re-read: the channel dir has already moved, so this is the file that
// will actually be on disk afterwards.
const patched = await patchChannelConfig(paths, newSlug, {
- dataDir: newTarget,
+ mediaDir: newTarget,
});
if (!patched) {
// No readable config.json at the new slug: write the one this rename
- // started from, so the moved data is not left unrecorded.
- await writeChannelConfig(paths, newSlug, { ...config, dataDir: newTarget });
+ // started from, so the moved media is not left unrecorded.
+ await writeChannelConfig(paths, newSlug, { ...config, mediaDir: newTarget });
}
} catch (err) {
// The link goes back too, and before the directory under it moves: a
- // failure between the symlink and the config write left `data/` pointing
- // at <root>/<newSlug>/data while everything else was rolled back to the
- // old slug — a dangling link, which reads as an unmounted drive.
+ // failure between the symlink and the config write left `media`
+ // pointing at <root>/<newSlug>/media while everything else was rolled
+ // back to the old slug — a dangling link, which reads as an unmounted
+ // drive.
if (relinked || unlinked) {
- await unlink(path.join(newChannelDir, "data")).catch(() => {});
- await symlink(conventional, path.join(newChannelDir, "data")).catch(
- () => {},
- );
+ await unlink(link).catch(() => {});
+ await symlink(conventional, link).catch(() => {});
}
if (movedMedia) {
await rename(path.dirname(newTarget), path.dirname(conventional)).catch(
diff --git a/common/controller/storageLocations.test.ts b/common/controller/storageLocations.test.ts
@@ -28,11 +28,6 @@ import {
} from "./storageLocations";
import { readChannelConfig } from "./channels";
-// RELEASE 17 SLICE T1 made a channel whose `data/` is a link (or whose config
-// carries `dataDir`) `legacy`; these cases still build that retired layout and
-// expect it to read `ok`. Slice T2 rebases them on `media/` and un-skips them.
-const T1_SKIP = "release 17 T2 rebases the mover on media/";
-
// Run with:
// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/storageLocations.test.ts
@@ -109,35 +104,40 @@ async function withTmp(fn: (h: Harness) => Promise<void>): Promise<void> {
}
}
-// A channel whose media lives under `root`: a real dir on the "drive", an
-// absolute symlink at channels/<slug>/data, and config.dataDir naming it —
-// exactly what a finished relocation leaves behind.
+// A channel whose media tier lives under `root` (release 17): a real dir on the
+// "drive" holding the big file, an absolute symlink at channels/<slug>/media,
+// config.mediaDir naming it, and a real `data/` on the corpus disk whose audio
+// is the relative link into `media/` — exactly what a finished relocation
+// leaves behind.
async function seedRelocated(
h: Harness,
slug: string,
root: string,
opts: { link?: boolean; media?: boolean } = {},
): Promise<string> {
- const target = path.join(root, slug, "data");
+ const target = path.join(root, slug, "media");
if (opts.media !== false) {
await mkdir(path.join(target, "20240101_aaaaaaaaaaa"), { recursive: true });
- await writeFile(
- path.join(target, "20240101_aaaaaaaaaaa", "transcript.en.vtt"),
- "WEBVTT\n",
- );
+ await writeFile(path.join(target, "20240101_aaaaaaaaaaa", "audio.mp3"), "ID3");
}
const channelDir = path.join(h.paths.channelsDir, slug);
- await mkdir(channelDir, { recursive: true });
+ const videoDir = path.join(channelDir, "data", "20240101_aaaaaaaaaaa");
+ await mkdir(videoDir, { recursive: true });
+ await writeFile(path.join(videoDir, "transcript.en.vtt"), "WEBVTT\n");
+ await symlink(
+ path.join("..", "..", "media", "20240101_aaaaaaaaaaa", "audio.mp3"),
+ path.join(videoDir, "audio.mp3"),
+ );
await writeFile(
path.join(channelDir, "config.json"),
JSON.stringify(
- { handling: "transcribe", url: `https://example.com/${slug}`, dataDir: target },
+ { handling: "transcribe", url: `https://example.com/${slug}`, mediaDir: target },
null,
2,
) + "\n",
);
if (opts.link !== false) {
- await symlink(target, path.join(channelDir, "data"));
+ await symlink(target, path.join(channelDir, "media"));
}
return target;
}
@@ -151,7 +151,7 @@ function loc(id: string, root: string, extra: Partial<StorageLocation> = {}): St
return { id, label: id, root, autoRepoint: false, ...extra };
}
-test("channelsOnLocation buckets ok / unreachable / moving and ignores channels elsewhere", { skip: T1_SKIP }, async () => {
+test("channelsOnLocation buckets ok / unreachable / moving and ignores channels elsewhere", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
// Media dir never created: the link dangles, which is what an unmounted
@@ -160,9 +160,9 @@ test("channelsOnLocation buckets ok / unreachable / moving and ignores channels
await seedRelocated(h, "gamma", h.rootA);
await writeFile(
path.join(h.paths.channelsDir, "gamma", ".relocating.json"),
- JSON.stringify({ target: path.join(h.rootA, "gamma", "data"), phase: "copy" }),
+ JSON.stringify({ target: path.join(h.rootA, "gamma", "media"), phase: "copy" }),
);
- // On the other drive, and one plain in-place channel with no dataDir.
+ // On the other drive, and one plain in-place channel with no mediaDir.
await seedRelocated(h, "delta", h.rootB);
await mkdir(path.join(h.paths.channelsDir, "plain", "data"), {
recursive: true,
@@ -183,16 +183,64 @@ test("channelsOnLocation buckets ok / unreachable / moving and ignores channels
assert.equal(rollups.a.moving, 1);
assert.deepEqual(rollups.b.slugs, ["delta"]);
assert.equal(rollups.b.ok, 1);
+ assert.equal(rollups.a.legacy, 0);
+ });
+});
+
+// THE RETIRED LAYOUT (release 17): a channel whose whole `data/` was moved
+// before the media tier is still ON its location (by its retired `dataDir`),
+// counted unreachable — its text and media are held until migrate-tier — and
+// named in `legacy`, the "(n to migrate)" beside it. Byte figures: the media
+// tier on the row, the corpus-disk text and clips beside it, an old snapshot's
+// missing text figure counted unknown.
+test("channelsOnLocation counts a legacy channel as unreachable and to migrate, and sums the tiers", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "alpha", h.rootA);
+ const oldTarget = path.join(h.rootA, "old", "data");
+ await mkdir(path.join(oldTarget, "v1"), { recursive: true });
+ const oldDir = path.join(h.paths.channelsDir, "old");
+ await mkdir(oldDir, { recursive: true });
+ await writeFile(
+ path.join(oldDir, "config.json"),
+ JSON.stringify({ handling: "transcribe", url: "https://x/o", dataDir: oldTarget }) + "\n",
+ );
+ await symlink(oldTarget, path.join(oldDir, "data"));
+ await mkdir(path.join(h.paths.channelsDir, "plain", "data"), { recursive: true });
+ await writeFile(
+ path.join(h.paths.channelsDir, "plain", "config.json"),
+ JSON.stringify({ handling: "transcribe", url: "https://x/y" }) + "\n",
+ );
+
+ const rollups = await channelsOnLocation({
+ paths: h.paths,
+ locations: [loc("a", h.rootA)],
+ includeInternal: true,
+ mediaBytes: { alpha: 100, old: 50, plain: 7 },
+ clipsBytes: { alpha: 3, old: 0, plain: 2 },
+ textBytes: { alpha: 10, plain: 5 },
+ });
+ assert.deepEqual(rollups.a.slugs, ["alpha", "old"]);
+ assert.equal(rollups.a.ok, 1);
+ assert.equal(rollups.a.unreachable, 1);
+ assert.equal(rollups.a.legacy, 1);
+ assert.equal(rollups.a.bytes, 150);
+ assert.equal(rollups.a.clipsBytes, 3);
+ assert.equal(rollups.a.textBytes, 10);
+ assert.equal(rollups.a.unknownTextBytes, 1, "the old snapshot has no text figure");
+ assert.deepEqual(rollups.internal.slugs, ["plain"]);
+ assert.equal(rollups.internal.bytes, 7);
+ assert.equal(rollups.internal.textBytes, 5);
+ assert.equal(rollups.internal.clipsBytes, 2);
});
});
-test("re-point rewrites both channels' links and configs, then the location", { skip: T1_SKIP }, async () => {
+test("re-point rewrites both channels' links and configs, then the location", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
await seedRelocated(h, "beta", h.rootA);
// The media is already on drive B — the disk moved, the bytes did not.
- await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
- await mkdir(path.join(h.rootB, "beta", "data"), { recursive: true });
+ await mkdir(path.join(h.rootB, "alpha", "media"), { recursive: true });
+ await mkdir(path.join(h.rootB, "beta", "media"), { recursive: true });
await setLocations(h, [
loc("cold", h.rootA, {
volume: {
@@ -215,12 +263,12 @@ test("re-point rewrites both channels' links and configs, then the location", {
assert.deepEqual(result.channels, ["alpha", "beta"]);
for (const slug of ["alpha", "beta"]) {
- const target = path.join(h.rootB, slug, "data");
+ const target = path.join(h.rootB, slug, "media");
assert.equal(
- await readlink(path.join(h.paths.channelsDir, slug, "data")),
+ await readlink(path.join(h.paths.channelsDir, slug, "media")),
target,
);
- assert.equal((await readChannelConfig(h.paths, slug))?.dataDir, target);
+ assert.equal((await readChannelConfig(h.paths, slug))?.mediaDir, target);
}
// The settings write is last and carries the new root; the identity is
// re-anchored so root === join(mountpoint, relPath) still holds.
@@ -233,11 +281,11 @@ test("re-point rewrites both channels' links and configs, then the location", {
});
});
-test("re-point refuses a target that has no media for a channel, naming the slug", { skip: T1_SKIP }, async () => {
+test("re-point refuses a target that has no media for a channel, naming the slug", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
await seedRelocated(h, "beta", h.rootA);
- await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
+ await mkdir(path.join(h.rootB, "alpha", "media"), { recursive: true });
await setLocations(h, [loc("cold", h.rootA)]);
const pre = await preflightRepoint({
@@ -262,19 +310,19 @@ test("re-point refuses a target that has no media for a channel, naming the slug
);
// Nothing was written: not the link that COULD have moved, not settings.
assert.equal(
- await readlink(path.join(h.paths.channelsDir, "alpha", "data")),
- path.join(h.rootA, "alpha", "data"),
+ await readlink(path.join(h.paths.channelsDir, "alpha", "media")),
+ path.join(h.rootA, "alpha", "media"),
);
assert.equal(h.writes.length, 0);
});
});
-test("a failure on the second channel rolls the first one back", { skip: T1_SKIP }, async () => {
+test("a failure on the second channel rolls the first one back", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
await seedRelocated(h, "beta", h.rootA);
- await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
- await mkdir(path.join(h.rootB, "beta", "data"), { recursive: true });
+ await mkdir(path.join(h.rootB, "alpha", "media"), { recursive: true });
+ await mkdir(path.join(h.rootB, "beta", "media"), { recursive: true });
await setLocations(h, [loc("cold", h.rootA)]);
// A READ-ONLY CHANNEL DIR is the cheapest real failure: the unlink of
@@ -299,12 +347,12 @@ test("a failure on the second channel rolls the first one back", { skip: T1_SKIP
// alpha is back where it started — link AND config, and it reads as the
// pre-job `unreachable`, never `inconsistent`.
assert.equal(
- await readlink(path.join(h.paths.channelsDir, "alpha", "data")),
- path.join(h.rootA, "alpha", "data"),
+ await readlink(path.join(h.paths.channelsDir, "alpha", "media")),
+ path.join(h.rootA, "alpha", "media"),
);
assert.equal(
- (await readChannelConfig(h.paths, "alpha"))?.dataDir,
- path.join(h.rootA, "alpha", "data"),
+ (await readChannelConfig(h.paths, "alpha"))?.mediaDir,
+ path.join(h.rootA, "alpha", "media"),
);
// And the location never moved.
assert.equal(h.settings().storage.locations[0].root, h.rootA);
@@ -312,10 +360,10 @@ test("a failure on the second channel rolls the first one back", { skip: T1_SKIP
});
});
-test("re-point refuses a busy channel and names it", { skip: T1_SKIP }, async () => {
+test("re-point refuses a busy channel and names it", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
- await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
+ await mkdir(path.join(h.rootB, "alpha", "media"), { recursive: true });
await setLocations(h, [loc("cold", h.rootA)]);
const pre = await preflightRepoint({
@@ -330,25 +378,25 @@ test("re-point refuses a busy channel and names it", { skip: T1_SKIP }, async ()
});
});
-test("a rerun after a crash finishes the channels that were left", { skip: T1_SKIP }, async () => {
+test("a rerun after a crash finishes the channels that were left", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
await seedRelocated(h, "beta", h.rootA);
- await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
- await mkdir(path.join(h.rootB, "beta", "data"), { recursive: true });
+ await mkdir(path.join(h.rootB, "alpha", "media"), { recursive: true });
+ await mkdir(path.join(h.rootB, "beta", "media"), { recursive: true });
await setLocations(h, [loc("cold", h.rootA)]);
// A CRASH, not a rollback: alpha was re-pointed and the process died before
// beta and before the settings write. Rebuilt by hand, which is exactly the
// state on disk.
- const alphaLink = path.join(h.paths.channelsDir, "alpha", "data");
+ const alphaLink = path.join(h.paths.channelsDir, "alpha", "media");
await rm(alphaLink);
- await symlink(path.join(h.rootB, "alpha", "data"), alphaLink);
+ await symlink(path.join(h.rootB, "alpha", "media"), alphaLink);
const alphaConfig = await readChannelConfig(h.paths, "alpha");
await writeFile(
path.join(h.paths.channelsDir, "alpha", "config.json"),
JSON.stringify(
- { ...alphaConfig, dataDir: path.join(h.rootB, "alpha", "data") },
+ { ...alphaConfig, mediaDir: path.join(h.rootB, "alpha", "media") },
null,
2,
) + "\n",
@@ -363,8 +411,8 @@ test("a rerun after a crash finishes the channels that were left", { skip: T1_SK
// alpha is NOT on the old root any more, so the rerun does not touch it.
assert.deepEqual(result.channels, ["beta"]);
assert.equal(
- await readlink(path.join(h.paths.channelsDir, "beta", "data")),
- path.join(h.rootB, "beta", "data"),
+ await readlink(path.join(h.paths.channelsDir, "beta", "media")),
+ path.join(h.rootB, "beta", "media"),
);
assert.equal(h.settings().storage.locations[0].root, h.rootB);
@@ -381,12 +429,12 @@ test("a rerun after a crash finishes the channels that were left", { skip: T1_SK
});
});
-test("a channel killed between its symlink and its config write is resumed, not refused", { skip: T1_SKIP }, async () => {
+test("a channel killed between its symlink and its config write is resumed, not refused", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
await seedRelocated(h, "beta", h.rootA);
- await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
- await mkdir(path.join(h.rootB, "beta", "data"), { recursive: true });
+ await mkdir(path.join(h.rootB, "alpha", "media"), { recursive: true });
+ await mkdir(path.join(h.rootB, "beta", "media"), { recursive: true });
await setLocations(h, [loc("cold", h.rootA)]);
// THE ONE-INSTRUCTION WINDOW. alpha's link was moved and the process died
@@ -394,9 +442,9 @@ test("a channel killed between its symlink and its config write is resumed, not
// `inconsistent`, and alpha is still "on" the old root — so a preflight
// that only accepted ok/unreachable would refuse the whole location for
// ever, naming a state whose only remedy is the job it is refusing.
- const alphaLink = path.join(h.paths.channelsDir, "alpha", "data");
+ const alphaLink = path.join(h.paths.channelsDir, "alpha", "media");
await rm(alphaLink);
- await symlink(path.join(h.rootB, "alpha", "data"), alphaLink);
+ await symlink(path.join(h.rootB, "alpha", "media"), alphaLink);
const pre = await preflightRepoint({
paths: h.paths,
@@ -420,16 +468,16 @@ test("a channel killed between its symlink and its config write is resumed, not
// alpha's config caught up with its link (which was never touched again),
// beta went the ordinary way, and the location moved.
assert.equal(
- (await readChannelConfig(h.paths, "alpha"))?.dataDir,
- path.join(h.rootB, "alpha", "data"),
+ (await readChannelConfig(h.paths, "alpha"))?.mediaDir,
+ path.join(h.rootB, "alpha", "media"),
);
assert.equal(
- await readlink(path.join(h.paths.channelsDir, "alpha", "data")),
- path.join(h.rootB, "alpha", "data"),
+ await readlink(path.join(h.paths.channelsDir, "alpha", "media")),
+ path.join(h.rootB, "alpha", "media"),
);
assert.equal(
- (await readChannelConfig(h.paths, "beta"))?.dataDir,
- path.join(h.rootB, "beta", "data"),
+ (await readChannelConfig(h.paths, "beta"))?.mediaDir,
+ path.join(h.rootB, "beta", "media"),
);
assert.equal(h.settings().storage.locations[0].root, h.rootB);
assert.ok(lines.some((l) => l.includes("link was already moved")));
@@ -439,10 +487,10 @@ test("a channel killed between its symlink and its config write is resumed, not
test("preflight refuses an in-transition channel and a root that is not a directory", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
- await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
+ await mkdir(path.join(h.rootB, "alpha", "media"), { recursive: true });
await writeFile(
path.join(h.paths.channelsDir, "alpha", ".relocating.json"),
- JSON.stringify({ target: path.join(h.rootA, "alpha", "data"), phase: "copy" }),
+ JSON.stringify({ target: path.join(h.rootA, "alpha", "media"), phase: "copy" }),
);
await setLocations(h, [loc("cold", h.rootA)]);
diff --git a/common/controller/storageLocations.ts b/common/controller/storageLocations.ts
@@ -27,8 +27,8 @@ import {
import {
forgetChannelMedia,
inspectChannelMedia,
- relocatedDataDir,
} from "../lib/channelMedia";
+import { MEDIA_LINK_NAME, relocatedMediaDir } from "../lib/mediaTier-server";
import {
isDriveNotAnswering,
onDrive,
@@ -54,8 +54,9 @@ import { relocationRootProblem } from "./relocateChannelMedia";
// knows the corpus: which channels are on a location, whether a re-point is
// safe, and the re-point itself.
//
-// RE-POINT MOVES NO BYTES. It rewrites each channel's `data/` symlink and its
-// `config.dataDir`, then the location's `root`. That is the entire operation —
+// RE-POINT MOVES NO BYTES. It rewrites each channel's `media` symlink and its
+// `config.mediaDir`, then the location's `root` (release 17: a channel's media
+// tier is what lives on a location; its text never leaves the corpus disk). That is the entire operation —
// the media is already where it is going, because the DISK came up somewhere
// else and took it along. The relocate job (which does move bytes) is a
// different thing entirely, and the two share a queue key precisely so they can
@@ -95,10 +96,12 @@ const DEFAULT_SETTINGS_IO: SettingsIO = {
export type LocationRollup = {
locationId: string;
- // Every channel whose `config.dataDir` is under this location's root, sorted.
+ // Every channel whose `config.mediaDir` is under this location's root (or,
+ // on a `legacy` channel, its retired `config.dataDir`), sorted.
slugs: string[];
total: number;
- // Sum of `snapshot.totalMediaBytes` over the channels on this location, and
+ // Sum of `snapshot.totalMediaBytes` over the channels on this location — the
+ // MEDIA TIER's bytes, which is what a location holds (release 17) — and
// how many of them could not contribute one (no snapshot, or one written
// before the field existed). THE SECOND NUMBER IS WHY THE FIRST IS HONEST: a
// location whose channels have never had a report reads `0 bytes` otherwise,
@@ -106,16 +109,20 @@ export type LocationRollup = {
// surface renders "+ n unknown" beside the total.
bytes: number;
unknownBytes: number;
- // THE `clips/` SHARE OF `bytes`, not a sibling of it. Fetched clip windows
- // live under `data/<id>/clips/` and are counted into `totalMediaBytes` like
- // everything else there — this is the "of which", because clips are the one
- // part of a location's bytes that is a CACHE nothing prunes, and an operator
- // looking at a full drive wants to know how much of it is that.
+ // THE CORPUS-DISK BYTES of these channels: `clips/` (the fetched clip
+ // windows, a cache nothing prunes) and the text tier. A SIBLING of `bytes`
+ // since release 17, not a share of it — both stay on the corpus volume
+ // whatever location the media is on, so only the INTERNAL row adds them to
+ // what it holds; a location row shows them as "on the corpus volume".
//
// No `unknownClipsBytes`: it would be the same set of channels
- // `unknownBytes` already counts (the two fields are written by one snapshot
+ // `unknownBytes` already counts (the fields are written by one snapshot
// pass), and a second copy of one number is a second thing to keep in step.
+ // `textBytes` has its own unknown count: a snapshot written before release
+ // 17 carries clips and media but no text figure.
clipsBytes: number;
+ textBytes: number;
+ unknownTextBytes: number;
// `inspectChannelMedia` status, bucketed into the three numbers the page
// shows. `unreachable` DELIBERATELY ABSORBS `inconsistent`: both mean "this
// channel's media is not readable through its link right now", which is the
@@ -126,6 +133,10 @@ export type LocationRollup = {
// `in-transition`: a relocation marker is present. A location with any of
// these is one no re-point may touch.
moving: number;
+ // `legacy` — the retired whole-directory layout, waiting for
+ // `archilyzer storage migrate-tier`. Counted in `unreachable` too (its
+ // text and media are held); this is the "(n to migrate)" beside it.
+ legacy: number;
};
function emptyRollup(locationId: string): LocationRollup {
@@ -136,9 +147,12 @@ function emptyRollup(locationId: string): LocationRollup {
bytes: 0,
unknownBytes: 0,
clipsBytes: 0,
+ textBytes: 0,
+ unknownTextBytes: 0,
ok: 0,
unreachable: 0,
moving: 0,
+ legacy: 0,
};
}
@@ -153,8 +167,8 @@ function emptyRollup(locationId: string): LocationRollup {
// `settings.storage.locations` would make it deletable and would make
// `locationOfDataDir` match every unrelocated channel — which would break the
// one rule the whole design rests on, that a channel is on a location iff its
-// `dataDir` is under that location's root, and an in-place channel HAS no
-// `dataDir`.
+// `mediaDir` is under that location's root, and an in-place channel HAS no
+// `mediaDir`.
//
// So it is assembled where it is rendered, out of the same three facts every
// other row carries, and it is the one row with no actions.
@@ -168,7 +182,7 @@ export { INTERNAL_LOCATION_ID, INTERNAL_LOCATION_LABEL };
// page draws a row per location and the channel list is the same list for all
// of them. Cost is `listChannelConfigs` (one readdir + one config read per
// channel) plus `inspectChannelMedia` (two stats) for the channels that are
-// actually on a location — an unrelocated channel has no `dataDir` and is
+// actually on a location — an unrelocated channel has no `mediaDir` and is
// skipped before it costs a stat.
export async function channelsOnLocation(opts: {
paths: Paths;
@@ -183,12 +197,16 @@ export async function channelsOnLocation(opts: {
// file in one render. A slug that is absent (or maps to undefined) counts
// towards `unknownBytes`, never towards `bytes`.
mediaBytes?: Readonly<Record<string, number | undefined>>;
- // `snapshot.totalClipsBytes` per slug — the `clips/` share of the above.
+ // `snapshot.totalClipsBytes` per slug — the clip cache, on the corpus disk.
// Absent for a snapshot written before the field existed, which is the same
// set `mediaBytes` is absent for.
clipsBytes?: Readonly<Record<string, number | undefined>>;
+ // `snapshot.totalTextBytes` per slug — the text tier, on the corpus disk.
+ // Absent for a snapshot written before release 17: unknown, never 0.
+ textBytes?: Readonly<Record<string, number | undefined>>;
// The in-place row (see INTERNAL_LOCATION_ID). When true, every channel with
- // NO `dataDir` is rolled up under that id alongside the configured ones.
+ // NO `mediaDir` (and no retired `dataDir`) is rolled up under that id
+ // alongside the configured ones.
includeInternal?: boolean;
}): Promise<Record<string, LocationRollup>> {
const out: Record<string, LocationRollup> = {};
@@ -200,17 +218,19 @@ export async function channelsOnLocation(opts: {
const configs = opts.configs ?? (await listChannelConfigs(opts.paths));
for (const { slug, config } of configs) {
- const dataDir = config.dataDir?.trim();
- const loc = dataDir
- ? locationOfDataDir(dataDir, opts.locations as StorageLocation[])
+ // Where the channel's media is: `mediaDir`, or on a legacy channel the
+ // retired `dataDir` (its whole tree is still there until migrate-tier).
+ const mediaDir = config.mediaDir?.trim() || config.dataDir?.trim();
+ const loc = mediaDir
+ ? locationOfDataDir(mediaDir, opts.locations as StorageLocation[])
: null;
- // In place: no recorded dataDir at all. A dataDir under a root NOBODY named
+ // In place: no recorded target at all. A target under a root NOBODY named
// is neither in place nor on a location, and it is deliberately counted in
// neither — /storage says so by the totals not adding up to the corpus, and
// the remedy is to name that root as a location.
const id = loc
? loc.id
- : !dataDir && opts.includeInternal
+ : !mediaDir && opts.includeInternal
? INTERNAL_LOCATION_ID
: null;
if (!id) continue;
@@ -222,10 +242,16 @@ export async function channelsOnLocation(opts: {
else roll.unknownBytes += 1;
const clips = opts.clipsBytes?.[slug];
if (typeof clips === "number") roll.clipsBytes += clips;
+ const text = opts.textBytes?.[slug];
+ if (typeof text === "number") roll.textBytes += text;
+ else roll.unknownTextBytes += 1;
const media = await inspectChannelMedia(opts.paths, slug, config);
if (media.status === "ok" || media.status === "in-place") roll.ok += 1;
else if (media.status === "in-transition") roll.moving += 1;
- else roll.unreachable += 1;
+ else {
+ roll.unreachable += 1;
+ if (media.status === "legacy") roll.legacy += 1;
+ }
}
for (const roll of Object.values(out)) roll.slugs.sort();
return out;
@@ -449,16 +475,18 @@ export type RepointPreflight = {
newRoot: string;
};
-// Does `channels/<slug>/data` already point exactly where a re-point would put
-// it? lstat/readlink, never stat: the target may not exist yet either, and a
-// stat would call a perfectly good link missing.
+// Does `channels/<slug>/media` already point exactly where a re-point would
+// put it? lstat/readlink, never stat: the target may not exist yet either, and
+// a stat would call a perfectly good link missing.
async function linkAlreadyAt(
paths: Paths,
slug: string,
newTarget: string,
): Promise<boolean> {
try {
- const linkTarget = await readlink(path.join(paths.channelsDir, slug, "data"));
+ const linkTarget = await readlink(
+ path.join(paths.channelsDir, slug, MEDIA_LINK_NAME),
+ );
return path.resolve(linkTarget) === path.resolve(newTarget);
} catch {
return false;
@@ -583,7 +611,7 @@ export async function preflightRepoint(opts: {
const resumable = await linkAlreadyAt(
opts.paths,
slug,
- relocatedDataDir(newRoot, slug),
+ relocatedMediaDir(newRoot, slug),
);
if (!resumable && media.status !== "ok" && media.status !== "unreachable") {
base.problems.push(
@@ -598,7 +626,7 @@ export async function preflightRepoint(opts: {
base.problems.push(`${slug}: ${busy}`);
continue;
}
- const target = relocatedDataDir(newRoot, slug);
+ const target = relocatedMediaDir(newRoot, slug);
if (!(await isDirectory(target))) {
missing.push(slug);
continue;
@@ -624,7 +652,7 @@ export async function preflightRepoint(opts: {
base.problems.push(
`${missing.length} channel(s) have no media under ${newRoot}: ` +
`${missing.join(", ")}. Expected ` +
- `${relocatedDataDir(newRoot, missing[0])} and friends.`,
+ `${relocatedMediaDir(newRoot, missing[0])} and friends.`,
);
}
@@ -669,11 +697,11 @@ async function rollbackChannel(
// Only the field this job changed goes back; anything else edited since
// stays. (It used to rewrite the whole config it found at the start.)
await patchChannelConfig(paths, entry.slug, {
- dataDir: entry.oldTarget,
+ mediaDir: entry.oldTarget,
}).catch(() => {});
}
if (entry.relinked || entry.unlinked) {
- const link = path.join(paths.channelsDir, entry.slug, "data");
+ const link = path.join(paths.channelsDir, entry.slug, MEDIA_LINK_NAME);
await unlink(link).catch(() => {});
await symlink(entry.oldTarget, link).catch(() => {});
}
@@ -692,7 +720,7 @@ async function rollbackChannel(
// direction.
//
// IDEMPOTENT RERUN. A channel that was already re-pointed is no longer ON the
-// old root (`locationOfDataDir` reads its `dataDir`), so the preflight does not
+// old root (`locationOfDataDir` reads its `mediaDir`), so the preflight does not
// list it and this does not touch it. A rerun after a crash finishes the rest;
// a rerun after a complete run finds no channels and refuses with "already at".
export async function repointStorageLocation(opts: {
@@ -735,8 +763,8 @@ export async function repointStorageLocation(opts: {
for (const slug of pre.channels) {
opts.signal?.throwIfAborted();
const fresh = await readChannelConfig(opts.paths, slug);
- const oldTarget = fresh?.dataDir?.trim() ?? "";
- const newTarget = relocatedDataDir(pre.newRoot, slug);
+ const oldTarget = fresh?.mediaDir?.trim() ?? "";
+ const newTarget = relocatedMediaDir(pre.newRoot, slug);
const entry: ChannelLedgerEntry = {
slug,
oldTarget,
@@ -747,11 +775,11 @@ export async function repointStorageLocation(opts: {
oldConfig: fresh,
};
ledger.push(entry);
- const link = path.join(opts.paths.channelsDir, slug, "data");
+ const link = path.join(opts.paths.channelsDir, slug, MEDIA_LINK_NAME);
// RESUMING SKIPS THE LINK, and must: it already points at newTarget, so
// unlinking and recreating it would be two syscalls to reach the state it
// is in — and a crash between them would turn a half-done channel into a
- // channel with no `data/` at all, which is strictly worse than what we
+ // channel with no `media` at all, which is strictly worse than what we
// found. The ledger records nothing for the link for the same reason: a
// rollback must undo what THIS run did, and this run did not move it.
const resuming = pre.resumable.includes(slug);
@@ -764,9 +792,9 @@ export async function repointStorageLocation(opts: {
}
// A null patch means config.json vanished or became unreadable after
// preflight listed the channel: fail this step, so the ledger rolls
- // the link back, rather than leave a link no dataDir records.
+ // the link back, rather than leave a link no mediaDir records.
const written = await patchChannelConfig(opts.paths, slug, {
- dataDir: newTarget,
+ mediaDir: newTarget,
});
if (!written) {
throw new Error(`channels/${slug}/config.json is missing or unreadable`);
@@ -831,7 +859,7 @@ export async function repointStorageLocation(opts: {
}
resetStorageProbeMemo();
- // Every channel on it has a new link and a new dataDir: the page memo's keys
+ // Every channel on it has a new link and a new mediaDir: the page memo's keys
// already differ, and this drops the old answers rather than letting them
// age out.
forgetChannelMedia();
diff --git a/common/controller/storageWatch.test.ts b/common/controller/storageWatch.test.ts
@@ -30,11 +30,6 @@ import {
type LocationHealthState,
} from "../lib/storageHealth";
-// RELEASE 17 SLICE T1 made a channel whose `data/` is a link (or whose config
-// carries `dataDir`) `legacy`; these cases still build that retired layout and
-// expect it to read `ok`. Slice T2 rebases them on `media/` and un-skips them.
-const T1_SKIP = "release 17 T2 rebases the storage watch on mediaDir";
-
// THE CONFIRMATION COUNT IS MODULE STATE (see storageWatch.ts rule 3), so each
// case starts from a clean one — otherwise the second test inherits the first
@@ -115,8 +110,9 @@ async function withTmp(fn: (h: H) => Promise<void>): Promise<void> {
}
}
-// A channel whose `data/` is a link to `<root>/<slug>/data`, with the target
-// present or not.
+// A channel whose media tier is relocated (release 17): `channels/<slug>/media`
+// a link to `<root>/<slug>/media`, recorded as `mediaDir`, with the target
+// present or not. Its `data/` (the text) is on the corpus disk throughout.
async function seedRelocated(
h: H,
slug: string,
@@ -124,13 +120,13 @@ async function seedRelocated(
): Promise<void> {
const channelDir = path.join(h.paths.channelsDir, slug);
await mkdir(channelDir, { recursive: true });
- const target = path.join(h.root, slug, "data");
+ const target = path.join(h.root, slug, "media");
if (opts.targetExists) await mkdir(target, { recursive: true });
await writeFile(
path.join(channelDir, "config.json"),
- JSON.stringify({ handling: "youtube", dataDir: target }),
+ JSON.stringify({ handling: "youtube", mediaDir: target }),
);
- await symlink(target, path.join(channelDir, "data"));
+ await symlink(target, path.join(channelDir, "media"));
}
function tierOf(h: H, slug: string): string | undefined {
@@ -155,7 +151,7 @@ async function twoPasses(h: H) {
return { first, second };
}
-test("a channel whose target is gone is auto-paused, once, in one write", { skip: T1_SKIP }, async () => {
+test("a channel whose target is gone is auto-paused, once, in one write", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "gone-a", { targetExists: false });
await seedRelocated(h, "gone-b", { targetExists: false });
@@ -195,7 +191,7 @@ test("a channel whose target is gone is auto-paused, once, in one write", { skip
});
});
-test("the drive coming back restores the tier it overwrote", { skip: T1_SKIP }, async () => {
+test("the drive coming back restores the tier it overwrote", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "away", { targetExists: false });
const settings = h.io.read();
@@ -210,7 +206,7 @@ test("the drive coming back restores the tier it overwrote", { skip: T1_SKIP },
await twoPasses(h);
assert.equal(tierOf(h, "away"), "paused");
- await mkdir(path.join(h.root, "away", "data"), { recursive: true });
+ await mkdir(path.join(h.root, "away", "media"), { recursive: true });
const back = await runStorageWatchPass({ paths: h.paths, io: h.io, bins: h.paths });
assert.deepEqual(back.restored, ["away"]);
assert.equal(tierOf(h, "away"), "low");
@@ -254,7 +250,7 @@ test("a channel mid-relocation is not auto-paused", async () => {
await writeFile(
path.join(h.paths.channelsDir, "moving", ".relocating.json"),
JSON.stringify({
- target: path.join(h.root, "moving", "data"),
+ target: path.join(h.root, "moving", "media"),
direction: "out",
startedAt: "",
phase: "copy",
@@ -301,7 +297,7 @@ test("a record on an in-place channel is restored", async () => {
// `write: false` IS IDLE BOOT. It observes and reports; the write is the work,
// and idle boot refuses work.
-test("write: false reports the transition and changes nothing", { skip: T1_SKIP }, async () => {
+test("write: false reports the transition and changes nothing", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "gone", { targetExists: false });
// The first pass only suspects, whatever `write` says.
@@ -340,7 +336,7 @@ test("no locations and nothing auto-paused is a free pass", async () => {
// has spun down and needs a beat to answer is indistinguishable from "not
// mounted" — and pausing on it rewrites the corpus's priority document for a
// drive that is fine.
-test("a drive that blips for one pass is never paused", { skip: T1_SKIP }, async () => {
+test("a drive that blips for one pass is never paused", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "blip", { targetExists: false });
const first = await runStorageWatchPass({
@@ -355,7 +351,7 @@ test("a drive that blips for one pass is never paused", { skip: T1_SKIP }, async
// It answers on the next pass. Nothing was ever paused, and the suspicion
// is dropped — so a LATER real outage starts its own two-pass count rather
// than pausing immediately on the strength of a blip an hour ago.
- await mkdir(path.join(h.root, "blip", "data"), { recursive: true });
+ await mkdir(path.join(h.root, "blip", "media"), { recursive: true });
const second = await runStorageWatchPass({
paths: h.paths,
io: h.io,
@@ -385,13 +381,13 @@ test("a drive that blips for one pass is never paused", { skip: T1_SKIP }, async
// RESTORE STAYS SINGLE-PASS, and the asymmetry is the point: being slow to
// pause costs a few refused units (the start-of-work guards catch those), while
// being slow to restore leaves a lane off after the operator fixed the cable.
-test("the restore needs only one good pass", { skip: T1_SKIP }, async () => {
+test("the restore needs only one good pass", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "back", { targetExists: false });
await twoPasses(h);
assert.equal(tierOf(h, "back"), "paused");
h.writes = 0;
- await mkdir(path.join(h.root, "back", "data"), { recursive: true });
+ await mkdir(path.join(h.root, "back", "media"), { recursive: true });
const r = await runStorageWatchPass({
paths: h.paths,
io: h.io,
@@ -415,7 +411,7 @@ function scripted(answers: LocationHealthState[]) {
return async () => answers[Math.min(i++, answers.length - 1)];
}
-test("one missed probe stalls the location; pages then answer 'stalled' without asking", { skip: T1_SKIP }, async () => {
+test("one missed probe stalls the location; pages then answer 'stalled' without asking", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "slow", { targetExists: true });
const lines: string[] = [];
@@ -440,7 +436,7 @@ test("one missed probe stalls the location; pages then answer 'stalled' without
});
});
-test("the stall clears only after two clean probes in a row", { skip: T1_SKIP }, async () => {
+test("the stall clears only after two clean probes in a row", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "slow", { targetExists: true });
const probe = scripted(["stalled", "ok", "stalled", "ok", "ok"]);
diff --git a/common/controller/storageWatch.ts b/common/controller/storageWatch.ts
@@ -211,7 +211,12 @@ export async function runStorageWatchPass(
for (const { slug, config } of configs) {
const wasAutoPaused = Boolean(model.channels[slug]?.autoPaused);
- const dataDir = config.dataDir?.trim();
+ // ONE TIER PER CHANNEL (release 17): the drive its MEDIA is on —
+ // `mediaDir`, or on a channel not yet migrated off the retired
+ // whole-directory layout its `dataDir`. The pause stays the channel's (one
+ // `autoPaused` record), not a per-lane one; its text never leaves the
+ // corpus disk, so there is no second drive to watch.
+ const dataDir = config.mediaDir?.trim() || config.dataDir?.trim();
if (!dataDir) {
// In place. It cannot be on a drive that went away — but it CAN carry a
// record from before it was moved back, and that record has to come off
diff --git a/common/lib/channelMedia.ts b/common/lib/channelMedia.ts
@@ -160,9 +160,10 @@ export class ChannelTextUnreadableError extends Error {
}
}
-// THE RETIRED layout's target shape, `<root>/<slug>/data`. Still exported for
-// the mover, the re-point and the rename until release 17 slice T2 rebases them
-// on `relocatedMediaDir` (lib/mediaTier-server.ts).
+// THE RETIRED layout's target shape, `<root>/<slug>/data`. Nothing moves a
+// channel to it any more (the mover, the re-point and the rename use
+// `relocatedMediaDir`, lib/mediaTier-server.ts); it names what a `legacy`
+// channel's tree is, for the tier migration and the tests that build one.
export function relocatedDataDir(root: string, slug: string): string {
return path.join(root.trim(), slug, "data");
}
diff --git a/common/lib/mediaTier-server.ts b/common/lib/mediaTier-server.ts
@@ -63,8 +63,8 @@ export function channelMediaLink(
// A relocated channel's media root: `<root>/<slug>/media`. The suffix is fixed,
// not configurable, so an empty mountpoint can never be mistaken for the media
-// and the movers can recognise a target by its shape (the same reason
-// `relocatedDataDir` fixed `<slug>/data`).
+// and the movers can recognise a target by its shape (the same reason the
+// retired `relocatedDataDir` fixed `<slug>/data`).
export function relocatedMediaDir(root: string, slug: string): string {
return path.join(root.trim(), slug, MEDIA_LINK_NAME);
}
diff --git a/common/lib/savedVideoStore.ts b/common/lib/savedVideoStore.ts
@@ -44,8 +44,8 @@ export function savedVideosMarkerPath(
}
// The store's home on a location: `<root>/saved-videos`. Flat, beside the
-// channels' `<slug>/data` dirs, and not configurable for the same reason
-// `relocatedDataDir` is not — a mover recognises a target by its shape.
+// channels' `<slug>/media` dirs, and not configurable for the same reason
+// `relocatedMediaDir` is not — a mover recognises a target by its shape.
export function relocatedSavedVideosDir(root: string): string {
return path.join(root.trim(), SAVED_VIDEOS_DIRNAME);
}
diff --git a/common/lib/storageLocations.ts b/common/lib/storageLocations.ts
@@ -14,7 +14,9 @@ import type { StorageHealthSettings } from "./storageHealthTimings";
// be reachable from a client component at all (`next build` enforces that).
//
// The entity is stored in `settings.storage`. A channel is NOT tagged with its
-// location: it is on location L iff its `config.dataDir` is under `L.root`.
+// location: it is on location L iff its `config.mediaDir` is under `L.root`
+// (release 17: the media tier is what a location holds; a `legacy` channel is
+// placed by its retired `config.dataDir` until it is migrated).
// That is a derivation, not a field, which is why a re-point only has to
// rewrite the location's root and each channel's symlink — there is no second
// copy of the association to keep in step, and `ChannelConfig`'s whitelisted
@@ -24,8 +26,8 @@ import type { StorageHealthSettings } from "./storageHealthTimings";
// rendered and NEVER stored in `settings.storage.locations`. A stored entry
// under this id would be deletable, and worse, `locationOfDataDir` would then
// match every unrelocated channel — breaking the one rule the whole design
-// rests on (a channel is on location L iff its `dataDir` is under `L.root`,
-// and an in-place channel has no `dataDir`).
+// rests on (a channel is on location L iff its `mediaDir` is under `L.root`,
+// and an in-place channel has no `mediaDir`).
//
// It is HERE rather than beside the row that uses it because `lib/settings.ts`
// has to refuse it as a stored id, and lib may not import controller.
@@ -108,7 +110,7 @@ export const STORAGE_SETTINGS_FIELD_DOCS: FieldDocs<StorageSettings> = {
"WHERE THE SAVED-VIDEO STORE IS, by location id. \"\" = in place, under " +
"the corpus at `paths.savedVideosDir`.\n\n" +
"A RECORD OF WHAT IS ON DISK, never an intention — the same contract as" +
- " a channel's `config.dataDir`. It is written by the move, on success, " +
+ " a channel's `config.mediaDir`. It is written by the move, on success, " +
"after the copy has verified and the symlink is in place; nothing else " +
"writes it, and a reader that disagrees with the disk trusts the disk. " +
"Optional so an older settings.json parses (and an older binary that " +
@@ -135,8 +137,10 @@ function normalizeRoot(root: string): string {
return stripped === "" ? "/" : stripped;
}
-// Which location a channel's `dataDir` sits on, or null when it sits on none
-// (the ordinary case: an unrelocated channel's data is inside the corpus).
+// Which location a path sits on — a channel's `mediaDir` (or a legacy one's
+// retired `dataDir`) — or null when it sits on none (the ordinary case: an
+// unrelocated channel's media is inside the corpus). The name is historical:
+// it is a pure prefix test, and the movers ask it of any target.
//
// NESTED ROOTS ARE ALLOWED and the LONGEST match wins. "/mnt/platter" and
// "/mnt/platter/archive" can both be locations; a channel under the latter is
@@ -144,8 +148,8 @@ function normalizeRoot(root: string): string {
// first. Two locations sharing one root is a misconfiguration the sanitizer
// does not forbid; the first in the list wins it.
//
-// "Under" is strict: `dataDir === root` is not a match. A channel's dataDir is
-// always `<root>/<slug>/data`, so equality only ever means a misconfiguration.
+// "Under" is strict: `dir === root` is not a match. A channel's mediaDir is
+// always `<root>/<slug>/media`, so equality only ever means a misconfiguration.
export function locationOfDataDir(
dataDir: string,
locations: StorageLocation[],
diff --git a/common/views/channelRow.test.ts b/common/views/channelRow.test.ts
@@ -144,7 +144,7 @@ test("media: in-place is dropped; the label is the volume's unless overridden",
assert.equal(overridden.media?.locationLabel, undefined);
});
-test("channelVolumeOf: no dataDir is internal, a named root wins, else unnamed", () => {
+test("channelVolumeOf: no mediaDir is internal, a named root wins, else unnamed", () => {
const locations = [
{ id: "big", label: "Big disk", root: "/mnt/big" },
{ id: "bigger", label: "", root: "/mnt/big/inner" },
@@ -153,20 +153,25 @@ test("channelVolumeOf: no dataDir is internal, a named root wins, else unnamed",
id: "internal",
label: "Internal",
});
- assert.deepEqual(channelVolumeOf(" ", locations), {
+ assert.deepEqual(channelVolumeOf({ mediaDir: " " }, locations), {
id: "internal",
label: "Internal",
});
- assert.deepEqual(channelVolumeOf("/mnt/big/alpha/data", locations), {
+ assert.deepEqual(channelVolumeOf({ mediaDir: "/mnt/big/alpha/media" }, locations), {
id: "big",
label: "Big disk",
});
- assert.deepEqual(channelVolumeOf("/mnt/big/inner/a/data", locations), {
+ assert.deepEqual(channelVolumeOf({ mediaDir: "/mnt/big/inner/a/media" }, locations), {
id: "bigger",
label: "bigger",
});
- assert.deepEqual(channelVolumeOf("/elsewhere/a/data", locations), {
+ assert.deepEqual(channelVolumeOf({ mediaDir: "/elsewhere/a/media" }, locations), {
id: "",
label: "Elsewhere",
});
+ // A legacy channel (release 17) is placed by its retired `dataDir`.
+ assert.deepEqual(channelVolumeOf({ dataDir: "/mnt/big/old/data" }, locations), {
+ id: "big",
+ label: "Big disk",
+ });
});
diff --git a/common/views/channelRow.ts b/common/views/channelRow.ts
@@ -112,13 +112,15 @@ export type ChannelRowView = {
// the lanes read the channel. The rack draws it beside the tier, because it
// is the reason the lanes are skipping the row (release 16 slice RM).
mediaHold: string | null;
- // WHICH VOLUME, as an id a filter can name: a location id, "internal" for the
- // corpus volume, or "" for a dataDir under a root nobody named.
+ // WHICH VOLUME THE MEDIA IS ON, as an id a filter can name: a location id,
+ // "internal" for the corpus volume, or "" for a mediaDir under a root nobody
+ // named. (The text is on the corpus volume whatever this says.)
volumeId: string;
volumeLabel: string;
- // `snapshot.totalMediaBytes`. NULL, not 0, for a report written before the
- // field existed: a 400 GB channel that has not been measured must not sort as
- // the smallest thing on the disk.
+ // `snapshot.totalMediaBytes` — the MEDIA TIER's bytes since release 17 (the
+ // audio and the raw live chat: what a move carries). NULL, not 0, for a
+ // report written before the field existed: a 400 GB channel that has not
+ // been measured must not sort as the smallest thing on the disk.
mediaBytes: number | null;
};
@@ -160,17 +162,18 @@ export type ChannelVolumeId = { id: string; label: string };
// WHICH VOLUME A CHANNEL'S MEDIA IS ON, as a filterable id.
//
-// Same derivation the badge uses (`config.dataDir` under a location's root,
-// longest match wins) with one addition: no `dataDir` at all means the corpus
-// volume, which is the row the operator is trying to empty and therefore the
-// one they most need to filter to. A `dataDir` under a root NOBODY named is
-// neither — it gets "" and falls out of every volume filter, which is the
-// honest answer and the nudge to name that root on /storage.
+// Same derivation the badge uses (`config.mediaDir` under a location's root,
+// longest match wins; a `legacy` channel by its retired `dataDir`) with one
+// addition: no target at all means the corpus volume, which is the row the
+// operator is trying to empty and therefore the one they most need to filter
+// to. A target under a root NOBODY named is neither — it gets "" and falls out
+// of every volume filter, which is the honest answer and the nudge to name
+// that root on /storage.
export function channelVolumeOf(
- dataDir: string | undefined,
+ config: Pick<ChannelConfig, "mediaDir" | "dataDir"> | undefined,
locations: StorageLocation[],
): ChannelVolumeId {
- const trimmed = dataDir?.trim();
+ const trimmed = config?.mediaDir?.trim() || config?.dataDir?.trim();
if (!trimmed) {
return { id: INTERNAL_LOCATION_ID, label: "Internal" };
}
diff --git a/common/views/storage.test.ts b/common/views/storage.test.ts
@@ -31,9 +31,12 @@ function rollup(partial: Partial<LocationRollup>): LocationRollup {
bytes: partial.bytes ?? 0,
unknownBytes: partial.unknownBytes ?? 0,
clipsBytes: partial.clipsBytes ?? 0,
+ textBytes: partial.textBytes ?? 0,
+ unknownTextBytes: partial.unknownTextBytes ?? 0,
ok: partial.ok ?? 0,
unreachable: partial.unreachable ?? 0,
moving: partial.moving ?? 0,
+ legacy: partial.legacy ?? 0,
};
}
@@ -375,29 +378,83 @@ test("the clip-window line is rendered only when there are windows", () => {
assert.match(storageClipsText(5 * 1024 ** 3), /5\.00 GB of it is fetched/);
});
-test("a location row carries the clips share of its bytes", () => {
+// RELEASE 17: a location holds the media tier of the channels on it, and
+// nothing else — their text and clip windows never leave the corpus volume, so
+// they are counted on the internal row, whose breakdown says so.
+test("a location row carries the media tier only; the internal row holds every channel's text and clips", () => {
const payload = buildStorageRows({
locations: [loc("cold", "/mnt/cold")],
+ internal: {
+ root: "/corpus/channels",
+ corpus: { textBytes: 3 * GB, clipsBytes: 2 * GB, unknown: 0 },
+ },
defaultLocationId: "",
probes: {},
rollups: {
cold: rollup({
locationId: "cold",
total: 2,
- bytes: 10_000_000_000,
- clipsBytes: 2_000_000_000,
+ bytes: 10 * GB,
+ clipsBytes: 1 * GB,
+ textBytes: 1 * GB,
ok: 2,
}),
+ internal: rollup({ locationId: "internal", total: 1, ok: 1, bytes: 5 * GB }),
},
registry: NO_JOBS,
udisksctlAvailable: false,
now: 0,
});
- const row = payload.rows.find((r) => r.id === "cold");
- assert.equal(row?.clipsBytes, 2_000_000_000);
- assert.match(String(row?.clipsText), /fetched clip windows/);
- // A SUBSET, not a sibling: the clips are already inside `bytes`.
- assert.equal(row?.bytes, 10_000_000_000);
+ const cold = payload.rows.find((r) => r.id === "cold");
+ assert.equal(cold?.bytes, 10 * GB);
+ assert.equal(cold?.clipsBytes, 0);
+ assert.equal(cold?.clipsText, "");
+ assert.equal(cold?.tiersText, "");
+ const internal = payload.rows.find((r) => r.id === "internal");
+ // 5 GB of in-place media + 3 GB of text + 2 GB of clips.
+ assert.equal(internal?.bytes, 10 * GB);
+ assert.equal(internal?.clipsBytes, 2 * GB);
+ assert.match(String(internal?.clipsText), /2\.00 GB of it is fetched clip windows/);
+ assert.equal(
+ internal?.tiersText,
+ "text 3.00 GB + clips 2.00 GB on the corpus volume, plus 5.00 GB media of in-place channels",
+ );
+ assert.equal(payload.bytesInPlace, 10 * GB);
+});
+
+test("an old report's missing text figure is unknown on the internal row, never 0", () => {
+ const { rows } = buildStorageRows({
+ locations: [],
+ internal: {
+ root: "/corpus/channels",
+ corpus: { textBytes: 1 * GB, clipsBytes: 0, unknown: 2 },
+ },
+ defaultLocationId: "",
+ probes: {},
+ rollups: {
+ internal: rollup({ locationId: "internal", total: 3, ok: 3, bytes: 4 * GB, unknownBytes: 1 }),
+ },
+ registry: NO_JOBS,
+ now: 0,
+ });
+ assert.equal(rows[0].bytes, 5 * GB);
+ assert.equal(rows[0].unknownBytes, 3);
+ assert.equal(rows[0].bytesText, "5.00 GB + 3 unmeasured");
+});
+
+test("a legacy channel counts as unreachable, with how many are to migrate", () => {
+ const { rows } = buildStorageRows({
+ locations: [loc("cold", "/mnt/cold")],
+ defaultLocationId: "",
+ probes: {},
+ rollups: {
+ cold: rollup({ locationId: "cold", total: 3, ok: 1, unreachable: 2, legacy: 2 }),
+ },
+ registry: NO_JOBS,
+ now: 0,
+ });
+ assert.equal(rows[0].channels.legacy, 2);
+ assert.equal(rows[0].channelsText, "1 ok / 2 unreachable (2 to migrate) / 0 moving");
});
test("a location whose drive is not answering reads so, and says since when", () => {
diff --git a/common/views/storage.ts b/common/views/storage.ts
@@ -50,6 +50,9 @@ export type StorageChannelCounts = {
unreachable: number;
moving: number;
total: number;
+ // The share of `unreachable` that is the retired whole-directory layout,
+ // waiting for `archilyzer storage migrate-tier` (release 17).
+ legacy: number;
};
export type StorageRow = {
@@ -80,23 +83,29 @@ export type StorageRow = {
// Built here rather than in the page so the param name has ONE spelling
// across the two surfaces that use it.
channelsHref: string;
- // Media bytes on this location, summed from each channel's last report, and
- // how many channels on it could not contribute a figure.
+ // What this row's volume holds, summed from each channel's last report, and
+ // how many channels could not contribute a figure. A LOCATION holds the
+ // media tier of the channels on it (release 17: their text never leaves the
+ // corpus volume). The INTERNAL row holds every channel's text and clip
+ // windows, plus the media of the channels in place.
bytes: number;
unknownBytes: number;
// "1.42 TB (2 channels unmeasured)" / "size unknown until Refresh report".
// ONE wording, and never a bare "0 B" for a location whose channels have
// simply never had a report — that reads as an empty drive.
bytesText: string;
- // The `clips/` share of `bytes` — fetched clip windows, the one part of a
- // location's media that is a cache nothing prunes. 0 is a real answer here
- // (no windows have been fetched onto this drive), unlike `bytes`, which is
- // why it needs no "unknown" companion: the channels that could not measure
- // clips are exactly the ones `unknownBytes` already counts.
+ // The `clips/` share of `bytes` — fetched clip windows, a cache nothing
+ // prunes. On the INTERNAL row only since release 17: `clips/` is never
+ // tiered, so a location holds none (0 there). 0 is a real answer, unlike
+ // `bytes`, which is why it needs no "unknown" companion.
clipsBytes: number;
- // "· 4.21 GB in clip windows", or "" when there are none. Built here so the
- // row and any future poll cannot word it differently.
+ // "4.21 GB of it is fetched clip windows", or "" when there are none. Built
+ // here so the row and any future poll cannot word it differently.
clipsText: string;
+ // THE INTERNAL ROW'S BREAKDOWN: "text 12.1 GB + clips 4.2 GB on the corpus
+ // volume, plus 310 GB media of in-place channels". "" on a location row,
+ // whose figure is the media tier alone.
+ tiersText: string;
// Absent when the location is not available, and ALSO when it is available
// but unmeasurable (getFreeBytes fails open to Infinity, which the probe
// drops rather than carry). Render "—" for both.
@@ -197,7 +206,18 @@ export type StorageRowsInputs = {
// The corpus volume as a row. Absent → no internal row (a caller that only
// wants the configured locations). `freeBytes` is a statfs of the root, taken
// by the shell, because nothing in views/ may touch a disk.
- internal?: { root: string; freeBytes?: number };
+ //
+ // `corpus` (release 17): what EVERY channel keeps on the corpus volume
+ // whatever its media's location — the text tier and the clip windows,
+ // summed off the reports — and how many channels could not contribute (a
+ // report written before release 17 has no text figure, beyond the in-place
+ // channels the internal rollup already counts unmeasured). Absent → only the
+ // in-place channels' media, as before.
+ internal?: {
+ root: string;
+ freeBytes?: number;
+ corpus?: { textBytes: number; clipsBytes: number; unknown: number };
+ };
defaultLocationId: string;
// By location id. A location with no entry has never been probed in this
// process — treated as `missing` with an unknown identity rather than
@@ -255,9 +275,32 @@ function countsOf(rollup: LocationRollup | undefined): StorageChannelCounts {
unreachable: rollup?.unreachable ?? 0,
moving: rollup?.moving ?? 0,
total: rollup?.total ?? 0,
+ legacy: rollup?.legacy ?? 0,
};
}
+// `n ok / n unreachable / n moving` — with "(n to migrate)" after the
+// unreachable count when any of it is the retired layout. ONE wording, for the
+// rows and any future poll.
+export function storageChannelsText(c: StorageChannelCounts): string {
+ const migrate = c.legacy > 0 ? ` (${c.legacy} to migrate)` : "";
+ return `${c.ok} ok / ${c.unreachable} unreachable${migrate} / ${c.moving} moving`;
+}
+
+// THE INTERNAL ROW'S BREAKDOWN (release 17): what every channel keeps on the
+// corpus volume, plus the in-place channels' media.
+export function storageTiersText(
+ textBytes: number,
+ clipsBytes: number,
+ inPlaceMediaBytes: number,
+): string {
+ return (
+ `text ${bytesLabel(textBytes)} + clips ${bytesLabel(clipsBytes)} on the ` +
+ `corpus volume, plus ${bytesLabel(inPlaceMediaBytes)} media of in-place ` +
+ `channels`
+ );
+}
+
// Bytes → GB with two decimals, or TB past a terabyte. Local rather than
// `lib/format`'s formatBytes because this module is reachable from the client
// bundle and is deliberately import-free.
@@ -308,7 +351,7 @@ function runningRepoint(registry: RegistryReader): string | null {
// the registry caps at concurrency 1 — so either one running is a fact about
// the machine and freezes every row on this page, not just its own. (A
// CHANNEL move is on that key too and deliberately does NOT freeze this page:
- // it touches one channel's `data/`, never a location's root or the store, and
+ // it touches one channel's `media/`, never a location's root or the store, and
// /storage has been usable during one since locations shipped.)
const job = registry
.list()
@@ -369,9 +412,10 @@ export function buildStorageRows(i: StorageRowsInputs): StorageRowsPayload {
];
const roll = i.rollups[loc.id];
+ // The media tier only: the channels' text and clip windows are on the
+ // corpus volume, counted on the internal row.
const bytes = roll?.bytes ?? 0;
const unknownBytes = roll?.unknownBytes ?? 0;
- const clipsBytes = roll?.clipsBytes ?? 0;
bytesOnLocation[loc.id] = bytes;
unknownBytesOnLocation[loc.id] = unknownBytes;
return {
@@ -385,13 +429,14 @@ export function buildStorageRows(i: StorageRowsInputs): StorageRowsPayload {
statusLabel: STORAGE_STATUS_LABEL[status],
identity: probe ? identityLine(probe.identity) : null,
channels: counts,
- channelsText: `${counts.ok} ok / ${counts.unreachable} unreachable / ${counts.moving} moving`,
+ channelsText: storageChannelsText(counts),
channelsHref: channelsHrefForLocation(loc.id),
bytes,
unknownBytes,
bytesText: storageBytesText(bytes, unknownBytes),
- clipsBytes,
- clipsText: storageClipsText(clipsBytes),
+ clipsBytes: 0,
+ clipsText: "",
+ tiersText: "",
...(probe?.freeBytes !== undefined ? { freeBytes: probe.freeBytes } : {}),
lastProbeAgeMs: probe ? Math.max(0, i.now - probe.probedAt) : 0,
...(i.notAnswering?.[loc.id]
@@ -479,9 +524,13 @@ function internalRow(
const internal = i.internal as NonNullable<StorageRowsInputs["internal"]>;
const roll = i.rollups[INTERNAL_ROW_ID];
const counts = countsOf(roll);
- const bytes = roll?.bytes ?? 0;
- const unknownBytes = roll?.unknownBytes ?? 0;
- const clipsBytes = roll?.clipsBytes ?? 0;
+ // The in-place channels' media, plus (release 17) every channel's text and
+ // clip windows, which stay on this volume wherever the media is.
+ const inPlaceMedia = roll?.bytes ?? 0;
+ const corpus = internal.corpus;
+ const clipsBytes = corpus ? corpus.clipsBytes : (roll?.clipsBytes ?? 0);
+ const bytes = inPlaceMedia + (corpus ? corpus.textBytes + corpus.clipsBytes : 0);
+ const unknownBytes = (roll?.unknownBytes ?? 0) + (corpus?.unknown ?? 0);
bytesOnLocation[INTERNAL_ROW_ID] = bytes;
unknownBytesOnLocation[INTERNAL_ROW_ID] = unknownBytes;
const withheld =
@@ -499,13 +548,16 @@ function internalRow(
statusLabel: STORAGE_STATUS_LABEL.available,
identity: null,
channels: counts,
- channelsText: `${counts.ok} ok / ${counts.unreachable} unreachable / ${counts.moving} moving`,
+ channelsText: storageChannelsText(counts),
channelsHref: channelsHrefForLocation(INTERNAL_ROW_ID),
bytes,
unknownBytes,
bytesText: storageBytesText(bytes, unknownBytes),
clipsBytes,
clipsText: storageClipsText(clipsBytes),
+ tiersText: corpus
+ ? storageTiersText(corpus.textBytes, corpus.clipsBytes, inPlaceMedia)
+ : "",
...(internal.freeBytes !== undefined ? { freeBytes: internal.freeBytes } : {}),
lastProbeAgeMs: 0,
busy: null,
@@ -609,7 +661,7 @@ function mountAction(
// DELETE IS REFUSED WHILE ANYBODY LIVES THERE. Deleting the location would not
// touch a byte — but it would erase the only record of which disk those
-// channels' absolute `dataDir`s belong to, which is precisely the knowledge
+// channels' absolute `mediaDir`s belong to, which is precisely the knowledge
// this page exists to keep. Move the channels off it (or re-point it) first.
function deleteAction(
loc: StorageLocation,
diff --git a/editor/app/channels/page.tsx b/editor/app/channels/page.tsx
@@ -252,7 +252,7 @@ export default async function ChannelsPage({
// in-place one and names the location from the volume, which is the
// same prefix match the Location column shows — one answer, not two.
media: mediaBySlug.get(stat.slug) ?? null,
- volume: channelVolumeOf(brief?.config.dataDir, locations),
+ volume: channelVolumeOf(brief?.config, locations),
priority: {
tier: tierOf(priority, stat.slug),
rank: rankOf(priority, stat.slug),
diff --git a/editor/app/page.tsx b/editor/app/page.tsx
@@ -100,7 +100,7 @@ export default async function Dashboard({
bands: [],
priority: neutralChannelPriority(),
media,
- volume: channelVolumeOf(r.channel.config.dataDir, locations),
+ volume: channelVolumeOf(r.channel.config, locations),
// The named location, matched against the configured roots from the
// media TARGET, as this table always has. A pure prefix match, never a
// probe.
diff --git a/editor/app/storage/actions.ts b/editor/app/storage/actions.ts
@@ -190,7 +190,7 @@ export async function editStorageLocationAction(
}
// DELETING A LOCATION MOVES NOTHING, and that is exactly why it is refused
-// while channels are on it: their `config.dataDir` would keep naming an
+// while channels are on it: their `config.mediaDir` would keep naming an
// absolute path on a disk nothing in the corpus remembers the name of, which is
// the situation this page exists to end.
export async function deleteStorageLocationAction(
diff --git a/editor/app/storage/buildStorage.ts b/editor/app/storage/buildStorage.ts
@@ -50,14 +50,32 @@ export async function buildStorage(): Promise<StorageRowsPayload> {
// snapshot predates `totalMediaBytes` contributes to `unknownBytes` and the
// row says so rather than under-reporting.
const briefs = await listChannelBriefs(paths);
+ // THE THREE TIERS OFF EACH REPORT (release 17): the media tier (what a
+ // location holds), the text tier and the clip windows (on the corpus volume
+ // whatever the media's location). A report written before release 17 has no
+ // text figure: unknown, never 0.
const mediaBytes: Record<string, number | undefined> = {};
- // The `clips/` share, off the same snapshot. Absent for a report written
- // before the field existed — the same set `mediaBytes` is absent for, which
- // is why the rollup needs no second "unknown" counter.
const clipsBytes: Record<string, number | undefined> = {};
+ const textBytes: Record<string, number | undefined> = {};
+ const corpus = { textBytes: 0, clipsBytes: 0, unknown: 0 };
for (const b of briefs) {
mediaBytes[b.slug] = b.snapshot?.totalMediaBytes;
clipsBytes[b.slug] = b.snapshot?.totalClipsBytes;
+ textBytes[b.slug] = b.snapshot?.totalTextBytes;
+ const text = b.snapshot?.totalTextBytes;
+ const clips = b.snapshot?.totalClipsBytes;
+ if (typeof text === "number") corpus.textBytes += text;
+ if (typeof clips === "number") corpus.clipsBytes += clips;
+ // Unmeasured on the corpus volume: a report with no text figure — except
+ // an in-place channel whose MEDIA is unmeasured too, which the internal
+ // rollup already counts (one channel, one "unmeasured").
+ const inPlace = !b.config.mediaDir?.trim() && !b.config.dataDir?.trim();
+ if (
+ typeof text !== "number" &&
+ !(inPlace && typeof b.snapshot?.totalMediaBytes !== "number")
+ ) {
+ corpus.unknown += 1;
+ }
}
const configs = briefs.map((b) => ({ slug: b.slug, config: b.config }));
const [probes, rollups, udisksctl, freeOnCorpus] = await Promise.all([
@@ -68,6 +86,7 @@ export async function buildStorage(): Promise<StorageRowsPayload> {
configs,
mediaBytes,
clipsBytes,
+ textBytes,
includeInternal: true,
}),
// Memoised per binary path inside storageVolumes, so this is one
@@ -141,6 +160,7 @@ export async function buildStorage(): Promise<StorageRowsPayload> {
internal: {
root: paths.channelsDir,
...(Number.isFinite(freeOnCorpus) ? { freeBytes: freeOnCorpus } : {}),
+ corpus,
},
defaultLocationId: settings.storage.defaultLocationId,
probes,
diff --git a/editor/app/storage/components/StorageLocationsTable.tsx b/editor/app/storage/components/StorageLocationsTable.tsx
@@ -233,10 +233,22 @@ function LocationCard({
<dt className="text-muted-foreground">Media</dt>
<dd aria-label="location media bytes">
{row.bytesText}
- {/* THE CLIPS SHARE, inside the Media figure and never beside it as a
- second total. A fetched window lives under `data/<id>/clips/`, so
- it is already in the number above; what this adds is that some of
- it is a CACHE nothing prunes. Absent when there are none. */}
+ {/* THE INTERNAL ROW'S BREAKDOWN (release 17): every channel's text
+ and clip windows stay on the corpus volume, plus the media of the
+ channels in place. A location row's figure is its media tier
+ alone, and carries no breakdown. */}
+ {row.tiersText && (
+ <span
+ aria-label="location tier bytes"
+ className="block text-xs text-muted-foreground"
+ >
+ {row.tiersText}
+ </span>
+ )}
+ {/* THE CLIPS SHARE, inside the figure and never beside it as a
+ second total: on the internal row (clips/ is never tiered), it is
+ already in the number above; what this adds is that some of it is
+ a CACHE nothing prunes. Absent when there are none. */}
{row.clipsText && (
<span
aria-label="location clip bytes"
