commit 1eb37bde03bab3949985c10cf14e6cf7047e91ea
parent 9ee7f1eed846d48233677cfd6f9bb3bcb15d522b
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 2 Oct 2026 01:56:40 -0400
Merge r17/media-tier-mover (release 17 slice T2) — the mover works on channels/<slug>/media: a move out tiers a classic channel's big files first, copies, mirrors toward the copy only, verifies, parks and leaves one absolute link with config.mediaDir; a move back ends with a real media/ and no per-file link changes either way; every marker carries scope; a legacy channel and a tier-migration marker are refused with the migrate-tier sentence; re-point, rename, delete, the storage watch and /storage key on mediaDir (a legacy channel re-points the retired way); the Storage panel shows media and text paths and bytes, the badge says Media layout retired, the video pages list tiered links and the file route answers 503 with retry-after for a relocated drive that is away; reviewed SHIP
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
50 files changed, 2690 insertions(+), 863 deletions(-)
diff --git a/SETTINGS.md b/SETTINGS.md
@@ -572,7 +572,7 @@ Where a channel's downloaded media goes when it is relocated off the corpus disk
|---|---|---|
| `locations` | `[]` | The named storage locations a channel's media may be relocated to — one entry per root, each with an id, label, root, `autoRepoint` and the learned volume identity. Order is display order. Managed on /storage. |
| `defaultLocationId` | `""` | The location prefilled as the destination of a move. "" = no default. |
-| `savedVideosLocationId` | absent | WHERE THE SAVED-VIDEO STORE IS, by location id. "" = in place, under the corpus at `paths.savedVideosDir`.<br><br>A RECORD OF WHAT IS ON DISK, never an intention — the same contract as a channel's `config.dataDir`. It is written by the move, on success, after the copy has verified and the symlink is in place; nothing else writes it, and a reader that disagrees with the disk trusts the disk. Optional so an older settings.json parses (and an older binary that drops it leaves a store that still works, because the symlink is what every reader follows). |
+| `savedVideosLocationId` | absent | WHERE THE SAVED-VIDEO STORE IS, by location id. "" = in place, under the corpus at `paths.savedVideosDir`.<br><br>A RECORD OF WHAT IS ON DISK, never an intention — the same contract as a channel's `config.mediaDir`. It is written by the move, on success, after the copy has verified and the symlink is in place; nothing else writes it, and a reader that disagrees with the disk trusts the disk. Optional so an older settings.json parses (and an older binary that drops it leaves a store that still works, because the symlink is what every reader follows). |
| `health` | absent | THE DRIVE-HEALTH TIMINGS: how long a read may take before a drive counts as not answering, how often the health pass looks, how long its look may take, how many clean looks clear a stall, and how many reads may be on one drive at once. Edited on /storage (Drive health timing). Absent = every default, and only a value that differs from its default is written, so an untuned install follows a default changed later. See `storage.health` below. |
#### `storage.locations[]`
diff --git a/common/controller/archiveLiveChat.ts b/common/controller/archiveLiveChat.ts
@@ -18,6 +18,7 @@ import path from "node:path";
import {
access,
link,
+ lstat,
mkdir,
readdir,
rm,
@@ -26,7 +27,9 @@ import {
} from "node:fs/promises";
import pLimit from "p-limit";
import { listChannelStatsFromDisk, type ChannelStat } from "./channels";
-import { normalizeLiveChat } from "./normalizeLiveChat";
+import { isLiveChatCuesFresh, normalizeLiveChat } from "./normalizeLiveChat";
+import { inspectChannelMedia } from "../lib/channelMedia";
+import { LIVE_CHAT_MEDIA_FILENAME } from "../lib/mediaTier";
import {
archiveExtension,
archiveSidecarPath,
@@ -139,9 +142,26 @@ async function stageChannel(
log(`Archive live chat ${ch.slug}: scanning ${videoIds.length} videos`);
+ // THE RAW REPLAY IS MEDIA (release 17): `transcript.live_chat.json` is a
+ // tiered link onto the channel's media drive. A build re-normalizes a stale
+ // cues file from it only while that drive is reachable; otherwise the cues
+ // the corpus disk holds are staged as they are (the freshness check `lstat`s
+ // the link, so asking it costs the drive nothing), and a video with no cues
+ // yet is left out of this build. Never a read of a drive that is unmounted,
+ // stalled or mid-move.
+ const media = await inspectChannelMedia(paths, ch.slug, ch.config, {
+ fresh: true,
+ });
+ const rawReadable = media.status === "ok" || media.status === "in-place";
+
let normalizedCount = 0;
let linkedCount = 0;
let normalizeFailed = 0;
+ let keptStale = 0;
+ // Videos with a raw replay (a link onto the away drive, lstat'd) and no cues
+ // on disk yet: left out of this build, and COUNTED — a drive outage must not
+ // shrink a site silently (review L4).
+ let leftOut = 0;
const requiresRewrite = !build.includeMetadata || build.prettyPrint;
@@ -157,13 +177,31 @@ async function stageChannel(
}
let outcome;
try {
- outcome = await normalizeLiveChat({
- videoDir,
- channelSlug: ch.slug,
- configName: ch.config.name,
- // A build moves no media (release 17): the jobs tier, this reads.
- tier: false,
- });
+ if (rawReadable) {
+ outcome = await normalizeLiveChat({
+ videoDir,
+ channelSlug: ch.slug,
+ configName: ch.config.name,
+ // A build moves no media (release 17): the jobs tier, this reads.
+ tier: false,
+ });
+ } else {
+ const { fresh, cuesPath } = await isLiveChatCuesFresh(videoDir);
+ const have = await stat(cuesPath).then(
+ (st) => st.isFile(),
+ () => false,
+ );
+ if (!have) {
+ const hasRaw = await lstat(path.join(videoDir, LIVE_CHAT_MEDIA_FILENAME)).then(
+ () => true,
+ () => false,
+ );
+ if (hasRaw) leftOut++;
+ return;
+ }
+ if (!fresh) keptStale++;
+ outcome = { status: "fresh" as const, cuesPath };
+ }
} catch (err) {
normalizeFailed++;
log(` ! normalize ${ch.slug}/${id}: ${(err as Error).message}`);
@@ -189,6 +227,14 @@ async function stageChannel(
if (normalizeFailed > 0) {
log(` ${ch.slug}: ${normalizeFailed} videos failed to normalize`);
}
+ if (!rawReadable) {
+ log(
+ ` ${ch.slug}: its media is not reachable (${media.detail ?? media.status}) — ` +
+ `staged the live-chat cues on disk as they are` +
+ (keptStale > 0 ? ` (${keptStale} older than their raw replay)` : "") +
+ `; ${leftOut} video(s) with a raw replay and no cues left out of this build`,
+ );
+ }
if (linkedCount > 0) {
await writeFile(
diff --git a/common/controller/channels.test.ts b/common/controller/channels.test.ts
@@ -17,6 +17,7 @@ import {
relocatedDataDir,
RELOCATION_MARKER_FILENAME,
} from "../lib/channelMedia";
+import { relocatedMediaDir } from "../lib/mediaTier-server";
import {
channelConfigPath,
channelExists,
@@ -70,11 +71,11 @@ test("deleting an in-place channel removes its directory", async () => {
test("deleting a relocated channel reclaims the media on the other drive", async () => {
await withPaths(async (paths, mediaRoot) => {
- const target = relocatedDataDir(mediaRoot, "alpha");
- await writeChannelConfig(paths, "alpha", { ...config, dataDir: target });
+ const target = relocatedMediaDir(mediaRoot, "alpha");
+ await writeChannelConfig(paths, "alpha", { ...config, mediaDir: target });
await mkdir(path.join(target, "v1"), { recursive: true });
await writeFile(path.join(target, "v1", "audio.m4a"), "BYTES");
- await symlink(target, path.join(paths.channelsDir, "alpha", "data"));
+ await symlink(target, path.join(paths.channelsDir, "alpha", "media"));
await deleteChannel(paths, "alpha");
@@ -89,11 +90,11 @@ test("deleting a relocated channel reclaims the media on the other drive", async
test("a non-empty <root>/<slug> survives the delete", async () => {
await withPaths(async (paths, mediaRoot) => {
- const target = relocatedDataDir(mediaRoot, "alpha");
- await writeChannelConfig(paths, "alpha", { ...config, dataDir: target });
+ const target = relocatedMediaDir(mediaRoot, "alpha");
+ await writeChannelConfig(paths, "alpha", { ...config, mediaDir: target });
await mkdir(target, { recursive: true });
- await symlink(target, path.join(paths.channelsDir, "alpha", "data"));
- // Something of the operator's, alongside the data dir we own.
+ await symlink(target, path.join(paths.channelsDir, "alpha", "media"));
+ // Something of the operator's, alongside the media dir we own.
await writeFile(path.join(mediaRoot, "alpha", "NOTES.txt"), "mine");
await deleteChannel(paths, "alpha");
@@ -102,9 +103,26 @@ test("a non-empty <root>/<slug> survives the delete", async () => {
});
});
-test("deleting is refused while a relocation is in flight", async () => {
+// THE RETIRED LAYOUT (release 17): a channel never migrated off the whole-
+// directory move still carries `dataDir`; deleting it reclaims that tree too.
+test("deleting a legacy channel reclaims its retired data dir", async () => {
await withPaths(async (paths, mediaRoot) => {
const target = relocatedDataDir(mediaRoot, "alpha");
+ await writeChannelConfig(paths, "alpha", { ...config, dataDir: target });
+ await mkdir(path.join(target, "v1"), { recursive: true });
+ await writeFile(path.join(target, "v1", "transcript.json"), "{}");
+ await symlink(target, path.join(paths.channelsDir, "alpha", "data"));
+
+ await deleteChannel(paths, "alpha");
+ assert.equal(await channelExists(paths, "alpha"), false);
+ assert.equal(await there(target), false);
+ assert.equal(await there(path.join(mediaRoot, "alpha")), false);
+ });
+});
+
+test("deleting is refused while a relocation is in flight", async () => {
+ await withPaths(async (paths, mediaRoot) => {
+ const target = relocatedMediaDir(mediaRoot, "alpha");
await writeChannelConfig(paths, "alpha", config);
await mkdir(path.join(paths.channelsDir, "alpha", "data"), {
recursive: true,
@@ -131,9 +149,9 @@ test("an unmounted target does not make the channel undeletable", async () => {
// `force: true` makes a missing target a no-op, so the channel goes and its
// media stays on the unmounted platter. Deliberate: refusing here would
// make a channel whose drive is gone for good impossible to remove.
- const target = relocatedDataDir(mediaRoot, "alpha");
- await writeChannelConfig(paths, "alpha", { ...config, dataDir: target });
- await symlink(target, path.join(paths.channelsDir, "alpha", "data"));
+ const target = relocatedMediaDir(mediaRoot, "alpha");
+ await writeChannelConfig(paths, "alpha", { ...config, mediaDir: target });
+ await symlink(target, path.join(paths.channelsDir, "alpha", "media"));
await deleteChannel(paths, "alpha");
assert.equal(await channelExists(paths, "alpha"), false);
});
diff --git a/common/controller/channels.ts b/common/controller/channels.ts
@@ -591,10 +591,15 @@ export async function deleteChannel(
);
}
const dir = path.join(paths.channelsDir, slug);
- const target = (await readChannelConfig(paths, slug))?.dataDir?.trim();
- if (target) {
+ const config = await readChannelConfig(paths, slug);
+ // The media tier's target (`mediaDir`, release 17) and, on a channel never
+ // migrated off the retired whole-directory layout, its `dataDir` — each on
+ // another drive, each removed before the channel dir. An in-place `media/`
+ // is a real directory inside `dir` and goes with it.
+ for (const target of [config?.mediaDir?.trim(), config?.dataDir?.trim()]) {
+ if (!target) continue;
await rm(target, { recursive: true, force: true });
- // <root>/<slug> is ours by construction (relocatedDataDir fixes the
+ // <root>/<slug> is ours by construction (relocatedMediaDir fixes the
// suffix), so take it too — but only when nothing else landed in it.
const slugRoot = path.dirname(target);
if (path.basename(slugRoot) === slug) {
diff --git a/common/controller/relocateChannelMedia.test.ts b/common/controller/relocateChannelMedia.test.ts
@@ -23,11 +23,17 @@ import type { SiteSettings } from "../lib/settings";
import {
inspectChannelMedia,
readRelocationMarker,
- relocatedDataDir,
} from "../lib/channelMedia";
+import {
+ relocatedMediaDir,
+ tierChannelMedia,
+ tierLinkTarget,
+} from "../lib/mediaTier-server";
import type { ChannelWriter } from "./channelWriters";
+import type { JobRecord } from "../jobs/registry";
import {
assertRelocationRootPresent,
+ channelMediaWriters,
previewRelocation,
relocateChannelMedia,
relocationRootPresenceProblem,
@@ -42,16 +48,17 @@ import {
} from "../lib/storageVolumes";
import { readChannelConfig } from "./channels";
-// RELEASE 17 SLICE T1 made a channel whose `data/` is a link (or whose config
-// carries `dataDir`) `legacy`; these cases still build that retired layout and
-// expect it to read `ok`. Slice T2 rebases them on `media/` and un-skips them.
-const T1_SKIP = "release 17 T2 rebases the mover on media/";
-
// Run with:
// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/relocateChannelMedia.test.ts
//
// These exercise the REAL rsync binary (paths.rsyncBin -> "rsync"), like
// controller/backupSavedVideos.test.ts. Everything happens inside one mkdtemp.
+//
+// RELEASE 17: the unit of a move is `channels/<slug>/media` (the media tier),
+// never `data/`. A seeded channel is CLASSIC — its big files real in
+// `data/<id>/` — and the move's preflight tiers it; the crash cases build the
+// tiered state first (`tierInPlace`) and then the exact on-disk state of the
+// crash.
const MTIME = new Date("2021-03-04T05:06:07.000Z");
@@ -132,24 +139,36 @@ async function seed(
const file = path.join(videoDir, name);
await writeFile(file, content);
// A known mtime, so "rsync -a preserved it" is an assertion and not a
- // coincidence. The LMDB index keys on mtimes, so this is what makes a
- // relocation free of a reindex.
+ // coincidence.
await utimes(file, MTIME, MTIME);
}
}
return channelDir;
}
-test("out: copies, links, records the target, keeps mtimes and reclaims the source", { skip: T1_SKIP }, async () => {
+// A classic channel tiered in place: `media/` a real directory on the corpus
+// disk, every big file in it and a relative link left in `data/<id>/` — what a
+// move's preflight does first, and the state every crash case starts from.
+async function tierInPlace(paths: Paths, slug: string): Promise<string> {
+ await tierChannelMedia(paths, slug, { createMediaDir: true });
+ return path.join(paths.channelsDir, slug, "media");
+}
+
+test("out: copies, links, records the target, keeps mtimes and reclaims the source", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
- v1: { "audio.m4a": "one".repeat(500), "transcript.json": "{}" },
- v2: { "audio.m4a": "two".repeat(500) },
+ v1: {
+ "audio.m4a": "one".repeat(500),
+ "transcript.json": "{}",
+ "transcript.live_chat.json": "[]",
+ },
+ v2: { "audio.m4a": "two".repeat(500), "source-media.mp4": "container" },
});
- const target = relocatedDataDir(root, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ const mediaLink = path.join(channelDir, "media");
const res = await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -158,36 +177,50 @@ test("out: copies, links, records the target, keeps mtimes and reclaims the sour
});
assert.equal(res.direction, "out");
assert.equal(res.target, target);
+ // The two audio files and the raw live chat — the media tier. The text
+ // and the persisted container are not in it.
assert.equal(res.files, 3);
+ assert.equal(res.tiered, 3, "the classic channel was tiered first");
assert.equal(res.resumed, false);
- // data/ is an absolute symlink at the target...
- const link = await lstat(path.join(channelDir, "data"));
- assert.ok(link.isSymbolicLink());
- assert.equal(await readlink(path.join(channelDir, "data")), target);
-
- // ...so the on-disk contract channelDir/data/<id>/<file> still resolves.
- assert.equal(
- await readFile(
- path.join(channelDir, "data", "v1", "transcript.json"),
- "utf8",
- ),
- "{}",
+ // `media` is ONE absolute symlink at the target...
+ assert.ok((await lstat(mediaLink)).isSymbolicLink());
+ assert.equal(await readlink(mediaLink), target);
+ // ...`data/` stays a real directory on the corpus disk...
+ assert.ok((await lstat(path.join(channelDir, "data"))).isDirectory());
+ assert.ok(!(await lstat(path.join(channelDir, "data"))).isSymbolicLink());
+ // ...each big file is a RELATIVE link into `media/` that resolves through it.
+ const audio = path.join(channelDir, "data", "v1", "audio.m4a");
+ assert.ok((await lstat(audio)).isSymbolicLink());
+ assert.equal(await readlink(audio), tierLinkTarget("v1", "audio.m4a"));
+ assert.equal(await readFile(audio, "utf8"), "one".repeat(500));
+ assert.ok(
+ (await lstat(
+ path.join(channelDir, "data", "v1", "transcript.live_chat.json"),
+ )).isSymbolicLink(),
);
+ // ...and the text, and what is not tierable, never left.
+ for (const real of [["v1", "transcript.json"], ["v2", "source-media.mp4"]]) {
+ const st = await lstat(path.join(channelDir, "data", ...real));
+ assert.ok(st.isFile() && !st.isSymbolicLink(), real.join("/"));
+ }
+ await assert.rejects(() => stat(path.join(target, "v1", "transcript.json")));
assert.equal(
(await stat(path.join(target, "v1", "audio.m4a"))).mtime.getTime(),
MTIME.getTime(),
);
// The record is in config.json, written only now, on success.
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, target);
+ const config = await readChannelConfig(paths, "alpha");
+ assert.equal(config?.mediaDir, target);
+ assert.equal(config?.dataDir, undefined);
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
// The parked copy is gone and the marker with it — this is the step that
// actually frees the source volume.
const left = await readdir(channelDir);
assert.deepEqual(
- left.filter((n) => n.startsWith("data.")),
+ left.filter((n) => n.startsWith("media.") || n.startsWith("data.")),
[],
);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
@@ -197,12 +230,30 @@ test("out: copies, links, records the target, keeps mtimes and reclaims the sour
});
});
-test("abort from an onLog hook leaves the source intact, and the rerun completes", { skip: T1_SKIP }, async () => {
+test("a channel already tiered in place moves without tiering anything", async () => {
+ await withTmp(async (paths, root) => {
+ await seed(paths, "alpha", { v1: { "audio.m4a": "one" } });
+ await tierInPlace(paths, "alpha");
+ const res = await relocateChannelMedia({
+ io: TEST_IO,
+ paths,
+ slug: "alpha",
+ direction: "out",
+ root,
+ onLog: () => {},
+ });
+ assert.equal(res.tiered, 0);
+ assert.equal(res.files, 1);
+ assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
+ });
+});
+
+test("abort from an onLog hook leaves the source intact, and the rerun completes", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
- v1: { "audio.m4a": "x".repeat(20000) },
+ v1: { "audio.m4a": "x".repeat(20000), "transcript.json": "{}" },
});
- const target = relocatedDataDir(root, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
// Aborting from the log hook is how a Cancel button reaches this code: the
// job's onLog and its AbortSignal belong to the same run. Firing on the
@@ -224,29 +275,35 @@ test("abort from an onLog hook leaves the source intact, and the rerun completes
/Cancelled/,
);
- // The source is a REAL directory still, with its file.
- assert.ok((await lstat(path.join(channelDir, "data"))).isDirectory());
+ // The source is a REAL `media/` still, with its file, reached through the
+ // link in `data/`.
+ assert.ok((await lstat(path.join(channelDir, "media"))).isDirectory());
+ assert.ok(!(await lstat(path.join(channelDir, "media"))).isSymbolicLink());
assert.equal(
(await readFile(path.join(channelDir, "data", "v1", "audio.m4a"), "utf8"))
.length,
20000,
);
- // Nothing was recorded, and the marker says where it was going.
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, undefined);
+ // Nothing was recorded, and the marker says where it was going — and that
+ // it moves the media only.
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, undefined);
const marker = await readRelocationMarker(paths, "alpha");
assert.equal(marker?.target, target);
assert.equal(marker?.direction, "out");
assert.equal(marker?.phase, "copy");
- // And every guard now reads the channel as in transition.
- assert.equal(
- (await inspectChannelMedia(paths, "alpha")).status,
- "in-transition",
- );
+ assert.equal(marker?.scope, "media");
+ // Every media guard now reads the channel as in transition; its text stays
+ // readable.
+ const during = await inspectChannelMedia(paths, "alpha", undefined, {
+ fresh: true,
+ });
+ assert.equal(during.status, "in-transition");
+ assert.equal(during.text.readable, true);
// The rerun finishes it. It is the ONE caller allowed to look past its own
// marker.
const res = await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -255,7 +312,7 @@ test("abort from an onLog hook leaves the source intact, and the rerun completes
});
assert.equal(res.resumed, true);
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, target);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, target);
});
});
@@ -269,7 +326,7 @@ test("a stray file on the destination is removed by the mirror pass", async () =
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one" },
});
- const target = relocatedDataDir(root, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
await mkdir(path.join(target, "v9"), { recursive: true });
await writeFile(path.join(target, "v9", "stray.m4a"), "not ours");
@@ -287,7 +344,7 @@ test("a stray file on the destination is removed by the mirror pass", async () =
await readFile(path.join(target, "v1", "audio.m4a"), "utf8"),
"one",
);
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, target);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, target);
assert.deepEqual(await leftoverCopies(channelDir), []);
});
});
@@ -315,7 +372,7 @@ test("a verify failure keeps the source and does not write the config", async ()
if (m.startsWith("$ ") && m.includes("--dry-run")) {
n++;
writeFileSync(
- path.join(channelDir, "data", "v1", `chunk-${n}.json`),
+ path.join(channelDir, "media", "v1", `chunk-${n}.json`),
"{}",
);
}
@@ -324,24 +381,26 @@ test("a verify failure keeps the source and does not write the config", async ()
/Verification failed: after a second mirror pass .*1 missing on the destination \(v1\/chunk-2\.json\)/,
);
- assert.ok((await lstat(path.join(channelDir, "data"))).isDirectory());
+ assert.ok((await lstat(path.join(channelDir, "media"))).isDirectory());
+ assert.ok(!(await lstat(path.join(channelDir, "media"))).isSymbolicLink());
assert.equal(
await readFile(path.join(channelDir, "data", "v1", "audio.m4a"), "utf8"),
"one",
);
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, undefined);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, undefined);
assert.equal((await readRelocationMarker(paths, "alpha"))?.phase, "copy");
});
});
-test("back: restores a real directory, clears the config and reclaims the target", { skip: T1_SKIP }, async () => {
+test("back: restores a real directory, clears the config and reclaims the target", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one", "transcript.json": "{}" },
});
- const target = relocatedDataDir(root, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ const mediaLink = path.join(channelDir, "media");
await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -350,36 +409,48 @@ test("back: restores a real directory, clears the config and reclaims the target
});
const res = await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "back",
onLog: () => {},
});
assert.equal(res.direction, "back");
- assert.equal(res.files, 2);
+ assert.equal(res.files, 1);
+ assert.equal(res.tiered, 0);
- const back = await lstat(path.join(channelDir, "data"));
+ // `media/` is a REAL directory on the corpus disk again, and the per-file
+ // link was never touched: it resolves there now (no "untier").
+ const back = await lstat(mediaLink);
assert.ok(back.isDirectory());
assert.ok(!back.isSymbolicLink());
- assert.equal(
- await readFile(path.join(channelDir, "data", "v1", "audio.m4a"), "utf8"),
- "one",
- );
- assert.equal(
- (await stat(path.join(channelDir, "data", "v1", "audio.m4a"))).mtime.getTime(),
- MTIME.getTime(),
- );
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, undefined);
+ const audio = path.join(channelDir, "data", "v1", "audio.m4a");
+ assert.ok((await lstat(audio)).isSymbolicLink());
+ assert.equal(await readlink(audio), tierLinkTarget("v1", "audio.m4a"));
+ assert.equal(await readFile(audio, "utf8"), "one");
+ assert.equal((await stat(audio)).mtime.getTime(), MTIME.getTime());
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, undefined);
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "in-place");
// The target is reclaimed, and so is <root>/<slug> once it is empty.
assert.equal(await pathThere(target), false);
assert.equal(await pathThere(path.dirname(target)), false);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
assert.equal(
- (await readdir(channelDir)).includes("data.incoming"),
+ (await readdir(channelDir)).includes("media.incoming"),
false,
);
+
+ // And the next move out has nothing to tier.
+ const again = await relocateChannelMedia({
+ io: TEST_IO,
+ paths,
+ slug: "alpha",
+ direction: "out",
+ root,
+ onLog: () => {},
+ });
+ assert.equal(again.tiered, 0);
+ assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
});
});
@@ -404,7 +475,7 @@ test("a relocated channel is refused a second move without a move back", async (
await withTmp(async (paths, root, dir) => {
await seed(paths, "alpha", { v1: { "audio.m4a": "one" } });
await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -445,6 +516,11 @@ test("an unmounted root is refused before anything is written", async () => {
);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
assert.ok((await lstat(path.join(channelDir, "data"))).isDirectory());
+ // Not even tiered: the root is refused before the preflight runs.
+ assert.equal(await pathIsThere(path.join(channelDir, "media")), false);
+ assert.ok(
+ (await lstat(path.join(channelDir, "data", "v1", "audio.m4a"))).isFile(),
+ );
});
});
@@ -466,22 +542,137 @@ test("a social channel has no media to relocate", async () => {
});
});
-test("preview measures the tree and both volumes without moving anything", async () => {
+test("preview tiers a classic channel first, measures media/ and both volumes, and moves nothing", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "abcde", "transcript.json": "{}" },
v2: { "audio.m4a": "fgh" },
});
const preview = await previewRelocation({ paths, slug: "alpha", root });
- assert.equal(preview.target, relocatedDataDir(root, "alpha"));
- assert.equal(preview.files, 3);
- assert.equal(preview.bytesToMove, 5 + 2 + 3);
+ assert.equal(preview.target, relocatedMediaDir(root, "alpha"));
+ // The media tier only: two audio files, not the transcript.
+ assert.equal(preview.files, 2);
+ assert.equal(preview.bytesToMove, 5 + 3);
+ assert.equal(preview.tieredFirst, 2);
assert.equal(preview.existingPartial, false);
assert.ok(preview.freeOnRoot > 0);
assert.ok(preview.freeOnSource > 0);
// Both dirs are under one tmpdir, so this really is the same volume.
assert.equal(preview.sameDevice, true);
+ // Tiered in place — `media/` a real directory, the files linked — and
+ // nothing moved off the corpus disk.
assert.ok((await lstat(path.join(channelDir, "data"))).isDirectory());
+ assert.ok(!(await lstat(path.join(channelDir, "media"))).isSymbolicLink());
+ assert.ok(
+ (await lstat(path.join(channelDir, "data", "v1", "audio.m4a"))).isSymbolicLink(),
+ );
+ assert.equal(await pathThere(path.join(root, "alpha")), false);
+ assert.equal(await readRelocationMarker(paths, "alpha"), null);
+ assert.equal(
+ (await inspectChannelMedia(paths, "alpha", undefined, { fresh: true })).status,
+ "in-place",
+ );
+
+ // Idempotent: a second preview tiers nothing and says the same figures.
+ const again = await previewRelocation({ paths, slug: "alpha", root });
+ assert.equal(again.tieredFirst, 0);
+ assert.equal(again.files, 2);
+ assert.equal(again.bytesToMove, 8);
+ });
+});
+
+// THE RETIRED LAYOUT (release 17): a channel whose whole `data/` was moved by
+// the old mover is `legacy`, and the mover refuses it — out, back and the
+// preview — with the sentence that names the migration. Nothing is touched.
+test("a legacy channel is refused by the job, the move back and the preview, naming migrate-tier", async () => {
+ await withTmp(async (paths, root, dir) => {
+ const channelDir = await seed(paths, "alpha", { v1: { "audio.m4a": "one" } });
+ const oldTarget = path.join(root, "alpha", "data");
+ await mkdir(path.dirname(oldTarget), { recursive: true });
+ await rename(path.join(channelDir, "data"), oldTarget);
+ await symlink(oldTarget, path.join(channelDir, "data"));
+ await setConfigField(paths, "alpha", "dataDir", oldTarget);
+ assert.equal(
+ (await inspectChannelMedia(paths, "alpha", undefined, { fresh: true })).status,
+ "legacy",
+ );
+ const other = path.join(dir, "platter2");
+ await mkdir(other, { recursive: true });
+
+ const refused = /cannot be moved: its media layout is the retired whole-directory one — run archilyzer storage migrate-tier alpha/;
+ await assert.rejects(
+ () =>
+ relocateChannelMedia({
+ io: TEST_IO,
+ paths,
+ slug: "alpha",
+ direction: "out",
+ root: other,
+ onLog: () => {},
+ }),
+ refused,
+ );
+ await assert.rejects(
+ () =>
+ relocateChannelMedia({
+ io: TEST_IO,
+ paths,
+ slug: "alpha",
+ direction: "back",
+ onLog: () => {},
+ }),
+ refused,
+ );
+ await assert.rejects(
+ () => previewRelocation({ paths, slug: "alpha", root: other }),
+ refused,
+ );
+ // Untouched: no marker, no `media`, the old link and its bytes as they were.
+ assert.equal(await readRelocationMarker(paths, "alpha"), null);
+ assert.equal(await pathIsThere(path.join(channelDir, "media")), false);
+ assert.equal(await readlink(path.join(channelDir, "data")), oldTarget);
+ assert.equal(
+ await readFile(path.join(oldTarget, "v1", "audio.m4a"), "utf8"),
+ "one",
+ );
+ assert.equal(await pathThere(path.join(other, "alpha")), false);
+ });
+});
+
+test("a tier migration's marker is never resumed or replaced by the mover", async () => {
+ await withTmp(async (paths, root) => {
+ await seed(paths, "alpha", { v1: { "audio.m4a": "one" } });
+ const target = relocatedMediaDir(root, "alpha");
+ await seedMarker(paths, "alpha", {
+ target,
+ direction: "out",
+ phase: "copy",
+ scope: "tier-migration",
+ });
+ for (const direction of ["out", "back"] as const) {
+ await assert.rejects(
+ () =>
+ relocateChannelMedia({
+ io: TEST_IO,
+ paths,
+ slug: "alpha",
+ direction,
+ root,
+ onLog: () => {},
+ }),
+ /media-tier migration in flight or interrupted .* archilyzer storage migrate-tier alpha/,
+ );
+ }
+ // The preview gives the same sentence, and tiers nothing (review N9).
+ await assert.rejects(
+ () => previewRelocation({ paths, slug: "alpha", root }),
+ /media-tier migration in flight or interrupted .* archilyzer storage migrate-tier alpha/,
+ );
+ assert.equal(
+ await pathIsThere(path.join(paths.channelsDir, "alpha", "media")),
+ false,
+ );
+ assert.equal((await readRelocationMarker(paths, "alpha"))?.scope, "tier-migration");
});
});
@@ -506,8 +697,8 @@ async function pathThere(p: string): Promise<boolean> {
//
// Each seeds the exact on-disk state of a crash at that point and asserts the
// rerun reaches a clean `ok` / `in-place` — with the config right, the marker
-// gone and NO `data.relocated-*` or `data.incoming` left holding a second copy
-// on the volume the move exists to free.
+// gone and NO `media.relocated-*` or `media.incoming` left holding a second
+// copy on the volume the move exists to free.
async function seedMarker(
paths: Paths,
@@ -516,6 +707,7 @@ async function seedMarker(
target: string;
direction: "out" | "back";
phase: "copy" | "swap" | "reclaim";
+ scope?: "media" | "tier-migration";
},
): Promise<void> {
await writeFile(
@@ -524,9 +716,10 @@ async function seedMarker(
);
}
-async function setConfigDataDir(
+async function setConfigField(
paths: Paths,
slug: string,
+ key: "mediaDir" | "dataDir",
value: string | null,
): Promise<void> {
const file = path.join(paths.channelsDir, slug, "config.json");
@@ -534,20 +727,28 @@ async function setConfigDataDir(
string,
unknown
>;
- if (value === null) delete config.dataDir;
- else config.dataDir = value;
+ if (value === null) delete config[key];
+ else config[key] = value;
await writeFile(file, JSON.stringify(config, null, 2) + "\n");
}
+async function setConfigMediaDir(
+ paths: Paths,
+ slug: string,
+ value: string | null,
+): Promise<void> {
+ await setConfigField(paths, slug, "mediaDir", value);
+}
+
async function leftoverCopies(channelDir: string): Promise<string[]> {
return (await readdir(channelDir))
- .filter((n) => n.startsWith("data.relocated-") || n === "data.incoming")
+ .filter((n) => n.startsWith("media.relocated-") || n === "media.incoming")
.sort();
}
// rsync -a of the tree, so the seeded "already copied" target is byte-for-byte
// what a completed copy phase would have left — including mtimes, which the
-// re-verify compares.
+// re-verify compares. Over `media/` only: it holds no links.
async function copyTree(src: string, dest: string): Promise<void> {
await mkdir(dest, { recursive: true });
for (const entry of await readdir(src, { withFileTypes: true })) {
@@ -562,18 +763,35 @@ async function copyTree(src: string, dest: string): Promise<void> {
}
}
-test("out @ swap: crash before the rename — the rerun re-verifies and completes", { skip: T1_SKIP }, async () => {
+// A COMPLETED MOVE OUT, built by hand: tiered, `media/` copied to the target
+// and replaced by the absolute link, `mediaDir` recorded.
+async function seedRelocated(
+ paths: Paths,
+ slug: string,
+ root: string,
+): Promise<{ mediaLink: string; target: string }> {
+ const mediaLink = await tierInPlace(paths, slug);
+ const target = relocatedMediaDir(root, slug);
+ await copyTree(mediaLink, target);
+ await rm(mediaLink, { recursive: true, force: true });
+ await symlink(target, mediaLink);
+ await setConfigMediaDir(paths, slug, target);
+ return { mediaLink, target };
+}
+
+test("out @ swap: crash before the rename — the rerun re-verifies and completes", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
});
- const target = relocatedDataDir(root, "alpha");
- // The copy finished; the process died before `data/` was parked.
- await copyTree(path.join(channelDir, "data"), target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ // The copy finished; the process died before `media/` was parked.
+ await copyTree(mediaLink, target);
await seedMarker(paths, "alpha", { target, direction: "out", phase: "swap" });
const res = await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -582,33 +800,33 @@ test("out @ swap: crash before the rename — the rerun re-verifies and complete
});
assert.equal(res.resumed, true);
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, target);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, target);
assert.deepEqual(await leftoverCopies(channelDir), []);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
});
});
-test("out @ swap: crash after the config write — the first run's parked copy is still reclaimed", { skip: T1_SKIP }, async () => {
+test("out @ swap: crash after the config write — the first run's parked copy is still reclaimed", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
- await copyTree(dataDir, target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
// Everything through writeChannelConfig ran; the reclaim marker never
- // landed. A full second copy of the channel is parked on the source volume
- // — the disk the whole move exists to free — and the marker still says
+ // landed. A full second copy of the media is parked on the source volume —
+ // the disk the whole move exists to free — and the marker still says
// "swap", so the old code minted a SECOND parked name and reclaimed only
// that one, orphaning this forever.
- const parked = path.join(channelDir, "data.relocated-1700000000000");
- await rename(dataDir, parked);
- await symlink(target, dataDir);
- await setConfigDataDir(paths, "alpha", target);
+ const parked = path.join(channelDir, "media.relocated-1700000000000");
+ await rename(mediaLink, parked);
+ await symlink(target, mediaLink);
+ await setConfigMediaDir(paths, "alpha", target);
await seedMarker(paths, "alpha", { target, direction: "out", phase: "swap" });
await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -618,25 +836,25 @@ test("out @ swap: crash after the config write — the first run's parked copy i
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
assert.deepEqual(await leftoverCopies(channelDir), []);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
- // The media itself is untouched by the sweep.
+ // The media itself is untouched by the sweep, and reached from `data/`.
assert.equal(
- await readFile(path.join(dataDir, "v1", "audio.m4a"), "utf8"),
+ await readFile(path.join(channelDir, "data", "v1", "audio.m4a"), "utf8"),
"one".repeat(500),
);
});
});
-test("out @ reclaim: the rerun sweeps every parked copy and clears the marker", { skip: T1_SKIP }, async () => {
+test("out @ reclaim: the rerun sweeps every parked copy and clears the marker", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
- await copyTree(dataDir, target);
- await rename(dataDir, path.join(channelDir, "data.relocated-1700000000000"));
- await symlink(target, dataDir);
- await setConfigDataDir(paths, "alpha", target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
+ await rename(mediaLink, path.join(channelDir, "media.relocated-1700000000000"));
+ await symlink(target, mediaLink);
+ await setConfigMediaDir(paths, "alpha", target);
await seedMarker(paths, "alpha", {
target,
direction: "out",
@@ -644,7 +862,7 @@ test("out @ reclaim: the rerun sweeps every parked copy and clears the marker",
});
await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -661,45 +879,41 @@ test("out @ reclaim: the rerun sweeps every parked copy and clears the marker",
// {swap(before), swap(after), reclaim} all had cases, and back @ copy had none —
// the only one where the rerun has to finish an rsync rather than a rename, and
// the only one where the space check is asked to credit a PARTIAL copy.
-test("back @ copy: a half-copied data.incoming is resumed, not restarted", async () => {
+test("back @ copy: a half-copied media.incoming is resumed, not restarted", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500), "transcript.json": "{}" },
v2: { "audio.m4a": "two".repeat(500) },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
- await copyTree(dataDir, target);
- await rm(dataDir, { recursive: true, force: true });
- await symlink(target, dataDir);
- await setConfigDataDir(paths, "alpha", target);
+ const { mediaLink, target } = await seedRelocated(paths, "alpha", root);
// The copy got one video in and died. The marker says `copy`, so the rerun
// resumes the rsync — and the surviving file keeps its mtime, which is what
// says rsync skipped it rather than re-sending it.
- const incoming = path.join(channelDir, "data.incoming");
+ const incoming = path.join(channelDir, "media.incoming");
await copyTree(path.join(target, "v1"), path.join(incoming, "v1"));
await seedMarker(paths, "alpha", { target, direction: "back", phase: "copy" });
const res = await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "back",
onLog: () => {},
});
assert.equal(res.resumed, true);
- assert.equal(res.files, 3);
+ assert.equal(res.files, 2);
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "in-place");
- assert.ok((await lstat(dataDir)).isDirectory());
+ assert.ok((await lstat(mediaLink)).isDirectory());
+ assert.ok(!(await lstat(mediaLink)).isSymbolicLink());
assert.equal(
- await readFile(path.join(dataDir, "v2", "audio.m4a"), "utf8"),
+ await readFile(path.join(channelDir, "data", "v2", "audio.m4a"), "utf8"),
"two".repeat(500),
);
assert.equal(
- (await stat(path.join(dataDir, "v1", "audio.m4a"))).mtime.getTime(),
+ (await stat(path.join(mediaLink, "v1", "audio.m4a"))).mtime.getTime(),
MTIME.getTime(),
);
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, undefined);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, undefined);
assert.equal(await pathThere(target), false);
assert.deepEqual(await leftoverCopies(channelDir), []);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
@@ -711,18 +925,13 @@ test("back @ swap: crash before the rename — the rerun finishes the swap", asy
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
// A completed relocation, then a move-back whose copy finished.
- await copyTree(dataDir, target);
- await rm(dataDir, { recursive: true, force: true });
- await symlink(target, dataDir);
- await setConfigDataDir(paths, "alpha", target);
- await copyTree(target, path.join(channelDir, "data.incoming"));
+ const { mediaLink, target } = await seedRelocated(paths, "alpha", root);
+ await copyTree(target, path.join(channelDir, "media.incoming"));
await seedMarker(paths, "alpha", { target, direction: "back", phase: "swap" });
const res = await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "back",
@@ -730,8 +939,9 @@ test("back @ swap: crash before the rename — the rerun finishes the swap", asy
});
assert.equal(res.resumed, true);
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "in-place");
- assert.ok((await lstat(dataDir)).isDirectory());
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, undefined);
+ assert.ok((await lstat(mediaLink)).isDirectory());
+ assert.ok(!(await lstat(mediaLink)).isSymbolicLink());
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, undefined);
assert.deepEqual(await leftoverCopies(channelDir), []);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
});
@@ -742,25 +952,25 @@ test("back @ swap: crash AFTER the rename — the rerun does not ENOENT forever"
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
- await copyTree(dataDir, target);
- // rename(incoming, data) committed; the process died before the config was
- // cleared. `data/` is a real directory and `incoming` is gone — so the old
- // sequence unlinked nothing (swallowed), then renamed a path that no longer
- // exists, and failed identically on every rerun.
- await setConfigDataDir(paths, "alpha", target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
+ // rename(incoming, media) committed; the process died before the config
+ // was cleared. `media/` is a real directory and `incoming` is gone — so the
+ // old sequence unlinked nothing (swallowed), then renamed a path that no
+ // longer exists, and failed identically on every rerun.
+ await setConfigMediaDir(paths, "alpha", target);
await seedMarker(paths, "alpha", { target, direction: "back", phase: "swap" });
await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "back",
onLog: () => {},
});
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "in-place");
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, undefined);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, undefined);
assert.equal(await pathIsThere(target), false);
assert.deepEqual(await leftoverCopies(channelDir), []);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
@@ -772,11 +982,13 @@ test("back @ reclaim: the config is already clear, and the rerun still finishes"
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
});
- const target = relocatedDataDir(root, "alpha");
- // The swap completed and cleared config.dataDir — which is why this case
- // was UNREACHABLE: with no dataDir the entry point threw "is not relocated"
- // and the marker named the only place that knew where the media had been.
- await copyTree(path.join(channelDir, "data"), target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ // The swap completed and cleared config.mediaDir — which is why this case
+ // was UNREACHABLE: with no mediaDir the entry point threw "is not
+ // relocated" and the marker named the only place that knew where the media
+ // had been.
+ await copyTree(mediaLink, target);
await seedMarker(paths, "alpha", {
target,
direction: "back",
@@ -784,7 +996,7 @@ test("back @ reclaim: the config is already clear, and the rerun still finishes"
});
await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "back",
@@ -800,12 +1012,12 @@ test("back @ reclaim: the config is already clear, and the rerun still finishes"
test("a marker for the other direction is never resumed into", async () => {
await withTmp(async (paths, root) => {
await seed(paths, "alpha", { v1: { "audio.m4a": "one" } });
- const target = relocatedDataDir(root, "alpha");
- await setConfigDataDir(paths, "alpha", target);
+ const target = relocatedMediaDir(root, "alpha");
+ await setConfigMediaDir(paths, "alpha", target);
await seedMarker(paths, "alpha", { target, direction: "out", phase: "swap" });
await assert.rejects(
relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "back",
@@ -830,31 +1042,32 @@ async function pathIsThere(p: string): Promise<boolean> {
// UI and both ended with rm -r on the only copy, so both get their own cases.
// ---------------------------------------------------------------------------
-// `inconsistent` — config.json records a target while `data/` is a REAL
+// `inconsistent` — config.json records a target while `media/` is a REAL
// directory — is what `rsync --copy-links` of a channel produces, which
// WORKTREES.md documents as the way to carry media into a shard. inspect()
// reports relocated:true for it, so the panel offered "Move back in place", and
-// the run copied the target to data.incoming, verified it, found data/ already a
-// real dir, logged "the swap had completed", cleared the config, rm -r'd the
-// target and then swept data.incoming. Three copies in, zero out.
-test("back: an inconsistent channel is refused, and the target keeps its bytes", { skip: T1_SKIP }, async () => {
+// the run copied the target to the incoming dir, verified it, found the live
+// dir already real, logged "the swap had completed", cleared the config, rm -r'd
+// the target and then swept the incoming copy. Three copies in, zero out.
+test("back: an inconsistent channel is refused, and the target keeps its bytes", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one" },
});
- const target = relocatedDataDir(root, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ const mediaLink = path.join(channelDir, "media");
await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
root,
onLog: () => {},
});
- // Turn the link back into a real directory WITHOUT clearing config.dataDir —
- // the --copy-links shape, reproduced.
- await rm(path.join(channelDir, "data"));
- await copyTree(target, path.join(channelDir, "data"));
+ // Turn the link back into a real directory WITHOUT clearing
+ // config.mediaDir — the --copy-links shape, reproduced.
+ await rm(mediaLink);
+ await copyTree(target, mediaLink);
assert.equal(
(await inspectChannelMedia(paths, "alpha")).status,
"inconsistent",
@@ -882,21 +1095,21 @@ test("back: an inconsistent channel is refused, and the target keeps its bytes",
await readFile(path.join(channelDir, "data", "v1", "audio.m4a"), "utf8"),
"one",
);
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, target);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, target);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
- assert.equal((await readdir(channelDir)).includes("data.incoming"), false);
+ assert.equal((await readdir(channelDir)).includes("media.incoming"), false);
});
});
// An `unreachable` channel (the drive is not mounted) is refused for the same
// reason: nothing can vouch for what the target holds, and the run ends by
// deleting it.
-test("back: an unreachable channel is refused", { skip: T1_SKIP }, async () => {
+test("back: an unreachable channel is refused", async () => {
await withTmp(async (paths, root) => {
await seed(paths, "alpha", { v1: { "audio.m4a": "one" } });
- const target = relocatedDataDir(root, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -905,10 +1118,10 @@ test("back: an unreachable channel is refused", { skip: T1_SKIP }, async () => {
});
// The platter goes away. The link dangles; config still names the target.
await rm(path.dirname(target), { recursive: true, force: true });
- assert.equal(
- (await inspectChannelMedia(paths, "alpha")).status,
- "unreachable",
- );
+ const away = await inspectChannelMedia(paths, "alpha");
+ assert.equal(away.status, "unreachable");
+ // The text does not care.
+ assert.equal(away.text.readable, true);
await assert.rejects(
() =>
relocateChannelMedia({
@@ -925,7 +1138,7 @@ test("back: an unreachable channel is refused", { skip: T1_SKIP }, async () => {
// THE SELF-RELOCATION. root = <transcriptsDir>/channels makes the target the
// source: rsync src/ src/ succeeds, verifyCopy compares the tree with itself,
-// the swap parks `data/` (which IS the verified "target") and links to a path
+// the swap parks `media/` (which IS the verified "target") and links to a path
// that no longer exists, and the reclaim sweep deletes the only copy. The
// panel's own help text describes transcripts/channels almost word for word.
test("a root inside the corpus is refused by the job and by the preview", async () => {
@@ -944,7 +1157,7 @@ test("a root inside the corpus is refused by the job and by the preview", async
await assert.rejects(
() =>
relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -960,16 +1173,22 @@ test("a root inside the corpus is refused by the job and by the preview", async
`preview accepted ${bad}`,
);
}
- // Untouched: still a real directory, no config, no marker, no parked copy.
+ // Untouched: still a real file in a real directory (not even tiered), no
+ // config, no marker, no parked copy.
assert.ok((await lstat(path.join(channelDir, "data"))).isDirectory());
+ assert.ok(
+ (await lstat(path.join(channelDir, "data", "v1", "audio.m4a"))).isFile(),
+ );
assert.equal(
await readFile(path.join(channelDir, "data", "v1", "audio.m4a"), "utf8"),
"one",
);
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, undefined);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, undefined);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
assert.equal(
- (await readdir(channelDir)).filter((n) => n.startsWith("data.")).length,
+ (await readdir(channelDir)).filter(
+ (n) => n.startsWith("media") || n.startsWith("data."),
+ ).length,
0,
);
});
@@ -1015,24 +1234,24 @@ test("a relative root is refused by the preview, not only by the job", async ()
// THE OMNIMIRROR REFUSAL (2026-09-13), in a tmpdir.
//
// A 131 GB copy landed byte-complete and the verify refused it. The whole of the
-// drift was one directory timestamp: a sidecar written into `data/v4p31nz/`
-// while rsync was already past that directory bumped the SOURCE directory's
-// mtime and left the target's behind. `.d..t` is rsync's itemization for
-// exactly that — a directory, and only its time differs — and it means nothing
-// about the bytes.
+// drift was one directory timestamp: a sidecar written into a video dir while
+// rsync was already past that directory bumped the SOURCE directory's mtime and
+// left the target's behind. `.d..t` is rsync's itemization for exactly that — a
+// directory, and only its time differs — and it means nothing about the bytes.
//
// Both cases run at phase `swap`, where the re-verify runs on its own with no
// `rsync -a` ahead of it. That is the shape that can see the difference: in the
// copy phase the transfer itself would have set the timestamps.
-test("out @ swap: a directory timestamp is settled by one more pass, not refused", { skip: T1_SKIP }, async () => {
+test("out @ swap: a directory timestamp is settled by one more pass, not refused", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500), "transcript.json": "{}" },
v2: { "audio.m4a": "two".repeat(500) },
});
- const target = relocatedDataDir(root, "alpha");
- await copyTree(path.join(channelDir, "data"), target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
// The echo of the sidecar: the target's copy of v1/ carries a different
// mtime from the source's. Every file underneath is identical.
await utimes(path.join(target, "v1"), MTIME, MTIME);
@@ -1040,7 +1259,7 @@ test("out @ swap: a directory timestamp is settled by one more pass, not refused
const lines: string[] = [];
const res = await relocateChannelMedia({
- io: TEST_IO,
+ io: TEST_IO,
paths,
slug: "alpha",
direction: "out",
@@ -1056,31 +1275,35 @@ test("out @ swap: a directory timestamp is settled by one more pass, not refused
);
// And the move completed: link, config, no leftovers, no marker.
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, target);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, target);
assert.deepEqual(await leftoverCopies(channelDir), []);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
assert.equal(
- await readFile(path.join(target, "v1", "transcript.json"), "utf8"),
+ await readFile(path.join(channelDir, "data", "v1", "audio.m4a"), "utf8"),
+ "one".repeat(500),
+ );
+ assert.equal(
+ await readFile(path.join(channelDir, "data", "v1", "transcript.json"), "utf8"),
"{}",
);
});
});
// CONTENT drift at the swap's re-verify, arriving the way the timestamp did —
-// a sidecar written into the source after the copy. Until release 16 slice RM
-// any file line refused; now the re-verify runs in mirror mode while `data/` is
-// still the live directory, and one change gets one more mirror pass, exactly
-// as in the copy phase. A second change is still a refusal ("a verify failure
-// keeps the source …" above).
-test("out @ swap: a file the target is missing is mirrored by one more pass", { skip: T1_SKIP }, async () => {
+// a file written into the source after the copy. Until release 16 slice RM
+// any file line refused; now the re-verify runs in mirror mode while `media/`
+// is still the live directory, and one change gets one more mirror pass,
+// exactly as in the copy phase. A second change is still a refusal ("a verify
+// failure keeps the source …" above).
+test("out @ swap: a file the target is missing is mirrored by one more pass", async () => {
await withTmp(async (paths, root) => {
- const channelDir = await seed(paths, "alpha", {
+ await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
- await copyTree(dataDir, target);
- await writeFile(path.join(dataDir, "v1", "diarization.json"), "{}");
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
+ await writeFile(path.join(mediaLink, "v1", "transcript.live_chat.json"), "[]");
await seedMarker(paths, "alpha", { target, direction: "out", phase: "swap" });
const res = await relocateChannelMedia({
@@ -1093,16 +1316,16 @@ test("out @ swap: a file the target is missing is mirrored by one more pass", {
});
assert.equal(res.retried, true);
assert.equal(
- await readFile(path.join(target, "v1", "diarization.json"), "utf8"),
- "{}",
+ await readFile(path.join(target, "v1", "transcript.live_chat.json"), "utf8"),
+ "[]",
);
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
- assert.equal((await readChannelConfig(paths, "alpha"))?.dataDir, target);
+ assert.equal((await readChannelConfig(paths, "alpha"))?.mediaDir, target);
});
});
// A PARKED COPY IS NEVER MIRRORED FROM. Once the rename has committed, the
-// link is what readers follow and `data.relocated-*` is not live media any
+// link is what readers follow and `media.relocated-*` is not live media any
// more: the re-verify against it is the strict one, as it always was — a
// difference refuses, and nothing on the target is deleted to match a stale
// copy.
@@ -1111,12 +1334,12 @@ test("out @ swap: after the rename, the re-verify against the parked copy is str
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
- await copyTree(dataDir, target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
// A file on the target the parked copy lacks.
await writeFile(path.join(target, "v1", "newer.json"), "{}");
- await rename(dataDir, path.join(channelDir, "data.relocated-1"));
+ await rename(mediaLink, path.join(channelDir, "media.relocated-1"));
await seedMarker(paths, "alpha", { target, direction: "out", phase: "swap" });
await assert.rejects(
@@ -1155,8 +1378,8 @@ const TRANSCRIBING: ChannelWriter = {
// THE REFUSAL OVER A RUNNING JOB, the job's first step. The registry is
// injected: what is pinned is that the move asks, refuses naming the writer,
-// and has touched nothing — no marker, no target directory, the source as it
-// was.
+// and has touched nothing — no marker, no target directory, no tiering, the
+// source as it was.
test("a move refuses to start over a running job, naming it, and touches nothing", async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
@@ -1178,6 +1401,10 @@ test("a move refuses to start over a running job, naming it, and touches nothing
assert.equal(await readRelocationMarker(paths, "alpha"), null);
await assert.rejects(() => stat(path.join(root, "alpha")));
assert.ok((await lstat(path.join(channelDir, "data"))).isDirectory());
+ assert.equal(await pathIsThere(path.join(channelDir, "media")), false);
+ assert.ok(
+ (await lstat(path.join(channelDir, "data", "v1", "audio.m4a"))).isFile(),
+ );
});
});
@@ -1230,32 +1457,85 @@ test("a writer seen once the marker is written refuses, and a fresh move's marke
);
assert.equal(asked, 2);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
+ // Tiered in place (the preflight ran) and nothing else: an in-place channel.
assert.equal(
(await inspectChannelMedia(paths, "alpha", undefined, { fresh: true })).status,
"in-place",
);
assert.deepEqual(
- await readdir(path.join(relocatedDataDir(root, "alpha"))),
+ await readdir(path.join(relocatedMediaDir(root, "alpha"))),
[],
"nothing was copied",
);
assert.ok((await lstat(path.join(channelDir, "data"))).isDirectory());
+ assert.equal(
+ await readFile(path.join(channelDir, "data", "v1", "audio.m4a"), "utf8"),
+ "one",
+ );
});
});
+// A MOVE HOLDS ONLY THE MEDIA WRITERS (release 17 ruling): a digest — a job or
+// the digest lane's unit — reads and writes the text, which never moves, so it
+// is not a writer for this question. A running move of the channel IS one,
+// though its kind is not `needsMedia`: the preview and the panel must not
+// offer a second move over it.
+test("channelMediaWriters: media jobs, a move and the media lanes — never a digest", () => {
+ const job = (id: string, kind: string): JobRecord =>
+ ({
+ id,
+ kind,
+ queueKey: "q",
+ status: "running",
+ queuedAt: 1,
+ logPath: "/dev/null",
+ channelSlug: "alpha",
+ }) as JobRecord;
+ const jobs = [
+ job("J1", "digest-channel-local"),
+ job("J2", "whisper-all"),
+ job("J3", "relocate-channel-media"),
+ ];
+ const unit = (lane: "digest" | "transcription", videoId: string) => ({
+ lane,
+ unit: { videoId, leafId: "leaf", channelSlug: "alpha", startedAt: 1 },
+ });
+ const source = {
+ jobs: () => jobs,
+ units: () => [unit("digest", "d1"), unit("transcription", "t1")],
+ };
+ const names = (ws: ChannelWriter[]) =>
+ ws.map((w) => (w.source === "job" ? w.jobId : `${w.lane}:${w.videoId}`));
+ assert.deepEqual(names(channelMediaWriters("alpha", { source })), [
+ "J2",
+ "J3",
+ "transcription:t1",
+ ]);
+ // The move's own first step leaves itself out.
+ assert.deepEqual(
+ names(
+ channelMediaWriters("alpha", {
+ source,
+ ignoreKinds: ["relocate-channel-media"],
+ }),
+ ),
+ ["J2", "transcription:t1"],
+ );
+});
+
// THE 2026-10-01 CASE. A move killed mid-copy left a transcriber's scratch dir
// on the destination that has since gone from the source. The resume used to
// copy everything else and refuse on the counts (1755 against 1750), and no
// rerun could settle it; the mirror pass now does.
-test("a resume with a stale extra dir on the destination completes", { skip: T1_SKIP }, async () => {
+test("a resume with a stale extra dir on the destination completes", async () => {
await withTmp(async (paths, root) => {
- const channelDir = await seed(paths, "alpha", {
+ await seed(paths, "alpha", {
v50t5yt: { "audio.mp3": "a".repeat(64), "transcript.json": "{}" },
v51fpcd: { "audio.mp3": "b".repeat(64) },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
- await copyTree(dataDir, target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
const scratch = path.join(target, "v50t5yt", ".audio.mp3.parakeet");
await mkdir(scratch, { recursive: true });
for (const f of ["meta", "win-0000", "win-0001", "win-0002", "win-0003"]) {
@@ -1272,7 +1552,7 @@ test("a resume with a stale extra dir on the destination completes", { skip: T1_
onLog: () => {},
});
assert.equal(res.resumed, true);
- assert.equal(res.files, 3);
+ assert.equal(res.files, 2);
await assert.rejects(() => stat(scratch));
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
assert.equal(await readRelocationMarker(paths, "alpha"), null);
@@ -1292,10 +1572,10 @@ test("back: a stale extra on the copy coming home is removed on resume", async (
root,
onLog: () => {},
});
- const target = relocatedDataDir(root, "alpha");
- // An interrupted move back: `data.incoming` holds the copy, plus a file
+ const target = relocatedMediaDir(root, "alpha");
+ // An interrupted move back: `media.incoming` holds the copy, plus a file
// the target (the source of this direction) no longer has.
- const incoming = path.join(channelDir, "data.incoming");
+ const incoming = path.join(channelDir, "media.incoming");
await copyTree(target, incoming);
await writeFile(path.join(incoming, "v1", "gone.json"), "{}");
await seedMarker(paths, "alpha", { target, direction: "back", phase: "copy" });
@@ -1308,9 +1588,10 @@ test("back: a stale extra on the copy coming home is removed on resume", async (
onLog: () => {},
});
assert.equal(res.resumed, true);
- const dataDir = path.join(channelDir, "data");
- assert.ok((await lstat(dataDir)).isDirectory());
- assert.deepEqual(await readdir(path.join(dataDir, "v1")), ["audio.m4a"]);
+ const mediaLink = path.join(channelDir, "media");
+ assert.ok((await lstat(mediaLink)).isDirectory());
+ assert.ok(!(await lstat(mediaLink)).isSymbolicLink());
+ assert.deepEqual(await readdir(path.join(mediaLink, "v1")), ["audio.m4a"]);
assert.equal(await readRelocationMarker(paths, "alpha"), null);
});
});
@@ -1318,16 +1599,19 @@ test("back: a stale extra on the copy coming home is removed on resume", async (
// RECONCILE AND RESUME — the remediation (the ruling's last bullet). An extra
// file and a changed one on the destination: the job says what it found, by
// kind, makes the copy match the source and finishes the move.
-test("reconcile: an extra and a changed file on the destination are settled, and the move completes", { skip: T1_SKIP }, async () => {
+test("reconcile: an extra and a changed file on the destination are settled, and the move completes", async () => {
await withTmp(async (paths, root) => {
- const channelDir = await seed(paths, "alpha", {
- v1: { "audio.m4a": "one".repeat(100), "transcript.json": '{"v":2}' },
+ await seed(paths, "alpha", {
+ v1: {
+ "audio.m4a": "one".repeat(100),
+ "transcript.live_chat.json": '{"v":2}',
+ },
});
- const dataDir = path.join(channelDir, "data");
- const target = relocatedDataDir(root, "alpha");
- await copyTree(dataDir, target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
await writeFile(path.join(target, "v1", "stale.json"), "{}");
- await writeFile(path.join(target, "v1", "transcript.json"), '{"v":1}');
+ await writeFile(path.join(target, "v1", "transcript.live_chat.json"), '{"v":1}');
await seedMarker(paths, "alpha", { target, direction: "out", phase: "copy" });
const lines: string[] = [];
@@ -1344,13 +1628,13 @@ test("reconcile: an extra and a changed file on the destination are settled, and
assert.equal(res.reconciled, true);
const said = lines.find((l) => l.startsWith("Reconciling:")) ?? "";
assert.match(said, /1 extra on the destination \(v1\/stale\.json\)/);
- assert.match(said, /1 changed \(v1\/transcript\.json\)/);
+ assert.match(said, /1 changed \(v1\/transcript\.live_chat\.json\)/);
assert.deepEqual(
(await readdir(path.join(target, "v1"))).sort(),
- ["audio.m4a", "transcript.json"],
+ ["audio.m4a", "transcript.live_chat.json"],
);
assert.equal(
- await readFile(path.join(target, "v1", "transcript.json"), "utf8"),
+ await readFile(path.join(target, "v1", "transcript.live_chat.json"), "utf8"),
'{"v":2}',
);
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "ok");
@@ -1360,13 +1644,14 @@ test("reconcile: an extra and a changed file on the destination are settled, and
// A marker past the copy phase has nothing to reconcile: the run is a plain
// resume, and says so (the review's L3).
-test("reconcile: a marker past the copy phase resumes, and does not claim a reconcile", { skip: T1_SKIP }, async () => {
+test("reconcile: a marker past the copy phase resumes, and does not claim a reconcile", async () => {
await withTmp(async (paths, root) => {
- const channelDir = await seed(paths, "alpha", {
+ await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(100) },
});
- const target = relocatedDataDir(root, "alpha");
- await copyTree(path.join(channelDir, "data"), target);
+ const mediaLink = await tierInPlace(paths, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
+ await copyTree(mediaLink, target);
await seedMarker(paths, "alpha", { target, direction: "out", phase: "swap" });
const lines: string[] = [];
const res = await relocateChannelMedia({
diff --git a/common/controller/relocateChannelMedia.ts b/common/controller/relocateChannelMedia.ts
@@ -2,6 +2,7 @@ import path from "node:path";
import {
access,
constants as fsConstants,
+ lstat,
mkdir,
readdir,
realpath,
@@ -29,7 +30,9 @@ import {
channelWriters,
channelWritersRefusal,
type ChannelWriter,
+ type ChannelWritersOptions,
} from "./channelWriters";
+import { kindNeedsMedia } from "../jobs/jobKinds";
import { isSocialChannel } from "../lib/channelConfig";
import {
locationOfDataDir,
@@ -54,44 +57,67 @@ import { formatBytes } from "../lib/format";
import {
forgetChannelMedia,
inspectChannelMedia,
- relocatedDataDir,
+ legacyDetail,
relocationMarkerPath,
type RelocationDirection,
type RelocationMarker,
type RelocationPhase,
} from "../lib/channelMedia";
import {
+ channelMediaLink,
+ relocatedMediaDir,
+ tierChannelMedia,
+} from "../lib/mediaTier-server";
+import {
patchChannelConfig,
readChannelConfig,
writeChannelConfig,
} from "./channels";
-// MOVE A CHANNEL'S MEDIA TO ANOTHER DRIVE, AND BACK.
+// MOVE A CHANNEL'S MEDIA TIER TO ANOTHER DRIVE, AND BACK (release 17).
+//
+// The unit of a move is `channels/<slug>/media` — the channel's BIG files
+// (lib/mediaTier.ts says which), each reached from `data/<id>/<name>` by a
+// RELATIVE link `../../media/<id>/<name>`. The text — transcripts, cues,
+// metadata, every sidecar, `clips/` — never moves: `data/` stays a real
+// directory on the corpus disk. A move copies `media/` to `<root>/<slug>/media`,
+// verifies it, and makes `channels/<slug>/media` ONE absolute link to it while
+// `config.json` records the target in `mediaDir`. Not one per-file link changes,
+// in either direction: they are relative to `media`, whatever `media` is.
//
-// The mechanism is a symlink (see common/lib/channelMedia.ts for why): the media
-// is copied to `<root>/<slug>/data`, verified, and `channels/<slug>/data` becomes
-// an absolute link to it while `config.json` records the target in `dataDir`.
-// Nothing that reads a channel changes, because the on-disk contract
-// `channelDir/data/<id>/…` is preserved exactly.
+// A CLASSIC CHANNEL IS TIERED FIRST. One whose big files are still real files
+// in `data/<id>/` (no `media/` yet) is tiered in place by the move's preflight —
+// `media/` made a real directory, each file renamed into it and linked, all on
+// one filesystem, in seconds — and then moved like any other.
//
// THE SOURCE IS NEVER TOUCHED UNTIL THE COPY IS VERIFIED. An abort, a full
-// target, a crash, an rsync failure — all of them leave `data/` exactly where it
-// was, and leave the partial copy resumable. The only destructive step is the
-// reclaim at the very end, after the link is in place and the config is written.
+// target, a crash, an rsync failure — all of them leave `media/` exactly where
+// it was, and leave the partial copy resumable. The only destructive step is
+// the reclaim at the very end, after the link is in place and the config is
+// written.
+//
+// `rsync -a` preserves mtimes, and the LMDB index stats no media file at all
+// (presence by name, from one readdir of `data/<id>/`), so a move needs no
+// reindex.
//
-// `rsync -a` preserves mtimes, which is what makes this free for the LMDB index:
-// it stores ids and mtimes, no paths, so a relocated channel needs no reindex.
+// THE RETIRED LAYOUT IS REFUSED. A channel whose whole `data/` was moved by the
+// mover before release 17 (`data/` a link, `config.dataDir`) is `legacy`: it is
+// migrated by `archilyzer storage migrate-tier`, never moved by this.
const BYTES_PER_GB = 1024 ** 3;
export type RelocateChannelMediaResult = {
slug: string;
direction: RelocationDirection;
- // Absolute path of the relocated data dir. For "back" this is what was
- // reclaimed, not where the media now lives.
+ // Absolute path of the relocated media dir, `<root>/<slug>/media`. For
+ // "back" this is what was reclaimed, not where the media now lives.
target: string;
bytes: number;
files: number;
+ // How many big files the preflight tiered into `media/` before the copy (a
+ // classic channel's first move; 0 for "back" and for a channel already
+ // tiered).
+ tiered: number;
// True when the run picked up an interrupted one from its marker rather than
// starting from scratch.
resumed: boolean;
@@ -155,8 +181,66 @@ type RelocateOpts = {
// on this channel's slug, and the relocation queue runs one move at a time.
const RELOCATE_KIND = "relocate-channel-media";
+// MEDIA WRITERS ONLY (release 17 ruling): a move carries `media/` and nothing
+// else, so a digest, a normalize or any other reader/writer of the TEXT may run
+// while it does. Kept: the jobs whose kind opens or writes a big file
+// (`kindNeedsMedia`), the lane units of every lane but the digest one — and a
+// MOVE of this channel's media itself, whose kind is not `needsMedia` (it must
+// not be refused by the media guard it is the reason for) but which is, of
+// everything, the thing writing into `media/`. In the order `channelWriters`
+// gives (running jobs, queued jobs, lane units), so the refusal names the same
+// writer first.
+//
+// The Storage panel's actions, the preview and the job's own first step all
+// ask this; a rename or a delete of the channel asks every writer.
+export function channelMediaWriters(
+ slug: string,
+ opts: Omit<ChannelWritersOptions, "mediaOnly"> = {},
+): ChannelWriter[] {
+ return channelWriters(slug, opts).filter((w) =>
+ w.source === "job"
+ ? kindNeedsMedia(w.kind) || w.kind === RELOCATE_KIND
+ : w.lane !== "digest",
+ );
+}
+
function liveWriters(slug: string): ChannelWriter[] {
- return channelWriters(slug, { ignoreKinds: [RELOCATE_KIND] });
+ return channelMediaWriters(slug, { ignoreKinds: [RELOCATE_KIND] });
+}
+
+// THE RETIRED WHOLE-DIRECTORY LAYOUT, refused by every entry point of the mover
+// with the sentence that names the way out. Answered from the corpus disk alone
+// (`data/` a link, or `config.dataDir` recorded) — never a call to the far
+// drive.
+async function legacyRefusal(
+ channelDir: string,
+ slug: string,
+ config: { dataDir?: string } | null,
+): Promise<string | null> {
+ let dataIsLink = false;
+ try {
+ dataIsLink = (await lstat(path.join(channelDir, "data"))).isSymbolicLink();
+ } catch {
+ /* no data/ yet */
+ }
+ if (!dataIsLink && !config?.dataDir?.trim()) return null;
+ return `Channel "${slug}" cannot be moved: ${legacyDetail(slug)}.`;
+}
+
+// A TIER MIGRATION'S MARKER is not this mover's to resume, replace or clear:
+// the migration rebuilds `data/` itself, with the editor stopped, and resumes
+// from its marker's phase. ONE sentence, for the job, the preview, and the
+// Storage panel's Resume, Reconcile and Clear marker.
+export function tierMigrationRefusal(
+ slug: string,
+ marker: RelocationMarker | null,
+): string | null {
+ if (marker?.scope !== "tier-migration") return null;
+ return (
+ `Channel "${slug}" has a media-tier migration in flight or interrupted ` +
+ `(phase "${marker.phase}") — finish it with archilyzer storage ` +
+ `migrate-tier ${slug}.`
+ );
}
// A MOVE NEVER STARTS OVER A WRITER, and never waits silently for one either: it
@@ -184,13 +268,27 @@ export type RelocationPreview = {
sameDevice: boolean;
// A resumable partial copy from an earlier attempt is already at the target.
existingPartial: boolean;
+ // Big files the preview tiered into `media/` first (a classic channel: its
+ // audio and raw live chat renamed into `channels/<slug>/media/<id>/` and
+ // linked, on the corpus disk). Idempotent: 0 on a second preview.
+ tieredFirst: number;
};
-// Resolved through every symlink when the path exists, lexically when it does
-// not. A destination root that does not exist yet is refused elsewhere; a root
-// that exists and is a symlink back into the corpus is exactly what this is for.
+// Resolved through every symlink of its DEEPEST EXISTING ANCESTOR, the rest
+// joined on lexically. A destination root that does not exist yet is refused
+// elsewhere; a root that exists and is a symlink back into the corpus is
+// exactly what this is for. The ancestor walk matters since release 17: the
+// target `<root>/<slug>/media` usually does not exist yet, and a `<slug>` level
+// that links back into the channel dir must still resolve there.
async function realOrResolved(p: string): Promise<string> {
- return await realpath(p).catch(() => path.resolve(p));
+ const abs = path.resolve(p);
+ try {
+ return await realpath(abs);
+ } catch {
+ const parent = path.dirname(abs);
+ if (parent === abs) return abs;
+ return path.join(await realOrResolved(parent), path.basename(abs));
+ }
}
// `child` IS `parent`, or lives under it.
@@ -201,10 +299,10 @@ function isWithin(parent: string, child: string): boolean {
// WHY A ROOT INSIDE THE CORPUS IS NOT MERELY POINTLESS BUT DESTRUCTIVE.
//
-// Take `root = <transcriptsDir>/channels`. Then `relocatedDataDir(root, slug)`
-// is `<channels>/<slug>/data` — the SOURCE. `rsync -a src/ src/` succeeds,
-// verifyCopy compares the tree with itself and passes, the swap renames `data/`
-// to `data.relocated-<ts>` (which moves the "target" it just verified), creates
+// Take `root = <transcriptsDir>/channels`. Then `relocatedMediaDir(root, slug)`
+// is `<channels>/<slug>/media` — the SOURCE. `rsync -a src/ src/` succeeds,
+// verifyCopy compares the tree with itself and passes, the swap renames `media/`
+// to `media.relocated-<ts>` (which moves the "target" it just verified), creates
// a symlink pointing at a path that no longer exists, and the reclaim sweep then
// `rm -r`s the parked directory — the only copy of the media. Nothing in the
// happy path can notice, because every check it runs is comparing the tree with
@@ -246,17 +344,17 @@ export async function relocationRootProblem(opts: {
// Belt and braces for a root that is outside the corpus but whose `<slug>`
// level is a link back into it: the target is resolved separately, because
// realpath of the root cannot see through a link one level down.
- const realTarget = await realOrResolved(relocatedDataDir(realRoot, opts.slug));
+ const realTarget = await realOrResolved(relocatedMediaDir(realRoot, opts.slug));
if (isWithin(realChannel, realTarget) || isWithin(realTarget, realChannel)) {
return (
- `The destination ${relocatedDataDir(root, opts.slug)} resolves inside ` +
+ `The destination ${relocatedMediaDir(root, opts.slug)} resolves inside ` +
`the channel directory ${channelDir} — the media would be copied onto ` +
`itself and then reclaimed.`
);
}
if (isWithin(realCorpus, realTarget)) {
return (
- `The destination ${relocatedDataDir(root, opts.slug)} resolves inside ` +
+ `The destination ${relocatedMediaDir(root, opts.slug)} resolves inside ` +
`the corpus at ${opts.paths.transcriptsDir}. Pick a directory on the ` +
`other drive.`
);
@@ -290,7 +388,7 @@ export async function relocationRootProblem(opts: {
// Two checks, and the second is the one that catches the case above:
//
// 1. `stat(root)` must be a directory. The move creates `<root>/<slug>` and
-// `<root>/<slug>/data`, never the root itself — a root is a fact about the
+// `<root>/<slug>/media`, never the root itself — a root is a fact about the
// machine, not something a move gets to invent.
// 2. When the root belongs to a LOCATION that has learned a `volume.uuid`,
// the probe must answer `available` with a KNOWN identity whose uuid
@@ -313,8 +411,8 @@ function locationForRoot(
locations: StorageLocation[],
): StorageLocation | null {
// `root + "/x"` rather than `root`: `locationOfDataDir` is deliberately
- // STRICT ("under", not "equal to"), because a channel's dataDir is always
- // `<root>/<slug>/data` and equality there only ever means a misconfiguration.
+ // STRICT ("under", not "equal to"), because a channel's mediaDir is always
+ // `<root>/<slug>/media` and equality there only ever means a misconfiguration.
// Here equality is the ordinary case — the destination root IS the location
// root — so the question is asked about a path one level inside it.
return locationOfDataDir(path.join(root, "x"), locations);
@@ -394,10 +492,10 @@ export async function relocationRootPresenceProblem(
return null;
}
-// The inverse of `relocatedDataDir`: `<root>/<slug>/data` -> `<root>`. The
-// suffix is fixed (channelMedia.ts says so, and deleteChannel recognises a
+// The inverse of `relocatedMediaDir`: `<root>/<slug>/media` -> `<root>`. The
+// suffix is fixed (mediaTier-server.ts says so, and deleteChannel recognises a
// target by it), so this is two dirnames and not a guess.
-export function rootOfRelocatedDataDir(target: string, slug: string): string {
+export function rootOfRelocatedMediaDir(target: string, slug: string): string {
const parent = path.dirname(target);
return path.basename(parent) === slug ? path.dirname(parent) : parent;
}
@@ -417,7 +515,7 @@ export async function assertRelocationRootPresent(
if (problem) throw new Error(problem);
}
-// Every leftover a crashed run can have parked next to `data/`, in one list.
+// Every leftover a crashed run can have parked next to `media/`, in one list.
// The reclaim phase sweeps ALL of them rather than the one name the run that is
// finishing happens to hold: a crash between the config write and the reclaim
// marker leaves a full second copy of the channel on the source volume, and
@@ -426,7 +524,7 @@ export async function assertRelocationRootPresent(
async function parkedSiblings(channelDir: string): Promise<string[]> {
const names = await readdir(channelDir).catch(() => [] as string[]);
return names
- .filter((n) => n.startsWith("data.relocated-") || n === "data.incoming")
+ .filter((n) => n.startsWith("media.relocated-") || n === "media.incoming")
.map((n) => path.join(channelDir, n));
}
@@ -469,12 +567,14 @@ async function readMarkerRaw(
}
// What the operator sees before committing to a move. Cheap enough to run on a
-// form keystroke debounce: one tree walk of the channel plus two statfs calls.
+// form keystroke debounce: one tree walk of the channel's `media/` plus two
+// statfs calls — and, the first time for a classic channel, the tiering (the
+// same renames the job's preflight would make, on the corpus disk; idempotent).
export async function previewRelocation({
paths,
slug,
root,
- writers = (s) => channelWriters(s),
+ writers = (s) => channelMediaWriters(s),
}: {
paths: Paths;
slug: string;
@@ -489,6 +589,12 @@ export async function previewRelocation({
// the worst possible answer.
const problem = await relocationRootProblem({ paths, slug, root });
if (problem) throw new Error(problem);
+ const channelDir = path.join(paths.channelsDir, slug);
+ const config = await readChannelConfig(paths, slug);
+ const migrating = tierMigrationRefusal(slug, await readMarkerRaw(paths, slug));
+ if (migrating) throw new Error(migrating);
+ const legacy = await legacyRefusal(channelDir, slug, config);
+ if (legacy) throw new Error(legacy);
// The same refusal the job's first step gives, before the operator commits.
assertNoWriters(slug, writers);
// EXISTENCE, HERE AS WELL AS IN THE JOB. getFreeBytes walks up to the nearest
@@ -501,8 +607,25 @@ export async function previewRelocation({
`(is the drive mounted?)`,
);
}
- const source = path.join(paths.channelsDir, slug, "data");
- const target = relocatedDataDir(root, slug);
+ // TIER FIRST, as the job will: the figures below are then the ones the job
+ // moves. Only an in-place channel (a relocated one's tier is on the far
+ // drive, and a preview copies nothing there), only with nothing in flight (a
+ // marker means a move has already tiered it, and the hook writes nothing
+ // under one anyway), and only one with something downloaded (no `data/` → no
+ // `media/` invented).
+ let tieredFirst = 0;
+ const marker = await readMarkerRaw(paths, slug);
+ if (
+ !marker &&
+ !config?.mediaDir?.trim() &&
+ (await isDirectory(path.join(channelDir, "data")))
+ ) {
+ tieredFirst = (
+ await tierChannelMedia(paths, slug, { createMediaDir: true })
+ ).tiered;
+ }
+ const source = channelMediaLink(paths, slug);
+ const target = relocatedMediaDir(root, slug);
const [measured, freeOnRoot, freeOnSource, existingPartial] = await Promise.all([
measureTree(source),
getFreeBytes(root),
@@ -511,7 +634,7 @@ export async function previewRelocation({
]);
let sameDevice = false;
try {
- const [a, b] = await Promise.all([stat(source), stat(root)]);
+ const [a, b] = await Promise.all([stat(paths.channelsDir), stat(root)]);
sameDevice = a.dev === b.dev;
} catch {
/* an unmounted or absent root is not "same device" */
@@ -525,6 +648,7 @@ export async function previewRelocation({
freeOnSource,
sameDevice,
existingPartial,
+ tieredFirst,
};
}
@@ -551,8 +675,12 @@ export async function relocateChannelMedia(
assertNoWriters(slug, writers);
const channelDir = path.join(paths.channelsDir, slug);
- const dataDir = path.join(channelDir, "data");
+ const mediaLink = channelMediaLink(paths, slug);
const existingMarker = await readMarkerRaw(paths, slug);
+ const migrating = tierMigrationRefusal(slug, existingMarker);
+ if (migrating) throw new Error(migrating);
+ const legacy = await legacyRefusal(channelDir, slug, config);
+ if (legacy) throw new Error(legacy);
if (opts.reconcile && !existingMarker) {
throw new Error(
`Channel "${slug}" has no relocation marker — there is no interrupted ` +
@@ -563,13 +691,13 @@ export async function relocateChannelMedia(
if (direction === "back") {
// THE MARKER IS THE SECOND SOURCE OF TRUTH FOR THE TARGET, and without it
- // the resume path was unreachable. moveBack clears config.dataDir as part
+ // the resume path was unreachable. moveBack clears config.mediaDir as part
// of its swap, so a crash after that point left a rerun reading "this
// channel is not relocated" from the config and throwing — while a marker
// sat next to it naming the very target still holding the media, and
// inspect() reported in-transition forever.
const resume = existingMarker?.direction === "back" ? existingMarker : null;
- const target = config.dataDir?.trim() || resume?.target;
+ const target = config.mediaDir?.trim() || resume?.target;
if (!target) {
throw new Error(
`Channel "${slug}" is not relocated — its media is already in place`,
@@ -596,7 +724,7 @@ export async function relocateChannelMedia(
io,
config,
channelDir,
- dataDir,
+ mediaLink,
target,
log,
onProgress,
@@ -620,16 +748,16 @@ export async function relocateChannelMedia(
storage: io.read().storage,
});
if (rootProblem) throw new Error(rootProblem);
- const target = relocatedDataDir(root, slug);
+ const target = relocatedMediaDir(root, slug);
if (existingMarker && existingMarker.target !== target) {
throw new Error(
`A relocation to ${existingMarker.target} is already in progress for "${slug}" — ` +
`finish or clear it before moving to ${target}`,
);
}
- if (config.dataDir?.trim() && config.dataDir.trim() !== target) {
+ if (config.mediaDir?.trim() && config.mediaDir.trim() !== target) {
throw new Error(
- `Channel "${slug}" is already relocated to ${config.dataDir.trim()}. ` +
+ `Channel "${slug}" is already relocated to ${config.mediaDir.trim()}. ` +
`Move it back in place first.`,
);
}
@@ -648,7 +776,7 @@ export async function relocateChannelMedia(
io,
config,
channelDir,
- dataDir,
+ mediaLink,
root,
target,
log,
@@ -687,7 +815,7 @@ async function moveOut(args: {
io: { read: () => SiteSettings };
config: NonNullable<Awaited<ReturnType<typeof readChannelConfig>>>;
channelDir: string;
- dataDir: string;
+ mediaLink: string;
root: string;
target: string;
log: (m: string) => void;
@@ -698,7 +826,8 @@ async function moveOut(args: {
writers: (slug: string) => ChannelWriter[];
reconcile: boolean;
}): Promise<RelocateChannelMediaResult> {
- const { paths, slug, channelDir, dataDir, root, target, log, signal } = args;
+ const { paths, slug, channelDir, mediaLink, root, target, log, signal } = args;
+ const dataDir = path.join(channelDir, "data");
// PREFLIGHT. Everything that can refuse does so here, before a single byte is
// written and before the marker exists.
@@ -732,9 +861,8 @@ async function moveOut(args: {
);
}
- // A channel that has downloaded nothing has no data/ at all, and rsync exits
- // 23 on a missing source — an error message about a partial transfer for
- // something that is not a transfer. Say what is actually the matter.
+ // A channel that has downloaded nothing has no data/ at all. Say so, rather
+ // than invent a `media/` for it and move nothing.
if (args.phase === "copy" && !(await isDirectory(dataDir))) {
throw new Error(
`Channel "${slug}" has no ${dataDir} to move — nothing has been ` +
@@ -742,7 +870,36 @@ async function moveOut(args: {
);
}
- const measured = await measureTree(dataDir);
+ // TIER FIRST (release 17): after the writers check (the job's first step,
+ // above) and before anything is measured. A classic channel's big files are
+ // renamed into a new real `media/` and linked — the same filesystem, so a
+ // rename each — and a channel tiered already counts them as "already". A
+ // resumed run tiers nothing: its marker stands, and the hook writes nothing
+ // under one (a file a writer finished since stays real, on the corpus disk,
+ // until the next sweep tiers it onto the far side).
+ let tiered = 0;
+ if (args.phase === "copy") {
+ // THE DESTINATION'S IDENTITY FIRST (review N5): a bare mountpoint, or a
+ // drive whose learned volume does not match, is refused before the
+ // channel is touched at all — the tiering below is harmless, but "refused
+ // before anything" should be true here too. Asked again immediately before
+ // the mkdir below, for a job that sat in the queue.
+ await assertRelocationRootPresent(root, args.io.read().storage, paths);
+ const counts = await tierChannelMedia(paths, slug, {
+ createMediaDir: true,
+ onLog: (line) => log(line.replace(/\n$/, "")),
+ });
+ tiered = counts.tiered;
+ if (tiered > 0) log(`Tiered ${tiered} file(s) into ${mediaLink} first`);
+ if (!(await isDirectory(mediaLink))) {
+ throw new Error(
+ `Channel "${slug}" has no ${mediaLink} to move — it could not be ` +
+ `made a directory`,
+ );
+ }
+ }
+
+ const measured = await measureTree(mediaLink);
// Sticky across both verify points below: a retry at either one is the fact
// the caller wants reported, and neither overwrites the other's answer.
let verifyRetried = false;
@@ -793,6 +950,7 @@ async function moveOut(args: {
direction: "out",
startedAt: new Date().toISOString(),
phase: "copy",
+ scope: "media",
});
await assertNoWritersUnderMarker({
paths,
@@ -807,10 +965,10 @@ async function moveOut(args: {
verifyRetried ||= (
await copyMirrorVerify({
rsyncBin: paths.rsyncBin,
- src: dataDir,
+ src: mediaLink,
dest: target,
// The channel's media, which --delete may never reach.
- live: dataDir,
+ live: mediaLink,
log,
progress: makeProgressSink({
totalBytes: measured.bytes,
@@ -822,7 +980,7 @@ async function moveOut(args: {
reconcile: args.reconcile,
cancelled: () =>
new Error(
- `Cancelled. ${dataDir} is untouched and the partial copy at ${target} ` +
+ `Cancelled. ${mediaLink} is untouched and the partial copy at ${target} ` +
`is resumable — rerun to continue.`,
),
})
@@ -837,14 +995,15 @@ async function moveOut(args: {
direction: "out",
startedAt: new Date().toISOString(),
phase: "swap",
+ scope: "media",
});
// EVERY STEP BELOW OBSERVES THE DISK INSTEAD OF ASSUMING THE LAST ONE RAN.
// The marker says how far the previous attempt got; it cannot say how far
// it got THROUGH a phase, and a crash lands between any two syscalls.
- const state = await linkOrDirState(dataDir);
+ const state = await linkOrDirState(mediaLink);
const parked = (await parkedSiblings(channelDir)).filter((p) =>
- path.basename(p).startsWith("data.relocated-"),
+ path.basename(p).startsWith("media.relocated-"),
);
// Re-verify, because a resumed run did not do the copy in this process and
@@ -852,13 +1011,13 @@ async function moveOut(args: {
// against is whichever copy of it still exists; once the swap has committed
// there is none, and there is nothing left to check.
//
- // MIRRORED ONLY FROM THE LIVE MEDIA. While `data/` is still the real
+ // MIRRORED ONLY FROM THE LIVE MEDIA. While `media/` is still the real
// directory it is the source, and a difference gets the mirror pass a copy
- // phase would give it. A parked `data.relocated-*` is not live any more —
+ // phase would give it. A parked `media.relocated-*` is not live any more —
// the link already points at the target — so the target is never mirrored
// FROM it: that verify is the strict one, as it always was.
const verifySrc =
- state.kind === "real-dir" ? dataDir : (parked[0] ?? null);
+ state.kind === "real-dir" ? mediaLink : (parked[0] ?? null);
if (verifySrc) {
log("Verifying the copy…");
verifyRetried ||= (
@@ -868,26 +1027,29 @@ async function moveOut(args: {
dest: target,
log,
signal,
- ...(verifySrc === dataDir
- ? { mirror: { live: dataDir } }
+ ...(verifySrc === mediaLink
+ ? { mirror: { live: mediaLink } }
: {}),
})
).retried;
}
if (state.kind === "real-dir") {
- // Same device, so the rename is atomic: `data/` is a real dir one instant
- // and the parked copy the next, never half of each. A fresh name even
- // when a parked dir already exists — renaming onto a non-empty directory
- // is ENOTEMPTY, and the reclaim sweep takes all of them anyway.
- await rename(dataDir, path.join(channelDir, `data.relocated-${Date.now()}`));
+ // Same device, so the rename is atomic: `media/` is a real dir one
+ // instant and the parked copy the next, never half of each. A fresh name
+ // even when a parked dir already exists — renaming onto a non-empty
+ // directory is ENOTEMPTY, and the reclaim sweep takes all of them anyway.
+ // Between this rename and the symlink below every per-file link in
+ // `data/<id>/` dangles for one syscall; the marker stands, so every media
+ // guard reads the channel as in transition and nothing opens one.
+ await rename(mediaLink, path.join(channelDir, `media.relocated-${Date.now()}`));
} else if (state.kind === "other") {
throw new Error(
- `${dataDir} is neither a directory nor a symlink — refusing to replace it`,
+ `${mediaLink} is neither a directory nor a symlink — refusing to replace it`,
);
}
- const after = await linkOrDirState(dataDir);
+ const after = await linkOrDirState(mediaLink);
if (
after.kind === "link" &&
path.resolve(after.linkTarget) !== path.resolve(target)
@@ -895,14 +1057,14 @@ async function moveOut(args: {
// A link to somewhere else is not this move's work to reinterpret, and
// silently repointing it would strand whatever it does point at.
throw new Error(
- `${dataDir} already points at ${after.linkTarget}, not ${target}`,
+ `${mediaLink} already points at ${after.linkTarget}, not ${target}`,
);
}
if (after.kind === "missing") {
- await symlink(target, dataDir);
+ await symlink(target, mediaLink);
}
- // Written only now, on success: config.dataDir is a record of what is on
+ // Written only now, on success: config.mediaDir is a record of what is on
// disk, never an intention. Skipped when it already says so, so a rerun
// does not rewrite a file it agrees with.
// No readable config.json (it vanished mid-move, before the read or
@@ -911,18 +1073,19 @@ async function moveOut(args: {
// nothing.
const fresh = await readChannelConfig(paths, slug);
const patched =
- fresh && fresh.dataDir?.trim() !== target
- ? await patchChannelConfig(paths, slug, { dataDir: target })
+ fresh && fresh.mediaDir?.trim() !== target
+ ? await patchChannelConfig(paths, slug, { mediaDir: target })
: fresh;
if (!patched) {
- await writeChannelConfig(paths, slug, { ...args.config, dataDir: target });
+ await writeChannelConfig(paths, slug, { ...args.config, mediaDir: target });
}
- log(`Swapped: ${dataDir} -> ${target}`);
+ log(`Swapped: ${mediaLink} -> ${target}`);
await writeMarker(paths, slug, {
target,
direction: "out",
startedAt: new Date().toISOString(),
phase: "reclaim",
+ scope: "media",
});
}
@@ -945,6 +1108,7 @@ async function moveOut(args: {
target,
bytes: measured.bytes,
files: measured.files,
+ tiered,
resumed: args.resumed,
retried: verifyRetried,
reconciled,
@@ -957,7 +1121,7 @@ async function moveBack(args: {
io: { read: () => SiteSettings };
config: NonNullable<Awaited<ReturnType<typeof readChannelConfig>>>;
channelDir: string;
- dataDir: string;
+ mediaLink: string;
target: string;
log: (m: string) => void;
onProgress?: (p: RelocationProgress) => void;
@@ -967,21 +1131,21 @@ async function moveBack(args: {
writers: (slug: string) => ChannelWriter[];
reconcile: boolean;
}): Promise<RelocateChannelMediaResult> {
- const { paths, slug, channelDir, dataDir, target, log, signal } = args;
- const incoming = path.join(channelDir, "data.incoming");
+ const { paths, slug, channelDir, mediaLink, target, log, signal } = args;
+ const incoming = path.join(channelDir, "media.incoming");
// THE MIRROR OF moveOut's "must be in-place", and it is not symmetry for its
// own sake: move back is the only direction that ENDS by deleting the target.
//
// `relocated` is true for `inconsistent` and `unreachable` as well as `ok` —
- // config.dataDir is set in all three — so without this the UI offers Move back
- // for a channel whose config records a target while `data/` is a REAL
+ // config.mediaDir is set in all three — so without this the UI offers Move
+ // back for a channel whose config records a target while `media/` is a REAL
// directory. That state is not hypothetical: it is what `rsync --copy-links`
// of a channel produces, which WORKTREES.md documents as the way to carry
- // media into a shard. The run then copies the target to `data.incoming`,
- // verifies it, finds `data/` already a real dir, logs "the swap had
+ // media into a shard. The run then copies the target to `media.incoming`,
+ // verifies it, finds `media/` already a real dir, logs "the swap had
// completed", clears the config, `rm -r`s the target and finally sweeps
- // `data.incoming` — three copies in, zero out.
+ // `media.incoming` — three copies in, zero out.
//
// `in-transition` is allowed because a marker is what a resume carries, and a
// rerun is the caller this precondition must not refuse.
@@ -1015,7 +1179,7 @@ async function moveBack(args: {
? await measureTree(target)
: (await isDirectory(incoming))
? await measureTree(incoming)
- : await measureTree(dataDir);
+ : await measureTree(mediaLink);
log(
`Moving ${slug} back in place: ${measured.files} file(s), ` +
`${formatBytes(measured.bytes)} <- ${target}`,
@@ -1026,7 +1190,7 @@ async function moveBack(args: {
// THE SAME BAR MOVE-OUT USES, and for the same reason: landing the media
// with nothing to spare puts the corpus volume under the disk gate's floor
// the moment it arrives. And a resumed move-back must only be charged for
- // what is still MISSING — the bytes already sitting in `data.incoming` are
+ // what is still MISSING — the bytes already sitting in `media.incoming` are
// not about to be written twice, and counting them refused reruns on a disk
// that had room for the remainder.
const settings = args.io.read();
@@ -1047,14 +1211,14 @@ async function moveBack(args: {
);
}
// THE GUARD GOES ON THE LOCATION ROOT, NOT ON `incoming`. `incoming` is
- // `channels/<slug>/data.incoming`, a corpus directory a move-back is
+ // `channels/<slug>/media.incoming`, a corpus directory a move-back is
// entitled to create. What must be present is the SOURCE side: the
// `isDirectory(target)` precondition above covers existence, and this
// covers IDENTITY — an empty mountpoint directory with the platter
// unplugged is a directory, and copying it back would report a successful
// move of zero bytes and then delete the target.
await assertRelocationRootPresent(
- rootOfRelocatedDataDir(target, slug),
+ rootOfRelocatedMediaDir(target, slug),
settings.storage,
paths,
);
@@ -1064,6 +1228,7 @@ async function moveBack(args: {
direction: "back",
startedAt: new Date().toISOString(),
phase: "copy",
+ scope: "media",
});
await assertNoWritersUnderMarker({
paths,
@@ -1071,21 +1236,21 @@ async function moveBack(args: {
resumed: args.resumed,
writers: args.writers,
});
- // The copy under construction is `data.incoming`; the target on the other
+ // The copy under construction is `media.incoming`; the target on the other
// drive is the source, and is never the target of the mirror's --delete.
- // `live` is `data/` — the link, which resolves to that target — so a call
+ // `live` is `media` — the link, which resolves to that target — so a call
// with source and destination swapped is refused before rsync runs.
verifyRetried ||= (
await copyMirrorVerify({
rsyncBin: paths.rsyncBin,
src: target,
dest: incoming,
- live: dataDir,
+ live: mediaLink,
log,
progress: makeProgressSink({
totalBytes: measured.bytes,
// The same figure the space check above is priced in — what is
- // already in `data.incoming` from an interrupted run.
+ // already in `media.incoming` from an interrupted run.
alreadyBytes: already,
log,
onProgress: args.onProgress,
@@ -1109,49 +1274,55 @@ async function moveBack(args: {
direction: "back",
startedAt: new Date().toISOString(),
phase: "swap",
+ scope: "media",
});
// OBSERVE, DO NOT ASSUME. The old sequence was `unlink(data)` (swallowing
// its error) then `rename(incoming, data)`, which is idempotent in exactly
// the case that never happens: a crash AFTER the rename left `data/` a real
// directory, the swallowed unlink then failed on it, and the rename ENOENTed
- // on an `incoming` that no longer existed — forever, on every rerun.
- const state = await linkOrDirState(dataDir);
+ // on an `incoming` that no longer existed — forever, on every rerun. The
+ // same holds for `media` since release 17.
+ const state = await linkOrDirState(mediaLink);
if (state.kind === "real-dir") {
// The rename already committed. Nothing to swap; the leftovers are the
// reclaim's business.
- log(`${dataDir} is already a real directory — the swap had completed`);
+ log(`${mediaLink} is already a real directory — the swap had completed`);
} else if (state.kind === "other") {
- // A regular file (or a socket, or a fifo) where `data/` should be is not
+ // A regular file (or a socket, or a fifo) where `media/` should be is not
// a link to replace and not a directory to keep. moveOut refuses the same
// shape at its own swap; refusing here too is what keeps `unlink` below
// meaning "remove the symlink" and nothing else.
throw new Error(
- `${dataDir} is neither a directory nor a symlink — refusing to replace it`,
+ `${mediaLink} is neither a directory nor a symlink — refusing to replace it`,
);
} else {
if (!(await isDirectory(incoming))) {
throw new Error(
- `Cannot finish moving "${slug}" back: ${dataDir} is not a directory ` +
+ `Cannot finish moving "${slug}" back: ${mediaLink} is not a directory ` +
`and there is no verified copy at ${incoming}`,
);
}
- // unlink, not rm -r: `data` is the LINK here, and removing it recursively
- // would be the one way this whole design eats the media.
- if (state.kind !== "missing") await unlink(dataDir);
- await rename(incoming, dataDir);
- log(`Swapped: ${dataDir} is a real directory again`);
+ // unlink, not rm -r: `media` is the LINK here, and removing it
+ // recursively would be the one way this whole design eats the media.
+ // Every per-file link in `data/<id>/` is relative to `media`, so the
+ // rename below makes each of them resolve on the corpus disk — not one
+ // of them is rewritten (no "untier").
+ if (state.kind !== "missing") await unlink(mediaLink);
+ await rename(incoming, mediaLink);
+ log(`Swapped: ${mediaLink} is a real directory again`);
}
const fresh = await readChannelConfig(paths, slug);
- if (fresh?.dataDir !== undefined) {
- await patchChannelConfig(paths, slug, {}, { unset: ["dataDir"] });
+ if (fresh?.mediaDir !== undefined) {
+ await patchChannelConfig(paths, slug, {}, { unset: ["mediaDir"] });
}
await writeMarker(paths, slug, {
target,
direction: "back",
startedAt: new Date().toISOString(),
phase: "reclaim",
+ scope: "media",
});
}
@@ -1161,7 +1332,7 @@ async function moveBack(args: {
if ((await readdir(slugRoot).catch(() => ["keep"])).length === 0) {
await rm(slugRoot, { recursive: true, force: true });
}
- // Any half-copied `data.incoming` (or a parked dir from an earlier move out)
+ // Any half-copied `media.incoming` (or a parked dir from an earlier move out)
// goes with it — the same sweep, for the same reason.
await sweepParked(channelDir, log);
await clearMarker(paths, slug);
@@ -1172,6 +1343,7 @@ async function moveBack(args: {
target,
bytes: measured.bytes,
files: measured.files,
+ tiered: 0,
resumed: args.resumed,
retried: verifyRetried,
reconciled,
diff --git a/common/controller/relocateDir.ts b/common/controller/relocateDir.ts
@@ -27,8 +27,8 @@ import type {
//
// This is `relocateChannelMedia.ts`'s core, lifted out unchanged so a SECOND
// thing can be moved by it: the saved-video store. What stayed behind in that
-// file is everything specific to a channel — `config.dataDir`, the parked
-// `data.relocated-*` siblings, the social-channel refusal, `inspectChannelMedia`
+// file is everything specific to a channel — `config.mediaDir`, the parked
+// `media.relocated-*` siblings, the social-channel refusal, `inspectChannelMedia`
// — because none of it generalises and pretending it did would be the worse
// abstraction.
//
@@ -160,6 +160,12 @@ export async function readDirMarker(
r.phase === "swap" || r.phase === "reclaim"
? r.phase
: ("copy" as RelocationPhase),
+ // Optional (release 17): what a channel's marker is moving. Kept so a
+ // resume rewrites what it read, and so the mover can tell a tier
+ // migration's marker from its own.
+ ...(r.scope === "media" || r.scope === "tier-migration"
+ ? { scope: r.scope }
+ : {}),
};
} catch {
return null;
@@ -458,11 +464,11 @@ export class MirrorDirectionError extends Error {
// THE ONE RULE `--delete` LIVES UNDER, asserted before rsync is spawned: the
// destination is not the live media. `live` is supplied by the caller from
-// what it KNOWS is live — a channel's `channels/<slug>/data` (resolved through
+// what it KNOWS is live — a channel's `channels/<slug>/media` (resolved through
// its link, so on the way back it is the relocated target) or the saved-video
// store — and never derived from `src` or `dest`, so a call with the two
-// swapped is caught: out, the destination would be `data/` itself; back, it
-// would be the target `data/` points at. Refused when the destination's real
+// swapped is caught: out, the destination would be `media/` itself; back, it
+// would be the target `media` points at. Refused when the destination's real
// path is the live media's, contains it, or sits inside it; and, belt and
// braces, when the source and the destination contain one another.
export async function assertMirrorDirection(opts: {
diff --git a/common/controller/renameChannel.test.ts b/common/controller/renameChannel.test.ts
@@ -33,15 +33,9 @@ import {
import { renameChannel } from "./renameChannel";
import {
inspectChannelMedia,
- relocatedDataDir,
RELOCATION_MARKER_FILENAME,
} from "../lib/channelMedia";
-
-// RELEASE 17 SLICE T1 made a channel whose `data/` is a link (or whose config
-// carries `dataDir`) `legacy`; these cases still build that retired layout and
-// expect it to read `ok`. Slice T2 rebases them on `media/` and un-skips them.
-const T1_SKIP = "release 17 T2 rebases the mover on media/";
-
+import { relocatedMediaDir, tierLinkTarget } from "../lib/mediaTier-server";
// Run with:
// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/renameChannel.test.ts
@@ -156,29 +150,38 @@ test("renameChannel rejects invalid, same, and existing targets", async () => {
// --- relocated media ------------------------------------------------------
-test("rename re-points a convention-shaped relocated media dir", { skip: T1_SKIP }, async () => {
+test("rename re-points a convention-shaped relocated media dir", async () => {
await withPaths(async (paths) => {
const dir = path.dirname(paths.channelsDir);
const mediaRoot = path.join(dir, "platter");
- const target = relocatedDataDir(mediaRoot, "old");
- await writeChannelConfig(paths, "old", { ...config, dataDir: target });
+ const target = relocatedMediaDir(mediaRoot, "old");
+ await writeChannelConfig(paths, "old", { ...config, mediaDir: target });
await mkdir(path.join(target, "vid1"), { recursive: true });
await writeFile(path.join(target, "vid1", "audio.m4a"), "BYTES");
- await symlink(target, path.join(paths.channelsDir, "old", "data"));
+ await symlink(target, path.join(paths.channelsDir, "old", "media"));
+ // The tiered file's RELATIVE link in the real `data/` on the corpus disk.
+ const videoDir = path.join(paths.channelsDir, "old", "data", "vid1");
+ await mkdir(videoDir, { recursive: true });
+ await symlink(tierLinkTarget("vid1", "audio.m4a"), path.join(videoDir, "audio.m4a"));
const result = await renameChannel(paths, "old", "new", {
...config,
- dataDir: target,
+ mediaDir: target,
});
assert.deepEqual(result.warnings, []);
- const newTarget = relocatedDataDir(mediaRoot, "new");
- assert.equal((await readChannelConfig(paths, "new"))?.dataDir, newTarget);
+ const newTarget = relocatedMediaDir(mediaRoot, "new");
+ assert.equal((await readChannelConfig(paths, "new"))?.mediaDir, newTarget);
assert.equal(
- await readlink(path.join(paths.channelsDir, "new", "data")),
+ await readlink(path.join(paths.channelsDir, "new", "media")),
newTarget,
);
- // The media reads through the new link at the old on-disk contract path.
+ // The per-file link is untouched — relative, it moved with the channel —
+ // and the media reads through it at the on-disk contract path.
+ assert.equal(
+ await readlink(path.join(paths.channelsDir, "new", "data", "vid1", "audio.m4a")),
+ tierLinkTarget("vid1", "audio.m4a"),
+ );
assert.equal(
await readFile(
path.join(paths.channelsDir, "new", "data", "vid1", "audio.m4a"),
@@ -198,19 +201,19 @@ test("rename re-points a convention-shaped relocated media dir", { skip: T1_SKIP
test("a media dir that does not follow the convention is left alone", async () => {
await withPaths(async (paths) => {
const dir = path.dirname(paths.channelsDir);
- // <root>/<something-else>/data — the link is absolute and still works after
- // the rename, so moving a directory whose name is not ours would be worse
- // than leaving it.
- const target = path.join(dir, "platter", "handpicked", "data");
- await writeChannelConfig(paths, "old", { ...config, dataDir: target });
+ // <root>/<something-else>/media — the link is absolute and still works
+ // after the rename, so moving a directory whose name is not ours would be
+ // worse than leaving it.
+ const target = path.join(dir, "platter", "handpicked", "media");
+ await writeChannelConfig(paths, "old", { ...config, mediaDir: target });
await mkdir(path.join(target, "vid1"), { recursive: true });
- await symlink(target, path.join(paths.channelsDir, "old", "data"));
+ await symlink(target, path.join(paths.channelsDir, "old", "media"));
- await renameChannel(paths, "old", "new", { ...config, dataDir: target });
+ await renameChannel(paths, "old", "new", { ...config, mediaDir: target });
- assert.equal((await readChannelConfig(paths, "new"))?.dataDir, target);
+ assert.equal((await readChannelConfig(paths, "new"))?.mediaDir, target);
assert.equal(
- await readlink(path.join(paths.channelsDir, "new", "data")),
+ await readlink(path.join(paths.channelsDir, "new", "media")),
target,
);
await stat(path.join(target, "vid1"));
@@ -221,15 +224,15 @@ test("rename rolls the channel dir back when the media move fails", async () =>
await withPaths(async (paths) => {
const dir = path.dirname(paths.channelsDir);
const mediaRoot = path.join(dir, "platter");
- const target = relocatedDataDir(mediaRoot, "old");
- await writeChannelConfig(paths, "old", { ...config, dataDir: target });
+ const target = relocatedMediaDir(mediaRoot, "old");
+ await writeChannelConfig(paths, "old", { ...config, mediaDir: target });
await mkdir(path.join(target, "vid1"), { recursive: true });
- await symlink(target, path.join(paths.channelsDir, "old", "data"));
+ await symlink(target, path.join(paths.channelsDir, "old", "media"));
// Something is already sitting at <root>/new.
await mkdir(path.join(mediaRoot, "new"), { recursive: true });
await assert.rejects(
- () => renameChannel(paths, "old", "new", { ...config, dataDir: target }),
+ () => renameChannel(paths, "old", "new", { ...config, mediaDir: target }),
/already exists/,
);
// Nothing half-renamed: the channel is still "old", still linked, still
@@ -237,20 +240,40 @@ test("rename rolls the channel dir back when the media move fails", async () =>
assert.equal(await channelExists(paths, "old"), true);
assert.equal(await channelExists(paths, "new"), false);
assert.equal(
- await readlink(path.join(paths.channelsDir, "old", "data")),
+ await readlink(path.join(paths.channelsDir, "old", "media")),
target,
);
await stat(path.join(target, "vid1"));
});
});
+// THE RETIRED LAYOUT (release 17 review L2): a legacy channel is migrated
+// before it is renamed, so the tier migration only ever sees
+// `dataDir = <root>/<slug>/data`.
+test("a legacy channel's rename is refused, naming migrate-tier, and nothing moves", async () => {
+ await withPaths(async (paths) => {
+ const dir = path.dirname(paths.channelsDir);
+ const target = path.join(dir, "platter", "old", "data");
+ await writeChannelConfig(paths, "old", { ...config, dataDir: target });
+ await mkdir(path.join(target, "vid1"), { recursive: true });
+ await symlink(target, path.join(paths.channelsDir, "old", "data"));
+ await assert.rejects(
+ () => renameChannel(paths, "old", "new", { ...config, dataDir: target }),
+ /cannot be renamed: .*archilyzer storage migrate-tier old/,
+ );
+ assert.equal(await channelExists(paths, "old"), true);
+ assert.equal(await channelExists(paths, "new"), false);
+ await stat(path.join(target, "vid1"));
+ });
+});
+
test("rename is refused while a relocation is in flight", async () => {
await withPaths(async (paths) => {
await writeChannelConfig(paths, "old", config);
await writeFile(
path.join(paths.channelsDir, "old", RELOCATION_MARKER_FILENAME),
JSON.stringify({
- target: "/mnt/platter/old/data",
+ target: "/mnt/platter/old/media",
direction: "out",
startedAt: new Date().toISOString(),
phase: "copy",
diff --git a/common/controller/renameChannel.ts b/common/controller/renameChannel.ts
@@ -1,5 +1,5 @@
import path from "node:path";
-import { readdir, rename, stat, symlink, unlink } from "node:fs/promises";
+import { lstat, readdir, rename, stat, symlink, unlink } from "node:fs/promises";
import type { Paths } from "../lib/paths";
import type { ChannelConfig } from "../lib/channelConfig";
import {
@@ -9,10 +9,8 @@ import {
writeChannelConfig,
} from "./channels";
import { savedVideoRoot } from "../lib/savedVideo";
-import {
- readRelocationMarker,
- relocatedDataDir,
-} from "../lib/channelMedia";
+import { legacyDetail, readRelocationMarker } from "../lib/channelMedia";
+import { MEDIA_LINK_NAME, relocatedMediaDir } from "../lib/mediaTier-server";
import { rewriteSavedVideoDir } from "../lib/savedVideo-server";
import { getSite, listSiteIds, writeSite } from "../lib/site";
import {
@@ -25,7 +23,9 @@ import {
// every other store that keys by slug and would otherwise be orphaned:
// - the saved-video store dir + each saved-video.json pointer's absolute `dir`
// - a relocated media dir on another drive, when it follows the
-// <root>/<slug>/data convention, plus the symlink and config.dataDir
+// <root>/<slug>/media convention, plus the `media` symlink and
+// config.mediaDir (the per-file links in `data/<id>/` are RELATIVE to
+// `media`, so they move with the channel dir and need nothing)
// - site.json memberships across all sites
// - the sync scheduler's per-channel backoff state
//
@@ -91,6 +91,22 @@ export async function renameChannel(
);
}
+ // THE RETIRED LAYOUT (release 17) is migrated before it is renamed: the
+ // tier migration renames `<root>/<slug>/data` to `<root>/<slug>/media` and
+ // reads `dataDir` as exactly that shape, so a rename that left the tree
+ // under the old slug would break it. Refused before anything moves.
+ let dataIsLink = false;
+ try {
+ dataIsLink = (await lstat(path.join(oldChannelDir, "data"))).isSymbolicLink();
+ } catch {
+ /* no data/ */
+ }
+ if (dataIsLink || config.dataDir?.trim()) {
+ throw new Error(
+ `Channel "${oldSlug}" cannot be renamed: ${legacyDetail(oldSlug)}.`,
+ );
+ }
+
const storeRoot = savedVideoRoot(paths, config);
const oldStoreDir = path.join(storeRoot, oldSlug);
const newStoreDir = path.join(storeRoot, newSlug);
@@ -122,20 +138,21 @@ export async function renameChannel(
}
}
- // 3. Move the relocated media dir when it follows the <root>/<slug>/data
- // convention, and re-point the symlink at it. The link is ABSOLUTE, so a
+ // 3. Move the relocated media dir when it follows the <root>/<slug>/media
+ // convention, and re-point the `media` symlink at it. The link is ABSOLUTE, so a
// target that does NOT follow the convention is deliberately left alone —
// it still works, and moving someone else's directory because its name
// happened to match would be worse than leaving it. Rolls the channel-dir
// (and store) move back on failure, same shape as step 2.
- const relocated = config.dataDir?.trim();
+ const relocated = config.mediaDir?.trim();
const conventional =
- relocated && relocated === relocatedDataDir(path.dirname(path.dirname(relocated)), oldSlug)
+ relocated && relocated === relocatedMediaDir(path.dirname(path.dirname(relocated)), oldSlug)
? relocated
: null;
if (conventional) {
const mediaRoot = path.dirname(path.dirname(conventional));
- const newTarget = relocatedDataDir(mediaRoot, newSlug);
+ const newTarget = relocatedMediaDir(mediaRoot, newSlug);
+ const link = path.join(newChannelDir, MEDIA_LINK_NAME);
// A ROLLBACK MUST ONLY UNDO WHAT ACTUALLY RAN. The pre-check below fails
// BECAUSE something unrelated already occupies <root>/<newSlug> — and the
// old catch then renamed that stranger to <root>/<oldSlug>, destroying a
@@ -145,7 +162,7 @@ export async function renameChannel(
let movedMedia = false;
// THE UNLINK IS A STEP TOO. It was untracked, so a throw from the symlink
// below (EACCES on a read-only channel dir, ENOSPC) rolled the media
- // directory back while `data/` stayed DELETED — config still naming the old
+ // directory back while the link stayed DELETED — config still naming the old
// target, nothing on disk pointing at it: `inconsistent`, which is now a
// state move-back refuses. The catch replays only what ran, and removing the
// link ran.
@@ -157,33 +174,32 @@ export async function renameChannel(
}
await rename(path.dirname(conventional), path.dirname(newTarget));
movedMedia = true;
- await unlink(path.join(newChannelDir, "data"))
+ await unlink(link)
.then(() => {
unlinked = true;
})
.catch(() => {});
- await symlink(newTarget, path.join(newChannelDir, "data"));
+ await symlink(newTarget, link);
relinked = true;
// Re-read: the channel dir has already moved, so this is the file that
// will actually be on disk afterwards.
const patched = await patchChannelConfig(paths, newSlug, {
- dataDir: newTarget,
+ mediaDir: newTarget,
});
if (!patched) {
// No readable config.json at the new slug: write the one this rename
- // started from, so the moved data is not left unrecorded.
- await writeChannelConfig(paths, newSlug, { ...config, dataDir: newTarget });
+ // started from, so the moved media is not left unrecorded.
+ await writeChannelConfig(paths, newSlug, { ...config, mediaDir: newTarget });
}
} catch (err) {
// The link goes back too, and before the directory under it moves: a
- // failure between the symlink and the config write left `data/` pointing
- // at <root>/<newSlug>/data while everything else was rolled back to the
- // old slug — a dangling link, which reads as an unmounted drive.
+ // failure between the symlink and the config write left `media`
+ // pointing at <root>/<newSlug>/media while everything else was rolled
+ // back to the old slug — a dangling link, which reads as an unmounted
+ // drive.
if (relinked || unlinked) {
- await unlink(path.join(newChannelDir, "data")).catch(() => {});
- await symlink(conventional, path.join(newChannelDir, "data")).catch(
- () => {},
- );
+ await unlink(link).catch(() => {});
+ await symlink(conventional, link).catch(() => {});
}
if (movedMedia) {
await rename(path.dirname(newTarget), path.dirname(conventional)).catch(
diff --git a/common/controller/storageLocations.test.ts b/common/controller/storageLocations.test.ts
@@ -28,11 +28,6 @@ import {
} from "./storageLocations";
import { readChannelConfig } from "./channels";
-// RELEASE 17 SLICE T1 made a channel whose `data/` is a link (or whose config
-// carries `dataDir`) `legacy`; these cases still build that retired layout and
-// expect it to read `ok`. Slice T2 rebases them on `media/` and un-skips them.
-const T1_SKIP = "release 17 T2 rebases the mover on media/";
-
// Run with:
// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/storageLocations.test.ts
@@ -109,35 +104,40 @@ async function withTmp(fn: (h: Harness) => Promise<void>): Promise<void> {
}
}
-// A channel whose media lives under `root`: a real dir on the "drive", an
-// absolute symlink at channels/<slug>/data, and config.dataDir naming it —
-// exactly what a finished relocation leaves behind.
+// A channel whose media tier lives under `root` (release 17): a real dir on the
+// "drive" holding the big file, an absolute symlink at channels/<slug>/media,
+// config.mediaDir naming it, and a real `data/` on the corpus disk whose audio
+// is the relative link into `media/` — exactly what a finished relocation
+// leaves behind.
async function seedRelocated(
h: Harness,
slug: string,
root: string,
opts: { link?: boolean; media?: boolean } = {},
): Promise<string> {
- const target = path.join(root, slug, "data");
+ const target = path.join(root, slug, "media");
if (opts.media !== false) {
await mkdir(path.join(target, "20240101_aaaaaaaaaaa"), { recursive: true });
- await writeFile(
- path.join(target, "20240101_aaaaaaaaaaa", "transcript.en.vtt"),
- "WEBVTT\n",
- );
+ await writeFile(path.join(target, "20240101_aaaaaaaaaaa", "audio.mp3"), "ID3");
}
const channelDir = path.join(h.paths.channelsDir, slug);
- await mkdir(channelDir, { recursive: true });
+ const videoDir = path.join(channelDir, "data", "20240101_aaaaaaaaaaa");
+ await mkdir(videoDir, { recursive: true });
+ await writeFile(path.join(videoDir, "transcript.en.vtt"), "WEBVTT\n");
+ await symlink(
+ path.join("..", "..", "media", "20240101_aaaaaaaaaaa", "audio.mp3"),
+ path.join(videoDir, "audio.mp3"),
+ );
await writeFile(
path.join(channelDir, "config.json"),
JSON.stringify(
- { handling: "transcribe", url: `https://example.com/${slug}`, dataDir: target },
+ { handling: "transcribe", url: `https://example.com/${slug}`, mediaDir: target },
null,
2,
) + "\n",
);
if (opts.link !== false) {
- await symlink(target, path.join(channelDir, "data"));
+ await symlink(target, path.join(channelDir, "media"));
}
return target;
}
@@ -151,7 +151,7 @@ function loc(id: string, root: string, extra: Partial<StorageLocation> = {}): St
return { id, label: id, root, autoRepoint: false, ...extra };
}
-test("channelsOnLocation buckets ok / unreachable / moving and ignores channels elsewhere", { skip: T1_SKIP }, async () => {
+test("channelsOnLocation buckets ok / unreachable / moving and ignores channels elsewhere", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
// Media dir never created: the link dangles, which is what an unmounted
@@ -160,9 +160,9 @@ test("channelsOnLocation buckets ok / unreachable / moving and ignores channels
await seedRelocated(h, "gamma", h.rootA);
await writeFile(
path.join(h.paths.channelsDir, "gamma", ".relocating.json"),
- JSON.stringify({ target: path.join(h.rootA, "gamma", "data"), phase: "copy" }),
+ JSON.stringify({ target: path.join(h.rootA, "gamma", "media"), phase: "copy" }),
);
- // On the other drive, and one plain in-place channel with no dataDir.
+ // On the other drive, and one plain in-place channel with no mediaDir.
await seedRelocated(h, "delta", h.rootB);
await mkdir(path.join(h.paths.channelsDir, "plain", "data"), {
recursive: true,
@@ -183,16 +183,196 @@ test("channelsOnLocation buckets ok / unreachable / moving and ignores channels
assert.equal(rollups.a.moving, 1);
assert.deepEqual(rollups.b.slugs, ["delta"]);
assert.equal(rollups.b.ok, 1);
+ assert.equal(rollups.a.legacy, 0);
+ });
+});
+
+// A LOCATION SHARED BY A MIGRATED AND A NOT-YET-MIGRATED CHANNEL still follows
+// its disk (release 17 review H1): the migration stops before the biggest
+// channels, so both layouts sit on the platter for a while. The legacy one is
+// re-pointed the retired way — its `data` link and `dataDir` to
+// `<newRoot>/<slug>/data`, the one shape the tier migration reads.
+async function seedLegacy(h: Harness, slug: string, root: string): Promise<string> {
+ const target = path.join(root, slug, "data");
+ await mkdir(path.join(target, "v1"), { recursive: true });
+ const channelDir = path.join(h.paths.channelsDir, slug);
+ await mkdir(channelDir, { recursive: true });
+ await writeFile(
+ path.join(channelDir, "config.json"),
+ JSON.stringify({ handling: "transcribe", url: `https://x/${slug}`, dataDir: target }) + "\n",
+ );
+ await symlink(target, path.join(channelDir, "data"));
+ return target;
+}
+
+test("re-point moves a legacy channel the retired way beside a migrated one", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "alpha", h.rootA);
+ await seedLegacy(h, "old", h.rootA);
+ // The disk moved: both trees are now under B.
+ await mkdir(path.join(h.rootB, "alpha", "media"), { recursive: true });
+ await mkdir(path.join(h.rootB, "old", "data", "v1"), { recursive: true });
+ await setLocations(h, [loc("cold", h.rootA)]);
+
+ const pre = await preflightRepoint({
+ paths: h.paths,
+ locationId: "cold",
+ newRoot: h.rootB,
+ io: h.io,
+ });
+ assert.deepEqual(pre.problems, []);
+ assert.deepEqual(pre.channels, ["alpha", "old"]);
+ assert.deepEqual(pre.legacy, ["old"]);
+
+ const result = await repointStorageLocation({
+ paths: h.paths,
+ locationId: "cold",
+ newRoot: h.rootB,
+ io: h.io,
+ });
+ assert.deepEqual(result.channels, ["alpha", "old"]);
+ assert.equal(
+ await readlink(path.join(h.paths.channelsDir, "alpha", "media")),
+ path.join(h.rootB, "alpha", "media"),
+ );
+ assert.equal(
+ (await readChannelConfig(h.paths, "alpha"))?.mediaDir,
+ path.join(h.rootB, "alpha", "media"),
+ );
+ const oldCfg = await readChannelConfig(h.paths, "old");
+ assert.equal(oldCfg?.dataDir, path.join(h.rootB, "old", "data"));
+ assert.equal(oldCfg?.mediaDir, undefined);
+ assert.equal(
+ await readlink(path.join(h.paths.channelsDir, "old", "data")),
+ path.join(h.rootB, "old", "data"),
+ );
+ // No `media` link was invented for the legacy channel.
+ await assert.rejects(() => readlink(path.join(h.paths.channelsDir, "old", "media")));
+ assert.equal(h.settings().storage.locations[0].root, h.rootB);
+ });
+});
+
+// A MIXED LOCATION ROLLS BACK AS ONE (the review's re-review): whichever of a
+// media channel and a legacy one is rewritten first, a failure on the other
+// puts both back — link, config key, no `media` link invented — and the
+// settings are never written.
+for (const [first, second, failing] of [
+ ["alpha", "old", "old"], // media channel rewritten, the legacy one fails
+ ["aaa", "zzz", "zzz"], // legacy rewritten first, the media one fails
+] as const) {
+ test(`mixed rollback: ${first} then ${second}, ${failing} fails`, async () => {
+ await withTmp(async (h) => {
+ const legacySlug = first === "alpha" ? second : first;
+ const mediaSlug = first === "alpha" ? first : second;
+ await seedRelocated(h, mediaSlug, h.rootA);
+ await seedLegacy(h, legacySlug, h.rootA);
+ await mkdir(path.join(h.rootB, mediaSlug, "media"), { recursive: true });
+ await mkdir(path.join(h.rootB, legacySlug, "data", "v1"), { recursive: true });
+ await setLocations(h, [loc("cold", h.rootA)]);
+ const failDir = path.join(h.paths.channelsDir, failing);
+ await chmod(failDir, 0o555);
+ try {
+ await assert.rejects(
+ repointStorageLocation({ paths: h.paths, locationId: "cold", newRoot: h.rootB, io: h.io }),
+ new RegExp(`${failing}: failed while`),
+ );
+ } finally {
+ await chmod(failDir, 0o755);
+ }
+ assert.equal(await readlink(path.join(h.paths.channelsDir, mediaSlug, "media")), path.join(h.rootA, mediaSlug, "media"));
+ assert.equal((await readChannelConfig(h.paths, mediaSlug))?.mediaDir, path.join(h.rootA, mediaSlug, "media"));
+ assert.equal(await readlink(path.join(h.paths.channelsDir, legacySlug, "data")), path.join(h.rootA, legacySlug, "data"));
+ const lc = await readChannelConfig(h.paths, legacySlug);
+ assert.equal(lc?.dataDir, path.join(h.rootA, legacySlug, "data"));
+ assert.equal(lc?.mediaDir, undefined);
+ await assert.rejects(() => readlink(path.join(h.paths.channelsDir, legacySlug, "media")));
+ assert.equal(h.settings().storage.locations[0].root, h.rootA);
+ assert.equal(h.writes.length, 0);
+ });
+ });
+}
+
+test("a legacy channel whose retired tree is missing under the new root is named, and nothing moves", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "alpha", h.rootA);
+ const oldTarget = await seedLegacy(h, "old", h.rootA);
+ await mkdir(path.join(h.rootB, "alpha", "media"), { recursive: true });
+ await setLocations(h, [loc("cold", h.rootA)]);
+ await assert.rejects(
+ () =>
+ repointStorageLocation({
+ paths: h.paths,
+ locationId: "cold",
+ newRoot: h.rootB,
+ io: h.io,
+ }),
+ /old\. Expected .*not yet migrated/,
+ );
+ assert.equal(
+ await readlink(path.join(h.paths.channelsDir, "old", "data")),
+ oldTarget,
+ );
+ assert.equal(
+ await readlink(path.join(h.paths.channelsDir, "alpha", "media")),
+ path.join(h.rootA, "alpha", "media"),
+ );
+ });
+});
+
+// THE RETIRED LAYOUT (release 17): a channel whose whole `data/` was moved
+// before the media tier is still ON its location (by its retired `dataDir`),
+// counted unreachable — its text and media are held until migrate-tier — and
+// named in `legacy`, the "(n to migrate)" beside it. Byte figures: the media
+// tier on the row, the corpus-disk text and clips beside it, an old snapshot's
+// missing text figure counted unknown.
+test("channelsOnLocation counts a legacy channel as unreachable and to migrate, and sums the tiers", async () => {
+ await withTmp(async (h) => {
+ await seedRelocated(h, "alpha", h.rootA);
+ const oldTarget = path.join(h.rootA, "old", "data");
+ await mkdir(path.join(oldTarget, "v1"), { recursive: true });
+ const oldDir = path.join(h.paths.channelsDir, "old");
+ await mkdir(oldDir, { recursive: true });
+ await writeFile(
+ path.join(oldDir, "config.json"),
+ JSON.stringify({ handling: "transcribe", url: "https://x/o", dataDir: oldTarget }) + "\n",
+ );
+ await symlink(oldTarget, path.join(oldDir, "data"));
+ await mkdir(path.join(h.paths.channelsDir, "plain", "data"), { recursive: true });
+ await writeFile(
+ path.join(h.paths.channelsDir, "plain", "config.json"),
+ JSON.stringify({ handling: "transcribe", url: "https://x/y" }) + "\n",
+ );
+
+ const rollups = await channelsOnLocation({
+ paths: h.paths,
+ locations: [loc("a", h.rootA)],
+ includeInternal: true,
+ mediaBytes: { alpha: 100, old: 50, plain: 7 },
+ clipsBytes: { alpha: 3, old: 0, plain: 2 },
+ textBytes: { alpha: 10, plain: 5 },
+ });
+ assert.deepEqual(rollups.a.slugs, ["alpha", "old"]);
+ assert.equal(rollups.a.ok, 1);
+ assert.equal(rollups.a.unreachable, 1);
+ assert.equal(rollups.a.legacy, 1);
+ assert.equal(rollups.a.bytes, 150);
+ assert.equal(rollups.a.clipsBytes, 3);
+ assert.equal(rollups.a.textBytes, 10);
+ assert.equal(rollups.a.unknownTextBytes, 1, "the old snapshot has no text figure");
+ assert.deepEqual(rollups.internal.slugs, ["plain"]);
+ assert.equal(rollups.internal.bytes, 7);
+ assert.equal(rollups.internal.textBytes, 5);
+ assert.equal(rollups.internal.clipsBytes, 2);
});
});
-test("re-point rewrites both channels' links and configs, then the location", { skip: T1_SKIP }, async () => {
+test("re-point rewrites both channels' links and configs, then the location", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
await seedRelocated(h, "beta", h.rootA);
// The media is already on drive B — the disk moved, the bytes did not.
- await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
- await mkdir(path.join(h.rootB, "beta", "data"), { recursive: true });
+ await mkdir(path.join(h.rootB, "alpha", "media"), { recursive: true });
+ await mkdir(path.join(h.rootB, "beta", "media"), { recursive: true });
await setLocations(h, [
loc("cold", h.rootA, {
volume: {
@@ -215,12 +395,12 @@ test("re-point rewrites both channels' links and configs, then the location", {
assert.deepEqual(result.channels, ["alpha", "beta"]);
for (const slug of ["alpha", "beta"]) {
- const target = path.join(h.rootB, slug, "data");
+ const target = path.join(h.rootB, slug, "media");
assert.equal(
- await readlink(path.join(h.paths.channelsDir, slug, "data")),
+ await readlink(path.join(h.paths.channelsDir, slug, "media")),
target,
);
- assert.equal((await readChannelConfig(h.paths, slug))?.dataDir, target);
+ assert.equal((await readChannelConfig(h.paths, slug))?.mediaDir, target);
}
// The settings write is last and carries the new root; the identity is
// re-anchored so root === join(mountpoint, relPath) still holds.
@@ -233,11 +413,11 @@ test("re-point rewrites both channels' links and configs, then the location", {
});
});
-test("re-point refuses a target that has no media for a channel, naming the slug", { skip: T1_SKIP }, async () => {
+test("re-point refuses a target that has no media for a channel, naming the slug", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
await seedRelocated(h, "beta", h.rootA);
- await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
+ await mkdir(path.join(h.rootB, "alpha", "media"), { recursive: true });
await setLocations(h, [loc("cold", h.rootA)]);
const pre = await preflightRepoint({
@@ -262,19 +442,19 @@ test("re-point refuses a target that has no media for a channel, naming the slug
);
// Nothing was written: not the link that COULD have moved, not settings.
assert.equal(
- await readlink(path.join(h.paths.channelsDir, "alpha", "data")),
- path.join(h.rootA, "alpha", "data"),
+ await readlink(path.join(h.paths.channelsDir, "alpha", "media")),
+ path.join(h.rootA, "alpha", "media"),
);
assert.equal(h.writes.length, 0);
});
});
-test("a failure on the second channel rolls the first one back", { skip: T1_SKIP }, async () => {
+test("a failure on the second channel rolls the first one back", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
await seedRelocated(h, "beta", h.rootA);
- await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
- await mkdir(path.join(h.rootB, "beta", "data"), { recursive: true });
+ await mkdir(path.join(h.rootB, "alpha", "media"), { recursive: true });
+ await mkdir(path.join(h.rootB, "beta", "media"), { recursive: true });
await setLocations(h, [loc("cold", h.rootA)]);
// A READ-ONLY CHANNEL DIR is the cheapest real failure: the unlink of
@@ -299,12 +479,12 @@ test("a failure on the second channel rolls the first one back", { skip: T1_SKIP
// alpha is back where it started — link AND config, and it reads as the
// pre-job `unreachable`, never `inconsistent`.
assert.equal(
- await readlink(path.join(h.paths.channelsDir, "alpha", "data")),
- path.join(h.rootA, "alpha", "data"),
+ await readlink(path.join(h.paths.channelsDir, "alpha", "media")),
+ path.join(h.rootA, "alpha", "media"),
);
assert.equal(
- (await readChannelConfig(h.paths, "alpha"))?.dataDir,
- path.join(h.rootA, "alpha", "data"),
+ (await readChannelConfig(h.paths, "alpha"))?.mediaDir,
+ path.join(h.rootA, "alpha", "media"),
);
// And the location never moved.
assert.equal(h.settings().storage.locations[0].root, h.rootA);
@@ -312,10 +492,10 @@ test("a failure on the second channel rolls the first one back", { skip: T1_SKIP
});
});
-test("re-point refuses a busy channel and names it", { skip: T1_SKIP }, async () => {
+test("re-point refuses a busy channel and names it", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
- await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
+ await mkdir(path.join(h.rootB, "alpha", "media"), { recursive: true });
await setLocations(h, [loc("cold", h.rootA)]);
const pre = await preflightRepoint({
@@ -330,25 +510,25 @@ test("re-point refuses a busy channel and names it", { skip: T1_SKIP }, async ()
});
});
-test("a rerun after a crash finishes the channels that were left", { skip: T1_SKIP }, async () => {
+test("a rerun after a crash finishes the channels that were left", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
await seedRelocated(h, "beta", h.rootA);
- await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
- await mkdir(path.join(h.rootB, "beta", "data"), { recursive: true });
+ await mkdir(path.join(h.rootB, "alpha", "media"), { recursive: true });
+ await mkdir(path.join(h.rootB, "beta", "media"), { recursive: true });
await setLocations(h, [loc("cold", h.rootA)]);
// A CRASH, not a rollback: alpha was re-pointed and the process died before
// beta and before the settings write. Rebuilt by hand, which is exactly the
// state on disk.
- const alphaLink = path.join(h.paths.channelsDir, "alpha", "data");
+ const alphaLink = path.join(h.paths.channelsDir, "alpha", "media");
await rm(alphaLink);
- await symlink(path.join(h.rootB, "alpha", "data"), alphaLink);
+ await symlink(path.join(h.rootB, "alpha", "media"), alphaLink);
const alphaConfig = await readChannelConfig(h.paths, "alpha");
await writeFile(
path.join(h.paths.channelsDir, "alpha", "config.json"),
JSON.stringify(
- { ...alphaConfig, dataDir: path.join(h.rootB, "alpha", "data") },
+ { ...alphaConfig, mediaDir: path.join(h.rootB, "alpha", "media") },
null,
2,
) + "\n",
@@ -363,8 +543,8 @@ test("a rerun after a crash finishes the channels that were left", { skip: T1_SK
// alpha is NOT on the old root any more, so the rerun does not touch it.
assert.deepEqual(result.channels, ["beta"]);
assert.equal(
- await readlink(path.join(h.paths.channelsDir, "beta", "data")),
- path.join(h.rootB, "beta", "data"),
+ await readlink(path.join(h.paths.channelsDir, "beta", "media")),
+ path.join(h.rootB, "beta", "media"),
);
assert.equal(h.settings().storage.locations[0].root, h.rootB);
@@ -381,12 +561,12 @@ test("a rerun after a crash finishes the channels that were left", { skip: T1_SK
});
});
-test("a channel killed between its symlink and its config write is resumed, not refused", { skip: T1_SKIP }, async () => {
+test("a channel killed between its symlink and its config write is resumed, not refused", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
await seedRelocated(h, "beta", h.rootA);
- await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
- await mkdir(path.join(h.rootB, "beta", "data"), { recursive: true });
+ await mkdir(path.join(h.rootB, "alpha", "media"), { recursive: true });
+ await mkdir(path.join(h.rootB, "beta", "media"), { recursive: true });
await setLocations(h, [loc("cold", h.rootA)]);
// THE ONE-INSTRUCTION WINDOW. alpha's link was moved and the process died
@@ -394,9 +574,9 @@ test("a channel killed between its symlink and its config write is resumed, not
// `inconsistent`, and alpha is still "on" the old root — so a preflight
// that only accepted ok/unreachable would refuse the whole location for
// ever, naming a state whose only remedy is the job it is refusing.
- const alphaLink = path.join(h.paths.channelsDir, "alpha", "data");
+ const alphaLink = path.join(h.paths.channelsDir, "alpha", "media");
await rm(alphaLink);
- await symlink(path.join(h.rootB, "alpha", "data"), alphaLink);
+ await symlink(path.join(h.rootB, "alpha", "media"), alphaLink);
const pre = await preflightRepoint({
paths: h.paths,
@@ -420,16 +600,16 @@ test("a channel killed between its symlink and its config write is resumed, not
// alpha's config caught up with its link (which was never touched again),
// beta went the ordinary way, and the location moved.
assert.equal(
- (await readChannelConfig(h.paths, "alpha"))?.dataDir,
- path.join(h.rootB, "alpha", "data"),
+ (await readChannelConfig(h.paths, "alpha"))?.mediaDir,
+ path.join(h.rootB, "alpha", "media"),
);
assert.equal(
- await readlink(path.join(h.paths.channelsDir, "alpha", "data")),
- path.join(h.rootB, "alpha", "data"),
+ await readlink(path.join(h.paths.channelsDir, "alpha", "media")),
+ path.join(h.rootB, "alpha", "media"),
);
assert.equal(
- (await readChannelConfig(h.paths, "beta"))?.dataDir,
- path.join(h.rootB, "beta", "data"),
+ (await readChannelConfig(h.paths, "beta"))?.mediaDir,
+ path.join(h.rootB, "beta", "media"),
);
assert.equal(h.settings().storage.locations[0].root, h.rootB);
assert.ok(lines.some((l) => l.includes("link was already moved")));
@@ -439,10 +619,10 @@ test("a channel killed between its symlink and its config write is resumed, not
test("preflight refuses an in-transition channel and a root that is not a directory", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
- await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
+ await mkdir(path.join(h.rootB, "alpha", "media"), { recursive: true });
await writeFile(
path.join(h.paths.channelsDir, "alpha", ".relocating.json"),
- JSON.stringify({ target: path.join(h.rootA, "alpha", "data"), phase: "copy" }),
+ JSON.stringify({ target: path.join(h.rootA, "alpha", "media"), phase: "copy" }),
);
await setLocations(h, [loc("cold", h.rootA)]);
diff --git a/common/controller/storageLocations.ts b/common/controller/storageLocations.ts
@@ -1,5 +1,5 @@
import path from "node:path";
-import { readlink, stat, symlink, unlink } from "node:fs/promises";
+import { lstat, readlink, stat, symlink, unlink } from "node:fs/promises";
import { getPaths, type Paths } from "../lib/paths";
import { getFreeBytes } from "../lib/diskSpace";
import {
@@ -29,6 +29,7 @@ import {
inspectChannelMedia,
relocatedDataDir,
} from "../lib/channelMedia";
+import { MEDIA_LINK_NAME, relocatedMediaDir } from "../lib/mediaTier-server";
import {
isDriveNotAnswering,
onDrive,
@@ -54,8 +55,9 @@ import { relocationRootProblem } from "./relocateChannelMedia";
// knows the corpus: which channels are on a location, whether a re-point is
// safe, and the re-point itself.
//
-// RE-POINT MOVES NO BYTES. It rewrites each channel's `data/` symlink and its
-// `config.dataDir`, then the location's `root`. That is the entire operation —
+// RE-POINT MOVES NO BYTES. It rewrites each channel's `media` symlink and its
+// `config.mediaDir`, then the location's `root` (release 17: a channel's media
+// tier is what lives on a location; its text never leaves the corpus disk). That is the entire operation —
// the media is already where it is going, because the DISK came up somewhere
// else and took it along. The relocate job (which does move bytes) is a
// different thing entirely, and the two share a queue key precisely so they can
@@ -95,10 +97,12 @@ const DEFAULT_SETTINGS_IO: SettingsIO = {
export type LocationRollup = {
locationId: string;
- // Every channel whose `config.dataDir` is under this location's root, sorted.
+ // Every channel whose `config.mediaDir` is under this location's root (or,
+ // on a `legacy` channel, its retired `config.dataDir`), sorted.
slugs: string[];
total: number;
- // Sum of `snapshot.totalMediaBytes` over the channels on this location, and
+ // Sum of `snapshot.totalMediaBytes` over the channels on this location — the
+ // MEDIA TIER's bytes, which is what a location holds (release 17) — and
// how many of them could not contribute one (no snapshot, or one written
// before the field existed). THE SECOND NUMBER IS WHY THE FIRST IS HONEST: a
// location whose channels have never had a report reads `0 bytes` otherwise,
@@ -106,16 +110,20 @@ export type LocationRollup = {
// surface renders "+ n unknown" beside the total.
bytes: number;
unknownBytes: number;
- // THE `clips/` SHARE OF `bytes`, not a sibling of it. Fetched clip windows
- // live under `data/<id>/clips/` and are counted into `totalMediaBytes` like
- // everything else there — this is the "of which", because clips are the one
- // part of a location's bytes that is a CACHE nothing prunes, and an operator
- // looking at a full drive wants to know how much of it is that.
+ // THE CORPUS-DISK BYTES of these channels: `clips/` (the fetched clip
+ // windows, a cache nothing prunes) and the text tier. A SIBLING of `bytes`
+ // since release 17, not a share of it — both stay on the corpus volume
+ // whatever location the media is on, so only the INTERNAL row adds them to
+ // what it holds; a location row shows them as "on the corpus volume".
//
// No `unknownClipsBytes`: it would be the same set of channels
- // `unknownBytes` already counts (the two fields are written by one snapshot
+ // `unknownBytes` already counts (the fields are written by one snapshot
// pass), and a second copy of one number is a second thing to keep in step.
+ // `textBytes` has its own unknown count: a snapshot written before release
+ // 17 carries clips and media but no text figure.
clipsBytes: number;
+ textBytes: number;
+ unknownTextBytes: number;
// `inspectChannelMedia` status, bucketed into the three numbers the page
// shows. `unreachable` DELIBERATELY ABSORBS `inconsistent`: both mean "this
// channel's media is not readable through its link right now", which is the
@@ -126,6 +134,10 @@ export type LocationRollup = {
// `in-transition`: a relocation marker is present. A location with any of
// these is one no re-point may touch.
moving: number;
+ // `legacy` — the retired whole-directory layout, waiting for
+ // `archilyzer storage migrate-tier`. Counted in `unreachable` too (its
+ // text and media are held); this is the "(n to migrate)" beside it.
+ legacy: number;
};
function emptyRollup(locationId: string): LocationRollup {
@@ -136,9 +148,12 @@ function emptyRollup(locationId: string): LocationRollup {
bytes: 0,
unknownBytes: 0,
clipsBytes: 0,
+ textBytes: 0,
+ unknownTextBytes: 0,
ok: 0,
unreachable: 0,
moving: 0,
+ legacy: 0,
};
}
@@ -153,8 +168,8 @@ function emptyRollup(locationId: string): LocationRollup {
// `settings.storage.locations` would make it deletable and would make
// `locationOfDataDir` match every unrelocated channel — which would break the
// one rule the whole design rests on, that a channel is on a location iff its
-// `dataDir` is under that location's root, and an in-place channel HAS no
-// `dataDir`.
+// `mediaDir` is under that location's root, and an in-place channel HAS no
+// `mediaDir`.
//
// So it is assembled where it is rendered, out of the same three facts every
// other row carries, and it is the one row with no actions.
@@ -168,7 +183,7 @@ export { INTERNAL_LOCATION_ID, INTERNAL_LOCATION_LABEL };
// page draws a row per location and the channel list is the same list for all
// of them. Cost is `listChannelConfigs` (one readdir + one config read per
// channel) plus `inspectChannelMedia` (two stats) for the channels that are
-// actually on a location — an unrelocated channel has no `dataDir` and is
+// actually on a location — an unrelocated channel has no `mediaDir` and is
// skipped before it costs a stat.
export async function channelsOnLocation(opts: {
paths: Paths;
@@ -183,12 +198,16 @@ export async function channelsOnLocation(opts: {
// file in one render. A slug that is absent (or maps to undefined) counts
// towards `unknownBytes`, never towards `bytes`.
mediaBytes?: Readonly<Record<string, number | undefined>>;
- // `snapshot.totalClipsBytes` per slug — the `clips/` share of the above.
+ // `snapshot.totalClipsBytes` per slug — the clip cache, on the corpus disk.
// Absent for a snapshot written before the field existed, which is the same
// set `mediaBytes` is absent for.
clipsBytes?: Readonly<Record<string, number | undefined>>;
+ // `snapshot.totalTextBytes` per slug — the text tier, on the corpus disk.
+ // Absent for a snapshot written before release 17: unknown, never 0.
+ textBytes?: Readonly<Record<string, number | undefined>>;
// The in-place row (see INTERNAL_LOCATION_ID). When true, every channel with
- // NO `dataDir` is rolled up under that id alongside the configured ones.
+ // NO `mediaDir` (and no retired `dataDir`) is rolled up under that id
+ // alongside the configured ones.
includeInternal?: boolean;
}): Promise<Record<string, LocationRollup>> {
const out: Record<string, LocationRollup> = {};
@@ -200,17 +219,19 @@ export async function channelsOnLocation(opts: {
const configs = opts.configs ?? (await listChannelConfigs(opts.paths));
for (const { slug, config } of configs) {
- const dataDir = config.dataDir?.trim();
- const loc = dataDir
- ? locationOfDataDir(dataDir, opts.locations as StorageLocation[])
+ // Where the channel's media is: `mediaDir`, or on a legacy channel the
+ // retired `dataDir` (its whole tree is still there until migrate-tier).
+ const mediaDir = config.mediaDir?.trim() || config.dataDir?.trim();
+ const loc = mediaDir
+ ? locationOfDataDir(mediaDir, opts.locations as StorageLocation[])
: null;
- // In place: no recorded dataDir at all. A dataDir under a root NOBODY named
+ // In place: no recorded target at all. A target under a root NOBODY named
// is neither in place nor on a location, and it is deliberately counted in
// neither — /storage says so by the totals not adding up to the corpus, and
// the remedy is to name that root as a location.
const id = loc
? loc.id
- : !dataDir && opts.includeInternal
+ : !mediaDir && opts.includeInternal
? INTERNAL_LOCATION_ID
: null;
if (!id) continue;
@@ -222,10 +243,16 @@ export async function channelsOnLocation(opts: {
else roll.unknownBytes += 1;
const clips = opts.clipsBytes?.[slug];
if (typeof clips === "number") roll.clipsBytes += clips;
+ const text = opts.textBytes?.[slug];
+ if (typeof text === "number") roll.textBytes += text;
+ else roll.unknownTextBytes += 1;
const media = await inspectChannelMedia(opts.paths, slug, config);
if (media.status === "ok" || media.status === "in-place") roll.ok += 1;
else if (media.status === "in-transition") roll.moving += 1;
- else roll.unreachable += 1;
+ else {
+ roll.unreachable += 1;
+ if (media.status === "legacy") roll.legacy += 1;
+ }
}
for (const roll of Object.values(out)) roll.slugs.sort();
return out;
@@ -439,6 +466,13 @@ export type RepointPreflight = {
// wholesale, naming a state with no remedy. For these the job writes the
// config and does NOT touch the link: the link is already right.
resumable: string[];
+ // A SUBSET of `channels`: the ones still on the RETIRED whole-directory
+ // layout (`legacy`, release 17). They are re-pointed the retired way — their
+ // `data` link and `config.dataDir` to `<newRoot>/<slug>/data` — so a
+ // location holding migrated and not-yet-migrated channels side by side
+ // still follows its disk, and the tier migration only ever sees
+ // `dataDir = <root>/<slug>/data`.
+ legacy: string[];
// The identity the new root actually has, when the preflight was able to
// probe it (bins passed AND the location has a recorded uuid to compare
// against). The job writes THIS rather than deriving a mountpoint from the
@@ -449,16 +483,39 @@ export type RepointPreflight = {
newRoot: string;
};
-// Does `channels/<slug>/data` already point exactly where a re-point would put
-// it? lstat/readlink, never stat: the target may not exist yet either, and a
-// stat would call a perfectly good link missing.
+// What a re-point rewrites for one channel: the `media` link and `mediaDir`,
+// or — a legacy channel — the retired `data` link and `dataDir`.
+type RepointShape = {
+ linkName: string;
+ key: "mediaDir" | "dataDir";
+ targetFor: (root: string, slug: string) => string;
+};
+
+const MEDIA_SHAPE: RepointShape = {
+ linkName: MEDIA_LINK_NAME,
+ key: "mediaDir",
+ targetFor: relocatedMediaDir,
+};
+const LEGACY_SHAPE: RepointShape = {
+ linkName: "data",
+ key: "dataDir",
+ targetFor: relocatedDataDir,
+};
+
+// Does the channel's link (`media`, or a legacy channel's `data`) already point
+// exactly where a re-point would put it? lstat/readlink, never stat: the target
+// may not exist yet either, and a stat would call a perfectly good link
+// missing.
async function linkAlreadyAt(
paths: Paths,
slug: string,
newTarget: string,
+ linkName: string = MEDIA_LINK_NAME,
): Promise<boolean> {
try {
- const linkTarget = await readlink(path.join(paths.channelsDir, slug, "data"));
+ const linkTarget = await readlink(
+ path.join(paths.channelsDir, slug, linkName),
+ );
return path.resolve(linkTarget) === path.resolve(newTarget);
} catch {
return false;
@@ -495,6 +552,7 @@ export async function preflightRepoint(opts: {
problems: [],
channels: [],
resumable: [],
+ legacy: [],
locationId: opts.locationId,
oldRoot: loc?.root ?? "",
newRoot,
@@ -580,12 +638,29 @@ export async function preflightRepoint(opts: {
// is the job being refused. When the link already points exactly where this
// run would point it, the channel is not broken: it is half done, and the
// remaining half is the config write this job performs anyway.
+ // THE RETIRED LAYOUT (release 17) is re-pointed the retired way: its
+ // `data` link and `dataDir`. Only a channel whose `data/` IS a link — a
+ // recorded `dataDir` over a real `data/` is a disagreement, refused below.
+ const legacy = media.status === "legacy";
+ const shape = legacy ? LEGACY_SHAPE : MEDIA_SHAPE;
const resumable = await linkAlreadyAt(
opts.paths,
slug,
- relocatedDataDir(newRoot, slug),
+ shape.targetFor(newRoot, slug),
+ shape.linkName,
);
- if (!resumable && media.status !== "ok" && media.status !== "unreachable") {
+ if (legacy) {
+ const dataIsLink = await lstat(path.join(opts.paths.channelsDir, slug, "data"))
+ .then((l) => l.isSymbolicLink())
+ .catch(() => false);
+ if (!dataIsLink) {
+ base.problems.push(
+ `${slug}: config.json records the retired dataDir but its data/ is ` +
+ `not a link — ${media.detail ?? "run archilyzer storage migrate-tier"}.`,
+ );
+ continue;
+ }
+ } else if (!resumable && media.status !== "ok" && media.status !== "unreachable") {
base.problems.push(
`${slug}: its media reads as ${media.status} — ` +
`${media.detail ?? "disk and config do not agree"}. A re-point ` +
@@ -598,7 +673,7 @@ export async function preflightRepoint(opts: {
base.problems.push(`${slug}: ${busy}`);
continue;
}
- const target = relocatedDataDir(newRoot, slug);
+ const target = shape.targetFor(newRoot, slug);
if (!(await isDirectory(target))) {
missing.push(slug);
continue;
@@ -614,6 +689,7 @@ export async function preflightRepoint(opts: {
}
base.channels.push(slug);
if (resumable) base.resumable.push(slug);
+ if (legacy) base.legacy.push(slug);
}
// THE MISSING TARGETS ARE ONE REFUSAL, NOT n. A root that holds none of the
@@ -624,7 +700,8 @@ export async function preflightRepoint(opts: {
base.problems.push(
`${missing.length} channel(s) have no media under ${newRoot}: ` +
`${missing.join(", ")}. Expected ` +
- `${relocatedDataDir(newRoot, missing[0])} and friends.`,
+ `${relocatedMediaDir(newRoot, missing[0])} (or, for a channel not yet ` +
+ `migrated, ${relocatedDataDir(newRoot, missing[0])}) and friends.`,
);
}
@@ -638,6 +715,8 @@ export async function preflightRepoint(opts: {
type ChannelLedgerEntry = {
slug: string;
+ // Which link and config key this channel's re-point rewrites.
+ shape: RepointShape;
oldTarget: string;
newTarget: string;
unlinked: boolean;
@@ -669,11 +748,11 @@ async function rollbackChannel(
// Only the field this job changed goes back; anything else edited since
// stays. (It used to rewrite the whole config it found at the start.)
await patchChannelConfig(paths, entry.slug, {
- dataDir: entry.oldTarget,
+ [entry.shape.key]: entry.oldTarget,
}).catch(() => {});
}
if (entry.relinked || entry.unlinked) {
- const link = path.join(paths.channelsDir, entry.slug, "data");
+ const link = path.join(paths.channelsDir, entry.slug, entry.shape.linkName);
await unlink(link).catch(() => {});
await symlink(entry.oldTarget, link).catch(() => {});
}
@@ -692,7 +771,7 @@ async function rollbackChannel(
// direction.
//
// IDEMPOTENT RERUN. A channel that was already re-pointed is no longer ON the
-// old root (`locationOfDataDir` reads its `dataDir`), so the preflight does not
+// old root (`locationOfDataDir` reads its `mediaDir`), so the preflight does not
// list it and this does not touch it. A rerun after a crash finishes the rest;
// a rerun after a complete run finds no channels and refuses with "already at".
export async function repointStorageLocation(opts: {
@@ -735,10 +814,12 @@ export async function repointStorageLocation(opts: {
for (const slug of pre.channels) {
opts.signal?.throwIfAborted();
const fresh = await readChannelConfig(opts.paths, slug);
- const oldTarget = fresh?.dataDir?.trim() ?? "";
- const newTarget = relocatedDataDir(pre.newRoot, slug);
+ const shape = pre.legacy.includes(slug) ? LEGACY_SHAPE : MEDIA_SHAPE;
+ const oldTarget = fresh?.[shape.key]?.trim() ?? "";
+ const newTarget = shape.targetFor(pre.newRoot, slug);
const entry: ChannelLedgerEntry = {
slug,
+ shape,
oldTarget,
newTarget,
unlinked: false,
@@ -747,11 +828,11 @@ export async function repointStorageLocation(opts: {
oldConfig: fresh,
};
ledger.push(entry);
- const link = path.join(opts.paths.channelsDir, slug, "data");
+ const link = path.join(opts.paths.channelsDir, slug, shape.linkName);
// RESUMING SKIPS THE LINK, and must: it already points at newTarget, so
// unlinking and recreating it would be two syscalls to reach the state it
// is in — and a crash between them would turn a half-done channel into a
- // channel with no `data/` at all, which is strictly worse than what we
+ // channel with no `media` at all, which is strictly worse than what we
// found. The ledger records nothing for the link for the same reason: a
// rollback must undo what THIS run did, and this run did not move it.
const resuming = pre.resumable.includes(slug);
@@ -764,9 +845,9 @@ export async function repointStorageLocation(opts: {
}
// A null patch means config.json vanished or became unreadable after
// preflight listed the channel: fail this step, so the ledger rolls
- // the link back, rather than leave a link no dataDir records.
+ // the link back, rather than leave a link no mediaDir records.
const written = await patchChannelConfig(opts.paths, slug, {
- dataDir: newTarget,
+ [shape.key]: newTarget,
});
if (!written) {
throw new Error(`channels/${slug}/config.json is missing or unreadable`);
@@ -831,7 +912,7 @@ export async function repointStorageLocation(opts: {
}
resetStorageProbeMemo();
- // Every channel on it has a new link and a new dataDir: the page memo's keys
+ // Every channel on it has a new link and a new mediaDir: the page memo's keys
// already differ, and this drops the old answers rather than letting them
// age out.
forgetChannelMedia();
diff --git a/common/controller/storageWatch.test.ts b/common/controller/storageWatch.test.ts
@@ -30,11 +30,6 @@ import {
type LocationHealthState,
} from "../lib/storageHealth";
-// RELEASE 17 SLICE T1 made a channel whose `data/` is a link (or whose config
-// carries `dataDir`) `legacy`; these cases still build that retired layout and
-// expect it to read `ok`. Slice T2 rebases them on `media/` and un-skips them.
-const T1_SKIP = "release 17 T2 rebases the storage watch on mediaDir";
-
// THE CONFIRMATION COUNT IS MODULE STATE (see storageWatch.ts rule 3), so each
// case starts from a clean one — otherwise the second test inherits the first
@@ -115,8 +110,9 @@ async function withTmp(fn: (h: H) => Promise<void>): Promise<void> {
}
}
-// A channel whose `data/` is a link to `<root>/<slug>/data`, with the target
-// present or not.
+// A channel whose media tier is relocated (release 17): `channels/<slug>/media`
+// a link to `<root>/<slug>/media`, recorded as `mediaDir`, with the target
+// present or not. Its `data/` (the text) is on the corpus disk throughout.
async function seedRelocated(
h: H,
slug: string,
@@ -124,13 +120,13 @@ async function seedRelocated(
): Promise<void> {
const channelDir = path.join(h.paths.channelsDir, slug);
await mkdir(channelDir, { recursive: true });
- const target = path.join(h.root, slug, "data");
+ const target = path.join(h.root, slug, "media");
if (opts.targetExists) await mkdir(target, { recursive: true });
await writeFile(
path.join(channelDir, "config.json"),
- JSON.stringify({ handling: "youtube", dataDir: target }),
+ JSON.stringify({ handling: "youtube", mediaDir: target }),
);
- await symlink(target, path.join(channelDir, "data"));
+ await symlink(target, path.join(channelDir, "media"));
}
function tierOf(h: H, slug: string): string | undefined {
@@ -155,7 +151,7 @@ async function twoPasses(h: H) {
return { first, second };
}
-test("a channel whose target is gone is auto-paused, once, in one write", { skip: T1_SKIP }, async () => {
+test("a channel whose target is gone is auto-paused, once, in one write", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "gone-a", { targetExists: false });
await seedRelocated(h, "gone-b", { targetExists: false });
@@ -195,7 +191,7 @@ test("a channel whose target is gone is auto-paused, once, in one write", { skip
});
});
-test("the drive coming back restores the tier it overwrote", { skip: T1_SKIP }, async () => {
+test("the drive coming back restores the tier it overwrote", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "away", { targetExists: false });
const settings = h.io.read();
@@ -210,7 +206,7 @@ test("the drive coming back restores the tier it overwrote", { skip: T1_SKIP },
await twoPasses(h);
assert.equal(tierOf(h, "away"), "paused");
- await mkdir(path.join(h.root, "away", "data"), { recursive: true });
+ await mkdir(path.join(h.root, "away", "media"), { recursive: true });
const back = await runStorageWatchPass({ paths: h.paths, io: h.io, bins: h.paths });
assert.deepEqual(back.restored, ["away"]);
assert.equal(tierOf(h, "away"), "low");
@@ -254,7 +250,7 @@ test("a channel mid-relocation is not auto-paused", async () => {
await writeFile(
path.join(h.paths.channelsDir, "moving", ".relocating.json"),
JSON.stringify({
- target: path.join(h.root, "moving", "data"),
+ target: path.join(h.root, "moving", "media"),
direction: "out",
startedAt: "",
phase: "copy",
@@ -301,7 +297,7 @@ test("a record on an in-place channel is restored", async () => {
// `write: false` IS IDLE BOOT. It observes and reports; the write is the work,
// and idle boot refuses work.
-test("write: false reports the transition and changes nothing", { skip: T1_SKIP }, async () => {
+test("write: false reports the transition and changes nothing", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "gone", { targetExists: false });
// The first pass only suspects, whatever `write` says.
@@ -340,7 +336,7 @@ test("no locations and nothing auto-paused is a free pass", async () => {
// has spun down and needs a beat to answer is indistinguishable from "not
// mounted" — and pausing on it rewrites the corpus's priority document for a
// drive that is fine.
-test("a drive that blips for one pass is never paused", { skip: T1_SKIP }, async () => {
+test("a drive that blips for one pass is never paused", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "blip", { targetExists: false });
const first = await runStorageWatchPass({
@@ -355,7 +351,7 @@ test("a drive that blips for one pass is never paused", { skip: T1_SKIP }, async
// It answers on the next pass. Nothing was ever paused, and the suspicion
// is dropped — so a LATER real outage starts its own two-pass count rather
// than pausing immediately on the strength of a blip an hour ago.
- await mkdir(path.join(h.root, "blip", "data"), { recursive: true });
+ await mkdir(path.join(h.root, "blip", "media"), { recursive: true });
const second = await runStorageWatchPass({
paths: h.paths,
io: h.io,
@@ -385,13 +381,13 @@ test("a drive that blips for one pass is never paused", { skip: T1_SKIP }, async
// RESTORE STAYS SINGLE-PASS, and the asymmetry is the point: being slow to
// pause costs a few refused units (the start-of-work guards catch those), while
// being slow to restore leaves a lane off after the operator fixed the cable.
-test("the restore needs only one good pass", { skip: T1_SKIP }, async () => {
+test("the restore needs only one good pass", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "back", { targetExists: false });
await twoPasses(h);
assert.equal(tierOf(h, "back"), "paused");
h.writes = 0;
- await mkdir(path.join(h.root, "back", "data"), { recursive: true });
+ await mkdir(path.join(h.root, "back", "media"), { recursive: true });
const r = await runStorageWatchPass({
paths: h.paths,
io: h.io,
@@ -415,7 +411,7 @@ function scripted(answers: LocationHealthState[]) {
return async () => answers[Math.min(i++, answers.length - 1)];
}
-test("one missed probe stalls the location; pages then answer 'stalled' without asking", { skip: T1_SKIP }, async () => {
+test("one missed probe stalls the location; pages then answer 'stalled' without asking", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "slow", { targetExists: true });
const lines: string[] = [];
@@ -440,7 +436,7 @@ test("one missed probe stalls the location; pages then answer 'stalled' without
});
});
-test("the stall clears only after two clean probes in a row", { skip: T1_SKIP }, async () => {
+test("the stall clears only after two clean probes in a row", async () => {
await withTmp(async (h) => {
await seedRelocated(h, "slow", { targetExists: true });
const probe = scripted(["stalled", "ok", "stalled", "ok", "ok"]);
diff --git a/common/controller/storageWatch.ts b/common/controller/storageWatch.ts
@@ -211,7 +211,12 @@ export async function runStorageWatchPass(
for (const { slug, config } of configs) {
const wasAutoPaused = Boolean(model.channels[slug]?.autoPaused);
- const dataDir = config.dataDir?.trim();
+ // ONE TIER PER CHANNEL (release 17): the drive its MEDIA is on —
+ // `mediaDir`, or on a channel not yet migrated off the retired
+ // whole-directory layout its `dataDir`. The pause stays the channel's (one
+ // `autoPaused` record), not a per-lane one; its text never leaves the
+ // corpus disk, so there is no second drive to watch.
+ const dataDir = config.mediaDir?.trim() || config.dataDir?.trim();
if (!dataDir) {
// In place. It cannot be on a drive that went away — but it CAN carry a
// record from before it was moved back, and that record has to come off
diff --git a/common/jobs/jobKinds.ts b/common/jobs/jobKinds.ts
@@ -475,7 +475,7 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
// THE SAME LINKS, WITHOUT THE BYTES. A re-point rewrites every channel
// symlink on one storage location plus the location's root, for the case the
// relocation above cannot help with: the media never moved, the DISK did, and
- // every `config.dataDir` on it now names a mountpoint that is not there.
+ // every `config.mediaDir` on it now names a mountpoint that is not there.
//
// `needsMedia: false` for the relocation's reason, and more sharply: every
// channel this job touches is BY DEFINITION unreachable when it starts —
diff --git a/common/lib/channelMedia.ts b/common/lib/channelMedia.ts
@@ -160,9 +160,10 @@ export class ChannelTextUnreadableError extends Error {
}
}
-// THE RETIRED layout's target shape, `<root>/<slug>/data`. Still exported for
-// the mover, the re-point and the rename until release 17 slice T2 rebases them
-// on `relocatedMediaDir` (lib/mediaTier-server.ts).
+// THE RETIRED layout's target shape, `<root>/<slug>/data`. Nothing moves a
+// channel to it any more (the mover, the re-point and the rename use
+// `relocatedMediaDir`, lib/mediaTier-server.ts); it names what a `legacy`
+// channel's tree is, for the tier migration and the tests that build one.
export function relocatedDataDir(root: string, slug: string): string {
return path.join(root.trim(), slug, "data");
}
diff --git a/common/lib/mediaTier-server.ts b/common/lib/mediaTier-server.ts
@@ -63,8 +63,8 @@ export function channelMediaLink(
// A relocated channel's media root: `<root>/<slug>/media`. The suffix is fixed,
// not configurable, so an empty mountpoint can never be mistaken for the media
-// and the movers can recognise a target by its shape (the same reason
-// `relocatedDataDir` fixed `<slug>/data`).
+// and the movers can recognise a target by its shape (the same reason the
+// retired `relocatedDataDir` fixed `<slug>/data`).
export function relocatedMediaDir(root: string, slug: string): string {
return path.join(root.trim(), slug, MEDIA_LINK_NAME);
}
diff --git a/common/lib/savedVideoStore.ts b/common/lib/savedVideoStore.ts
@@ -44,8 +44,8 @@ export function savedVideosMarkerPath(
}
// The store's home on a location: `<root>/saved-videos`. Flat, beside the
-// channels' `<slug>/data` dirs, and not configurable for the same reason
-// `relocatedDataDir` is not — a mover recognises a target by its shape.
+// channels' `<slug>/media` dirs, and not configurable for the same reason
+// `relocatedMediaDir` is not — a mover recognises a target by its shape.
export function relocatedSavedVideosDir(root: string): string {
return path.join(root.trim(), SAVED_VIDEOS_DIRNAME);
}
diff --git a/common/lib/storageLocations.ts b/common/lib/storageLocations.ts
@@ -14,7 +14,9 @@ import type { StorageHealthSettings } from "./storageHealthTimings";
// be reachable from a client component at all (`next build` enforces that).
//
// The entity is stored in `settings.storage`. A channel is NOT tagged with its
-// location: it is on location L iff its `config.dataDir` is under `L.root`.
+// location: it is on location L iff its `config.mediaDir` is under `L.root`
+// (release 17: the media tier is what a location holds; a `legacy` channel is
+// placed by its retired `config.dataDir` until it is migrated).
// That is a derivation, not a field, which is why a re-point only has to
// rewrite the location's root and each channel's symlink — there is no second
// copy of the association to keep in step, and `ChannelConfig`'s whitelisted
@@ -24,8 +26,8 @@ import type { StorageHealthSettings } from "./storageHealthTimings";
// rendered and NEVER stored in `settings.storage.locations`. A stored entry
// under this id would be deletable, and worse, `locationOfDataDir` would then
// match every unrelocated channel — breaking the one rule the whole design
-// rests on (a channel is on location L iff its `dataDir` is under `L.root`,
-// and an in-place channel has no `dataDir`).
+// rests on (a channel is on location L iff its `mediaDir` is under `L.root`,
+// and an in-place channel has no `mediaDir`).
//
// It is HERE rather than beside the row that uses it because `lib/settings.ts`
// has to refuse it as a stored id, and lib may not import controller.
@@ -108,7 +110,7 @@ export const STORAGE_SETTINGS_FIELD_DOCS: FieldDocs<StorageSettings> = {
"WHERE THE SAVED-VIDEO STORE IS, by location id. \"\" = in place, under " +
"the corpus at `paths.savedVideosDir`.\n\n" +
"A RECORD OF WHAT IS ON DISK, never an intention — the same contract as" +
- " a channel's `config.dataDir`. It is written by the move, on success, " +
+ " a channel's `config.mediaDir`. It is written by the move, on success, " +
"after the copy has verified and the symlink is in place; nothing else " +
"writes it, and a reader that disagrees with the disk trusts the disk. " +
"Optional so an older settings.json parses (and an older binary that " +
@@ -135,8 +137,10 @@ function normalizeRoot(root: string): string {
return stripped === "" ? "/" : stripped;
}
-// Which location a channel's `dataDir` sits on, or null when it sits on none
-// (the ordinary case: an unrelocated channel's data is inside the corpus).
+// Which location a path sits on — a channel's `mediaDir` (or a legacy one's
+// retired `dataDir`) — or null when it sits on none (the ordinary case: an
+// unrelocated channel's media is inside the corpus). The name is historical:
+// it is a pure prefix test, and the movers ask it of any target.
//
// NESTED ROOTS ARE ALLOWED and the LONGEST match wins. "/mnt/platter" and
// "/mnt/platter/archive" can both be locations; a channel under the latter is
@@ -144,8 +148,8 @@ function normalizeRoot(root: string): string {
// first. Two locations sharing one root is a misconfiguration the sanitizer
// does not forbid; the first in the list wins it.
//
-// "Under" is strict: `dataDir === root` is not a match. A channel's dataDir is
-// always `<root>/<slug>/data`, so equality only ever means a misconfiguration.
+// "Under" is strict: `dir === root` is not a match. A channel's mediaDir is
+// always `<root>/<slug>/media`, so equality only ever means a misconfiguration.
export function locationOfDataDir(
dataDir: string,
locations: StorageLocation[],
diff --git a/common/views/channelRow.test.ts b/common/views/channelRow.test.ts
@@ -144,7 +144,7 @@ test("media: in-place is dropped; the label is the volume's unless overridden",
assert.equal(overridden.media?.locationLabel, undefined);
});
-test("channelVolumeOf: no dataDir is internal, a named root wins, else unnamed", () => {
+test("channelVolumeOf: no mediaDir is internal, a named root wins, else unnamed", () => {
const locations = [
{ id: "big", label: "Big disk", root: "/mnt/big" },
{ id: "bigger", label: "", root: "/mnt/big/inner" },
@@ -153,20 +153,25 @@ test("channelVolumeOf: no dataDir is internal, a named root wins, else unnamed",
id: "internal",
label: "Internal",
});
- assert.deepEqual(channelVolumeOf(" ", locations), {
+ assert.deepEqual(channelVolumeOf({ mediaDir: " " }, locations), {
id: "internal",
label: "Internal",
});
- assert.deepEqual(channelVolumeOf("/mnt/big/alpha/data", locations), {
+ assert.deepEqual(channelVolumeOf({ mediaDir: "/mnt/big/alpha/media" }, locations), {
id: "big",
label: "Big disk",
});
- assert.deepEqual(channelVolumeOf("/mnt/big/inner/a/data", locations), {
+ assert.deepEqual(channelVolumeOf({ mediaDir: "/mnt/big/inner/a/media" }, locations), {
id: "bigger",
label: "bigger",
});
- assert.deepEqual(channelVolumeOf("/elsewhere/a/data", locations), {
+ assert.deepEqual(channelVolumeOf({ mediaDir: "/elsewhere/a/media" }, locations), {
id: "",
label: "Elsewhere",
});
+ // A legacy channel (release 17) is placed by its retired `dataDir`.
+ assert.deepEqual(channelVolumeOf({ dataDir: "/mnt/big/old/data" }, locations), {
+ id: "big",
+ label: "Big disk",
+ });
});
diff --git a/common/views/channelRow.ts b/common/views/channelRow.ts
@@ -112,13 +112,15 @@ export type ChannelRowView = {
// the lanes read the channel. The rack draws it beside the tier, because it
// is the reason the lanes are skipping the row (release 16 slice RM).
mediaHold: string | null;
- // WHICH VOLUME, as an id a filter can name: a location id, "internal" for the
- // corpus volume, or "" for a dataDir under a root nobody named.
+ // WHICH VOLUME THE MEDIA IS ON, as an id a filter can name: a location id,
+ // "internal" for the corpus volume, or "" for a mediaDir under a root nobody
+ // named. (The text is on the corpus volume whatever this says.)
volumeId: string;
volumeLabel: string;
- // `snapshot.totalMediaBytes`. NULL, not 0, for a report written before the
- // field existed: a 400 GB channel that has not been measured must not sort as
- // the smallest thing on the disk.
+ // `snapshot.totalMediaBytes` — the MEDIA TIER's bytes since release 17 (the
+ // audio and the raw live chat: what a move carries). NULL, not 0, for a
+ // report written before the field existed: a 400 GB channel that has not
+ // been measured must not sort as the smallest thing on the disk.
mediaBytes: number | null;
};
@@ -160,17 +162,18 @@ export type ChannelVolumeId = { id: string; label: string };
// WHICH VOLUME A CHANNEL'S MEDIA IS ON, as a filterable id.
//
-// Same derivation the badge uses (`config.dataDir` under a location's root,
-// longest match wins) with one addition: no `dataDir` at all means the corpus
-// volume, which is the row the operator is trying to empty and therefore the
-// one they most need to filter to. A `dataDir` under a root NOBODY named is
-// neither — it gets "" and falls out of every volume filter, which is the
-// honest answer and the nudge to name that root on /storage.
+// Same derivation the badge uses (`config.mediaDir` under a location's root,
+// longest match wins; a `legacy` channel by its retired `dataDir`) with one
+// addition: no target at all means the corpus volume, which is the row the
+// operator is trying to empty and therefore the one they most need to filter
+// to. A target under a root NOBODY named is neither — it gets "" and falls out
+// of every volume filter, which is the honest answer and the nudge to name
+// that root on /storage.
export function channelVolumeOf(
- dataDir: string | undefined,
+ config: Pick<ChannelConfig, "mediaDir" | "dataDir"> | undefined,
locations: StorageLocation[],
): ChannelVolumeId {
- const trimmed = dataDir?.trim();
+ const trimmed = config?.mediaDir?.trim() || config?.dataDir?.trim();
if (!trimmed) {
return { id: INTERNAL_LOCATION_ID, label: "Internal" };
}
diff --git a/common/views/storage.test.ts b/common/views/storage.test.ts
@@ -31,9 +31,12 @@ function rollup(partial: Partial<LocationRollup>): LocationRollup {
bytes: partial.bytes ?? 0,
unknownBytes: partial.unknownBytes ?? 0,
clipsBytes: partial.clipsBytes ?? 0,
+ textBytes: partial.textBytes ?? 0,
+ unknownTextBytes: partial.unknownTextBytes ?? 0,
ok: partial.ok ?? 0,
unreachable: partial.unreachable ?? 0,
moving: partial.moving ?? 0,
+ legacy: partial.legacy ?? 0,
};
}
@@ -375,29 +378,101 @@ test("the clip-window line is rendered only when there are windows", () => {
assert.match(storageClipsText(5 * 1024 ** 3), /5\.00 GB of it is fetched/);
});
-test("a location row carries the clips share of its bytes", () => {
+// RELEASE 17: a location holds the media tier of the channels on it, and
+// nothing else — their text and clip windows never leave the corpus volume, so
+// they are counted on the internal row, whose breakdown says so.
+test("a location row carries the media tier only; the internal row holds every channel's text and clips", () => {
const payload = buildStorageRows({
locations: [loc("cold", "/mnt/cold")],
+ internal: {
+ root: "/corpus/channels",
+ corpus: { textBytes: 3 * GB, clipsBytes: 2 * GB, unknown: 0 },
+ },
defaultLocationId: "",
probes: {},
rollups: {
cold: rollup({
locationId: "cold",
total: 2,
- bytes: 10_000_000_000,
- clipsBytes: 2_000_000_000,
+ bytes: 10 * GB,
+ clipsBytes: 1 * GB,
+ textBytes: 1 * GB,
ok: 2,
}),
+ internal: rollup({ locationId: "internal", total: 1, ok: 1, bytes: 5 * GB }),
},
registry: NO_JOBS,
udisksctlAvailable: false,
now: 0,
});
- const row = payload.rows.find((r) => r.id === "cold");
- assert.equal(row?.clipsBytes, 2_000_000_000);
- assert.match(String(row?.clipsText), /fetched clip windows/);
- // A SUBSET, not a sibling: the clips are already inside `bytes`.
- assert.equal(row?.bytes, 10_000_000_000);
+ const cold = payload.rows.find((r) => r.id === "cold");
+ assert.equal(cold?.bytes, 10 * GB);
+ assert.equal(cold?.clipsBytes, 0);
+ assert.equal(cold?.clipsText, "");
+ assert.equal(cold?.tiersText, "");
+ const internal = payload.rows.find((r) => r.id === "internal");
+ // 5 GB of in-place media + 3 GB of text + 2 GB of clips.
+ assert.equal(internal?.bytes, 10 * GB);
+ assert.equal(internal?.clipsBytes, 2 * GB);
+ assert.match(String(internal?.clipsText), /2\.00 GB of it is fetched clip windows/);
+ assert.equal(
+ internal?.tiersText,
+ "text 3.00 GB + clips 2.00 GB on the corpus volume, plus 5.00 GB media of in-place channels",
+ );
+ assert.equal(payload.bytesInPlace, 10 * GB);
+});
+
+test("an old report's missing text figure is unknown on the internal row, never 0", () => {
+ const { rows } = buildStorageRows({
+ locations: [],
+ internal: {
+ root: "/corpus/channels",
+ corpus: { textBytes: 1 * GB, clipsBytes: 0, unknown: 2 },
+ },
+ defaultLocationId: "",
+ probes: {},
+ rollups: {
+ internal: rollup({ locationId: "internal", total: 3, ok: 3, bytes: 4 * GB, unknownBytes: 1 }),
+ },
+ registry: NO_JOBS,
+ now: 0,
+ });
+ assert.equal(rows[0].bytes, 5 * GB);
+ assert.equal(rows[0].unknownBytes, 3);
+ assert.equal(rows[0].bytesText, "5.00 GB + 3 unmeasured");
+});
+
+test("the corpus row names legacy channels to migrate, never as unmeasured", () => {
+ const { rows } = buildStorageRows({
+ locations: [],
+ internal: {
+ root: "/corpus/channels",
+ corpus: { textBytes: 1 * GB, clipsBytes: 0, unknown: 0, legacy: 2 },
+ },
+ defaultLocationId: "",
+ probes: {},
+ rollups: { internal: rollup({ locationId: "internal", total: 1, ok: 1, bytes: 1 * GB }) },
+ registry: NO_JOBS,
+ now: 0,
+ });
+ assert.equal(rows[0].unknownBytes, 0);
+ assert.equal(rows[0].bytesText, "2.00 GB");
+ assert.match(rows[0].tiersText, /legacy \(2 to migrate\)/);
+});
+
+test("a legacy channel counts as unreachable, with how many are to migrate", () => {
+ const { rows } = buildStorageRows({
+ locations: [loc("cold", "/mnt/cold")],
+ defaultLocationId: "",
+ probes: {},
+ rollups: {
+ cold: rollup({ locationId: "cold", total: 3, ok: 1, unreachable: 2, legacy: 2 }),
+ },
+ registry: NO_JOBS,
+ now: 0,
+ });
+ assert.equal(rows[0].channels.legacy, 2);
+ assert.equal(rows[0].channelsText, "1 ok / 2 unreachable (2 to migrate) / 0 moving");
});
test("a location whose drive is not answering reads so, and says since when", () => {
diff --git a/common/views/storage.ts b/common/views/storage.ts
@@ -50,6 +50,9 @@ export type StorageChannelCounts = {
unreachable: number;
moving: number;
total: number;
+ // The share of `unreachable` that is the retired whole-directory layout,
+ // waiting for `archilyzer storage migrate-tier` (release 17).
+ legacy: number;
};
export type StorageRow = {
@@ -80,23 +83,29 @@ export type StorageRow = {
// Built here rather than in the page so the param name has ONE spelling
// across the two surfaces that use it.
channelsHref: string;
- // Media bytes on this location, summed from each channel's last report, and
- // how many channels on it could not contribute a figure.
+ // What this row's volume holds, summed from each channel's last report, and
+ // how many channels could not contribute a figure. A LOCATION holds the
+ // media tier of the channels on it (release 17: their text never leaves the
+ // corpus volume). The INTERNAL row holds every channel's text and clip
+ // windows, plus the media of the channels in place.
bytes: number;
unknownBytes: number;
// "1.42 TB (2 channels unmeasured)" / "size unknown until Refresh report".
// ONE wording, and never a bare "0 B" for a location whose channels have
// simply never had a report — that reads as an empty drive.
bytesText: string;
- // The `clips/` share of `bytes` — fetched clip windows, the one part of a
- // location's media that is a cache nothing prunes. 0 is a real answer here
- // (no windows have been fetched onto this drive), unlike `bytes`, which is
- // why it needs no "unknown" companion: the channels that could not measure
- // clips are exactly the ones `unknownBytes` already counts.
+ // The `clips/` share of `bytes` — fetched clip windows, a cache nothing
+ // prunes. On the INTERNAL row only since release 17: `clips/` is never
+ // tiered, so a location holds none (0 there). 0 is a real answer, unlike
+ // `bytes`, which is why it needs no "unknown" companion.
clipsBytes: number;
- // "· 4.21 GB in clip windows", or "" when there are none. Built here so the
- // row and any future poll cannot word it differently.
+ // "4.21 GB of it is fetched clip windows", or "" when there are none. Built
+ // here so the row and any future poll cannot word it differently.
clipsText: string;
+ // THE INTERNAL ROW'S BREAKDOWN: "text 12.1 GB + clips 4.2 GB on the corpus
+ // volume, plus 310 GB media of in-place channels". "" on a location row,
+ // whose figure is the media tier alone.
+ tiersText: string;
// Absent when the location is not available, and ALSO when it is available
// but unmeasurable (getFreeBytes fails open to Infinity, which the probe
// drops rather than carry). Render "—" for both.
@@ -197,7 +206,25 @@ export type StorageRowsInputs = {
// The corpus volume as a row. Absent → no internal row (a caller that only
// wants the configured locations). `freeBytes` is a statfs of the root, taken
// by the shell, because nothing in views/ may touch a disk.
- internal?: { root: string; freeBytes?: number };
+ //
+ // `corpus` (release 17): what EVERY channel keeps on the corpus volume
+ // whatever its media's location — the text tier and the clip windows,
+ // summed off the reports — and how many channels could not contribute (a
+ // report written before release 17 has no text figure, beyond the in-place
+ // channels the internal rollup already counts unmeasured). Absent → only the
+ // in-place channels' media, as before.
+ internal?: {
+ root: string;
+ freeBytes?: number;
+ // `legacy`: channels still on the retired whole-directory layout, whose
+ // text is NOT on this volume — named "(n to migrate)", never unmeasured.
+ corpus?: {
+ textBytes: number;
+ clipsBytes: number;
+ unknown: number;
+ legacy?: number;
+ };
+ };
defaultLocationId: string;
// By location id. A location with no entry has never been probed in this
// process — treated as `missing` with an unknown identity rather than
@@ -255,9 +282,36 @@ function countsOf(rollup: LocationRollup | undefined): StorageChannelCounts {
unreachable: rollup?.unreachable ?? 0,
moving: rollup?.moving ?? 0,
total: rollup?.total ?? 0,
+ legacy: rollup?.legacy ?? 0,
};
}
+// `n ok / n unreachable / n moving` — with "(n to migrate)" after the
+// unreachable count when any of it is the retired layout. ONE wording, for the
+// rows and any future poll.
+export function storageChannelsText(c: StorageChannelCounts): string {
+ const migrate = c.legacy > 0 ? ` (${c.legacy} to migrate)` : "";
+ return `${c.ok} ok / ${c.unreachable} unreachable${migrate} / ${c.moving} moving`;
+}
+
+// THE INTERNAL ROW'S BREAKDOWN (release 17): what every channel keeps on the
+// corpus volume, plus the in-place channels' media.
+export function storageTiersText(
+ textBytes: number,
+ clipsBytes: number,
+ inPlaceMediaBytes: number,
+ legacy = 0,
+): string {
+ return (
+ `text ${bytesLabel(textBytes)} + clips ${bytesLabel(clipsBytes)} on the ` +
+ `corpus volume, plus ${bytesLabel(inPlaceMediaBytes)} media of in-place ` +
+ `channels` +
+ (legacy > 0
+ ? `; legacy (${legacy} to migrate) — their text is still on their media drive`
+ : "")
+ );
+}
+
// Bytes → GB with two decimals, or TB past a terabyte. Local rather than
// `lib/format`'s formatBytes because this module is reachable from the client
// bundle and is deliberately import-free.
@@ -308,7 +362,7 @@ function runningRepoint(registry: RegistryReader): string | null {
// the registry caps at concurrency 1 — so either one running is a fact about
// the machine and freezes every row on this page, not just its own. (A
// CHANNEL move is on that key too and deliberately does NOT freeze this page:
- // it touches one channel's `data/`, never a location's root or the store, and
+ // it touches one channel's `media/`, never a location's root or the store, and
// /storage has been usable during one since locations shipped.)
const job = registry
.list()
@@ -369,9 +423,10 @@ export function buildStorageRows(i: StorageRowsInputs): StorageRowsPayload {
];
const roll = i.rollups[loc.id];
+ // The media tier only: the channels' text and clip windows are on the
+ // corpus volume, counted on the internal row.
const bytes = roll?.bytes ?? 0;
const unknownBytes = roll?.unknownBytes ?? 0;
- const clipsBytes = roll?.clipsBytes ?? 0;
bytesOnLocation[loc.id] = bytes;
unknownBytesOnLocation[loc.id] = unknownBytes;
return {
@@ -385,13 +440,14 @@ export function buildStorageRows(i: StorageRowsInputs): StorageRowsPayload {
statusLabel: STORAGE_STATUS_LABEL[status],
identity: probe ? identityLine(probe.identity) : null,
channels: counts,
- channelsText: `${counts.ok} ok / ${counts.unreachable} unreachable / ${counts.moving} moving`,
+ channelsText: storageChannelsText(counts),
channelsHref: channelsHrefForLocation(loc.id),
bytes,
unknownBytes,
bytesText: storageBytesText(bytes, unknownBytes),
- clipsBytes,
- clipsText: storageClipsText(clipsBytes),
+ clipsBytes: 0,
+ clipsText: "",
+ tiersText: "",
...(probe?.freeBytes !== undefined ? { freeBytes: probe.freeBytes } : {}),
lastProbeAgeMs: probe ? Math.max(0, i.now - probe.probedAt) : 0,
...(i.notAnswering?.[loc.id]
@@ -479,9 +535,13 @@ function internalRow(
const internal = i.internal as NonNullable<StorageRowsInputs["internal"]>;
const roll = i.rollups[INTERNAL_ROW_ID];
const counts = countsOf(roll);
- const bytes = roll?.bytes ?? 0;
- const unknownBytes = roll?.unknownBytes ?? 0;
- const clipsBytes = roll?.clipsBytes ?? 0;
+ // The in-place channels' media, plus (release 17) every channel's text and
+ // clip windows, which stay on this volume wherever the media is.
+ const inPlaceMedia = roll?.bytes ?? 0;
+ const corpus = internal.corpus;
+ const clipsBytes = corpus ? corpus.clipsBytes : (roll?.clipsBytes ?? 0);
+ const bytes = inPlaceMedia + (corpus ? corpus.textBytes + corpus.clipsBytes : 0);
+ const unknownBytes = (roll?.unknownBytes ?? 0) + (corpus?.unknown ?? 0);
bytesOnLocation[INTERNAL_ROW_ID] = bytes;
unknownBytesOnLocation[INTERNAL_ROW_ID] = unknownBytes;
const withheld =
@@ -499,13 +559,21 @@ function internalRow(
statusLabel: STORAGE_STATUS_LABEL.available,
identity: null,
channels: counts,
- channelsText: `${counts.ok} ok / ${counts.unreachable} unreachable / ${counts.moving} moving`,
+ channelsText: storageChannelsText(counts),
channelsHref: channelsHrefForLocation(INTERNAL_ROW_ID),
bytes,
unknownBytes,
bytesText: storageBytesText(bytes, unknownBytes),
clipsBytes,
clipsText: storageClipsText(clipsBytes),
+ tiersText: corpus
+ ? storageTiersText(
+ corpus.textBytes,
+ corpus.clipsBytes,
+ inPlaceMedia,
+ corpus.legacy ?? 0,
+ )
+ : "",
...(internal.freeBytes !== undefined ? { freeBytes: internal.freeBytes } : {}),
lastProbeAgeMs: 0,
busy: null,
@@ -609,7 +677,7 @@ function mountAction(
// DELETE IS REFUSED WHILE ANYBODY LIVES THERE. Deleting the location would not
// touch a byte — but it would erase the only record of which disk those
-// channels' absolute `dataDir`s belong to, which is precisely the knowledge
+// channels' absolute `mediaDir`s belong to, which is precisely the knowledge
// this page exists to keep. Move the channels off it (or re-point it) first.
function deleteAction(
loc: StorageLocation,
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -33,6 +33,8 @@
- **The operations pages share one count of the lanes' pending work.** Every open operations page asks for the lanes' status every 3 seconds, and each request used to count every lane's pending videos afresh from every channel's report. That count is now made once and handed to every request in the next 3 seconds. Changing a lane's rules, a focus or a channel's priority counts again at once; otherwise a pending count can be up to 3 seconds behind a report that was just rewritten or a video a lane just picked. A lane's hold, its runner and its picks are still read fresh on every request.
- **Jobs a stopped editor left "running" are closed when it starts again.** A job that was still running when the editor's process ended (killed, crashed, or shut down before the job had finished unwinding) kept "running" in its record for good, and `/jobs` listed it as archived. On start the editor now marks each one **cancelled**, with "interrupted: the process running it stopped before it finished" as the reason on the job's page, and its end time is the last time its log was written. Nothing is run again; **Retry** works as for any cancelled job. A job that another live process is running, such as `archilyzer run`, is left alone, and the same check now keeps the start-up pass from closing that process's queued jobs. Such leftover jobs never blocked a media move.
- **A channel's text stays on the fast disk when its media moves, so a slow or unplugged media drive no longer holds its transcripts.** A channel's big files — the audio and the raw live-chat replay — can now live in the channel's own `media` folder, on this disk or another, while its transcripts, cues, metadata and every other small file stay in `data/` where they always were; each big file that moves leaves a small link behind, so everything that opens it by name still finds it. New downloads, transcodes and live-chat normalizes put their big files there as they finish, and every cleanup that deletes audio removes the file the link points to, not just the link. What that changes when a media drive is stalled, unplugged or mid-move: **the index and stats builds never wait on it or are held by it** (a live chat whose transcript cues are out of date keeps the cues the last build read until the drive answers), the channel's report still refreshes (its media size reads as unknown until the drive answers), **digests keep running — even during a move of that channel's media** — and so do normalize, the availability checks, the metadata scan and clip eviction. Transcription, downloads, the backfill lane and anything else that opens the audio are held as before. **A channel moved the old way — its whole `data/` on the other drive — is now shown as "Media layout retired" and held by everything, the builds and digests included, until `archilyzer storage migrate-tier <channel>` brings its text home;** every refusal says so. Deleting a video from its page is refused while its channel's media drive is not reachable, so its audio is never left behind on the drive. Needs a rebuild and restart of the editor.
+- **Moving a channel's media now moves only its big files.** The Storage panel's **Move media** copies the channel's audio and raw live-chat replays to `<root>/<channel>/media` on the destination and leaves its transcripts, metadata and every other small file in `data/` on this disk; a channel whose audio is still in `data/` has it put into the channel's `media` folder on this disk first, and the preview says how many files ("2 file(s) tiered first"). **Move back in place** brings the `media` folder home; the links in `data/` are not touched either way. The panel shows the media path and the text path, each with its size. While a channel's media is held — moving, or on a drive that is unplugged or not answering — its text stays readable: the video page and the videos list open as usual and list the audio with its size unknown, the transcript opens, the audio answers "not reachable, try again" (503) instead of "not found", and a digest may run, during a move too; only the jobs and lanes that open the audio wait. A move, its preview and **Resume move** refuse a channel still on the retired whole-directory layout and name `archilyzer storage migrate-tier`; `/storage` counts such channels as unreachable "(n to migrate)". On `/storage` a location's figure is now the media on it, and the corpus volume's row adds every channel's text and clip windows ("text … + clips … on the corpus volume, plus … media of in-place channels"). Re-pointing a location, renaming a channel and deleting one follow the `media` folder; a re-point also carries a channel not yet migrated (its whole `data/` link), so a drive holding both kinds still follows its disk, while renaming such a channel is refused until it is migrated, and **Clear marker** never removes an interrupted migration's marker. Needs a rebuild and restart of the editor.
+- **An export build no longer reads a raw live-chat replay from a media drive that is unplugged, not answering or mid-move.** It publishes the live-chat cues already on disk for that channel, and its log says how many are older than their replay and how many videos with a replay and no cues yet were left out of that build.
- **A video whose YouTube subtitles answer "Too Many Requests" (HTTP 429) is downloaded anyway, and YouTube is not put in a cooldown for it.** YouTube refuses a subtitle file per video while the video itself downloads fine; every one of the day's 429s on 2026-10-01 was a subtitle fetch, and each failed its download, put all of YouTube in a cooldown that reached 30 minutes, and deferred the video for 6 hours to fail the same way again. Now that refusal is noted and the download goes on to the audio, as for a video with no captions, so the transcription lane transcribes it; nothing platform-wide is backed off. The video's subtitles are deferred: **Download missing subs** skips them for 6 hours, and from the third time they are refused, for 7 days; a download from the video's own page still fetches them. The download lane's page lists them under **Deferred subtitles**, and the video page says how many times and when. **Download missing subs** also goes on to the next video when one video's subtitles are refused, instead of stopping. Any other subtitle failure (a 403, a missing file, a chat replay that fails) still fails the download, as before. Needs a rebuild and restart of the editor.
- **The download pace adapts to rate limits, a rate limit that outlasts the cooldown holds the platform, and the auto-download lane waits between downloads.** Every yt-dlp run against a platform now waits its platform's current pace between requests: 1 second for YouTube and Rumble, doubled by each real rate limit (up to 16 seconds) and eased back one step after every 5 clean downloads, and one step for every hour with no rate limit; a subtitle-only refusal never raises it. When a platform has failed three times in a row at the 30-minute cooldown, it is **held**: auto-download tries it once an hour instead of every 30 minutes, and until that try is due a manual **Sync**, download or metadata scan on it is refused with a sentence giving its time. A clean try lifts the hold — the lane's, or a manual Sync, download or scan once the try is due, which is how a hold ends while auto-download is off or has nothing to fetch on that platform — and so does a try whose video came down although its subtitles were refused. The lane page keeps a held platform listed until then (saying when the lane is off), with a **Clear hold** button that drops the hold, the cooldown and the raised pace at once and says so in a job log. The auto-download lane now waits **Sleep between downloads** between two downloads on one platform, as a channel's batch downloads always did, plus whatever the pace was raised by; batch downloads add that too. The lane page's **Rate-limit cooldown** box shows held platforms, the raised paces and the deferred subtitles, the lane says when it is idle because a platform is held or it is pausing between downloads, and `archilyzer doctor` warns about a platform in a cooldown or held, and about a raised pace. **Download missing subs** also waits that gap between videos. The four numbers are the new `pacing` block in `settings.json` (SETTINGS.md). **After the restart, YouTube may be held at its first real failure:** its cooldown count from before the update (the subtitle refusals) still stands, so one failure puts it straight past the cap — **Clear hold** on the download lane's page resets it, and a clean download does too. Needs a rebuild and restart of the editor.
diff --git a/editor/app/api/channels/[slug]/videos/[id]/files/[name]/route.ts b/editor/app/api/channels/[slug]/videos/[id]/files/[name]/route.ts
@@ -1,11 +1,15 @@
import path from "node:path";
import { createReadStream, type ReadStream } from "node:fs";
-import { stat } from "node:fs/promises";
+import { lstat, stat } from "node:fs/promises";
import { NextResponse } from "next/server";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import { makeSafeController } from "yt-dlp-transcript-common/lib/safeStreamController";
import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels";
-import { channelMediaStall } from "yt-dlp-transcript-common/lib/channelMedia";
+import {
+ channelMediaStall,
+ channelTextStall,
+} from "yt-dlp-transcript-common/lib/channelMedia";
+import { isTierable } from "yt-dlp-transcript-common/lib/mediaTier";
import {
NOT_ANSWERING,
isDriveNotAnswering,
@@ -139,15 +143,52 @@ export async function GET(
{ error: `Media not read: ${NOT_ANSWERING}.` },
{ status: 503, headers: { "retry-after": "15" } },
);
+ // THE DRIVE IS CHOSEN BY WHERE THE FILE IS (release 17). A TIERED file — a
+ // link (the `lstat` says so, on the corpus disk) with a tierable name — is on
+ // the media tier's drive, `mediaDir`, so its stat goes through that drive's
+ // watchdog and a stalled media drive refuses it. Everything else — the text,
+ // and a real file the tier never takes (`source-media.*`, a partial) — is on
+ // the corpus disk and is read directly: a stalled media drive never holds a
+ // transcript. (A legacy channel's whole `data/` is on its retired `dataDir`.)
const channelConfig = await readChannelConfig(paths, slug);
- if (channelMediaStall(channelConfig)) return notAnswering();
- const drive = channelConfig?.dataDir?.trim();
+ // THE TEXT STALL FIRST, with no I/O: on a legacy channel `data/` itself is a
+ // link onto the far drive, so even the `lstat` below would reach a drive
+ // already known not to answer (review R1).
+ if (channelTextStall(channelConfig)) return notAnswering();
+ const mediaDir = channelConfig?.mediaDir?.trim() || undefined;
+ const legacyDrive = channelConfig?.dataDir?.trim() || undefined;
+ // A legacy channel has no tiered links: its whole `data/` is on the retired
+ // drive, read through the watchdog below — no `lstat` of it here.
+ const isLink = legacyDrive
+ ? false
+ : await lstat(fullPath)
+ .then((l) => l.isSymbolicLink())
+ .catch(() => false);
+ const onMediaTier = isLink && isTierable(name);
+ if (onMediaTier && channelMediaStall(channelConfig)) return notAnswering();
+ const drive = onMediaTier ? mediaDir || legacyDrive : legacyDrive;
let stats;
try {
stats = await (drive ? onDrive(drive, () => stat(fullPath)) : stat(fullPath));
} catch (err) {
if (isDriveNotAnswering(err)) return notAnswering();
+ // A LINK WHOSE TARGET IS NOT THERE, on a channel whose media is on
+ // another drive, is a tiered file on a drive that is unmounted (or renamed
+ // away): the file exists, its bytes are not reachable now. 503 with a
+ // retry, never 404 — a 404 tells a player, and an operator, that there is
+ // no such file. On an in-place `media/` (no `mediaDir`) a dangling link is
+ // a file that is gone, and that IS a 404 (review N8).
+ if (onMediaTier && mediaDir) {
+ return NextResponse.json(
+ {
+ error:
+ `Media not read: ${name} is on this channel's media drive, which ` +
+ `is not mounted or not answering.`,
+ },
+ { status: 503, headers: { "retry-after": "15" } },
+ );
+ }
return NextResponse.json({ error: "Not found" }, { status: 404 });
}
if (!stats.isFile()) {
diff --git a/editor/app/channels/[slug]/components/stages/StorageStage.tsx b/editor/app/channels/[slug]/components/stages/StorageStage.tsx
@@ -84,10 +84,14 @@ type Props = {
// the badge. Undefined when it is on none — in place, or on a root nobody
// named. See common/lib/storageLocations.ts.
locationLabel?: string;
- // Audio bytes this channel holds, from the loaded snapshot — NOT a walk. Null
- // when the snapshot predates the field (or there is no snapshot), and rendered
- // as "—" rather than "0": a zero here would claim a measurement nobody took.
+ // THE TWO TIERS (release 17), from the loaded snapshot — NOT a walk. The
+ // MEDIA tier (`totalMediaBytes`: the audio and the raw live chat, what a move
+ // carries) and the TEXT tier (`totalTextBytes`: everything under `data/` on
+ // the corpus disk but the tier and the clips). Null when the snapshot
+ // predates the field (or there is no snapshot), and rendered as "—" rather
+ // than "0": a zero here would claim a measurement nobody took.
mediaBytes: number | null;
+ textBytes: number | null;
// Bytes this channel's `data/<id>/clips/` windows occupy, from the same
// snapshot. Null when the snapshot predates the field, for the same reason
// mediaBytes is: "0 B of clips" is a measurement nobody took.
@@ -124,6 +128,7 @@ export function StorageStage({
location,
locationLabel,
mediaBytes,
+ textBytes,
clipsBytes,
freeBytes,
volumeDir,
@@ -140,22 +145,31 @@ export function StorageStage({
<h3 className="text-base font-semibold">Location</h3>
<MediaLocationBadge media={location} locationLabel={locationLabel} />
</div>
+ {/* TWO ROWS UNDER ONE HEADING (release 17): the MEDIA tier — the big
+ files, `channels/<slug>/media`, which is what moves — and the TEXT
+ tier, `data/`, which never leaves the corpus disk. */}
<dl className="grid grid-cols-[max-content_1fr] gap-x-4 gap-y-1 text-sm">
<dt className="text-muted-foreground">Media path</dt>
<dd className="font-mono text-xs break-all" aria-label="media path">
{location.relocated && location.target
? location.target
- : location.dataDir}
+ : `${location.mediaLink} (in place)`}
</dd>
- <dt className="text-muted-foreground">Read as</dt>
- <dd className="font-mono text-xs break-all">
- {location.dataDir}
- {location.relocated ? " (symlink)" : ""}
- </dd>
- <dt className="text-muted-foreground">Audio on disk</dt>
+ <dt className="text-muted-foreground">Media on disk</dt>
<dd aria-label="media bytes">
{mediaBytes === null ? "—" : formatBytes(mediaBytes)}
</dd>
+ <dt className="text-muted-foreground">Text path</dt>
+ <dd className="font-mono text-xs break-all" aria-label="text path">
+ {location.dataDir}
+ {location.status === "legacy"
+ ? " (a symlink — the retired whole-directory layout)"
+ : " (in place)"}
+ </dd>
+ <dt className="text-muted-foreground">Text on disk</dt>
+ <dd aria-label="text bytes">
+ {textBytes === null ? "—" : formatBytes(textBytes)}
+ </dd>
<dt className="text-muted-foreground">Free on that volume</dt>
<dd aria-label="free on media volume">
{freeBytes === null ? "—" : formatBytes(freeBytes)}{" "}
@@ -183,11 +197,14 @@ export function StorageStage({
{location.status === "in-transition"
? " — when the move completes, or its marker is cleared below."
: "."}
+ {location.text.readable
+ ? " Its text stays readable: the video pages, the index and the digests go on."
+ : " Its text is held too until the channel is migrated."}
</p>
)}
{mediaBytes === null && (
<p className="text-xs text-muted-foreground">
- No audio total in this channel’s report yet — refresh the
+ No media-tier total in this channel’s report yet — refresh the
report for a figure. The preview below measures the real tree
regardless, and it is the number the move acts on.
</p>
@@ -238,13 +255,17 @@ export function StorageStage({
// is swept too. The controller refuses it; this is the same refusal with
// a reason, one click earlier.
unvouched={
- location.status === "inconsistent" ||
- location.status === "unreachable" ||
- location.status === "stalled"
- ? `This channel's media location is ${location.status}: ${
- location.detail ?? "disk and config do not agree"
- } Moving back would delete the relocated copy, so it is refused until the location reads "relocated · reachable".`
- : null
+ location.status === "legacy"
+ ? `This channel's media cannot be moved: ${
+ location.detail ?? "its media layout is the retired whole-directory one"
+ }.`
+ : location.status === "inconsistent" ||
+ location.status === "unreachable" ||
+ location.status === "stalled"
+ ? `This channel's media location is ${location.status}: ${
+ location.detail ?? "disk and config do not agree"
+ } Moving back would delete the relocated copy, so it is refused until the location reads "relocated · reachable".`
+ : null
}
/>
@@ -343,7 +364,7 @@ function MoveMedia({
// NO PREVIEW GATE ON THE WAY BACK, and it is not an oversight: the preview
// answers "is there room on the destination", and the controller's move-back
// asks the corpus volume that itself, charging only the bytes still missing
- // from a resumed `data.incoming`. There is no root for the operator to get
+ // from a resumed `media.incoming`. There is no root for the operator to get
// wrong either — it is the channel's own directory.
const confirmed = movingBack || (named && key === previewedKey);
const disabled = movingBack
@@ -373,15 +394,19 @@ function MoveMedia({
{movingBack ? (
<>
Copies <code>{target ?? "the target"}</code> back into the channel
- dir, verifies it, replaces the symlink with a real directory and
- clears the recorded location.
+ dir, verifies it, replaces the <code>media</code> symlink with a
+ real directory and clears the recorded location. The text never
+ moved.
</>
) : (
<>
- Copies <code>data/</code> to{" "}
- <code><root>/{slug}/data</code>, verifies it, and leaves a
- symlink behind so every reader, yt-dlp and the index keep working
- unchanged. The source is not touched until the copy verifies.
+ Copies the media tier — <code>media/</code>, the audio and the raw
+ live chat — to <code><root>/{slug}/media</code>, verifies
+ it, and leaves a symlink behind so every reader, yt-dlp and the
+ index keep working unchanged. A channel whose big files are still
+ in <code>data/</code> is tiered into <code>media/</code> first, on
+ this disk. The text stays here, and the source is not touched
+ until the copy verifies.
</>
)}
</p>
@@ -477,7 +502,7 @@ function MoveMedia({
/>
<span className="text-xs text-muted-foreground">
An absolute directory that already exists. One root holds many
- channels; each gets its own <code><slug>/data</code> under it.
+ channels; each gets its own <code><slug>/media</code> under it.
A root you expect to use again belongs on{" "}
<Link href="/storage" className="underline">
Storage
@@ -523,6 +548,13 @@ function MoveMedia({
<dd>{formatBytes(preview.freeOnSource)}</dd>
</dl>
)}
+ {preview && preview.tieredFirst > 0 && (
+ <p aria-label="tiered first" className="text-sm text-muted-foreground">
+ {preview.tieredFirst.toLocaleString()} file(s) tiered first — moved
+ into <code>media/</code> on this disk and linked from{" "}
+ <code>data/</code>, so the figures above are what the move carries.
+ </p>
+ )}
{preview && preview.freeOnRoot < preview.bytesToMove && (
<p role="alert" className="text-sm text-destructive">
The destination has less free space than the media needs. The job will
@@ -625,8 +657,8 @@ function StaleMarker({
{marker
? ` at phase "${marker.phase}" (${marker.direction === "out" ? "to" : "from"} ${marker.target})`
: ""}
- . While the marker stands this channel is skipped by every lane
- and its media jobs are refused.
+ . While the marker stands this channel is skipped by the media
+ lanes and its media jobs are refused; its text stays readable.
</p>
<p className="text-sm text-muted-foreground">
<strong>Resume move</strong> runs the same move again from where
diff --git a/editor/app/channels/[slug]/lib/videoDirFiles.ts b/editor/app/channels/[slug]/lib/videoDirFiles.ts
@@ -0,0 +1,81 @@
+import path from "node:path";
+import { lstat, readdir, stat } from "node:fs/promises";
+import type { Dirent } from "node:fs";
+import { isTierable } from "yt-dlp-transcript-common/lib/mediaTier";
+import { onDrive } from "yt-dlp-transcript-common/lib/storageHealth";
+import type { VideoFile } from "../videos/[id]/components/cards/videoFiles";
+
+// ONE VIDEO DIRECTORY'S FILES, for the video page and the videos list's detail
+// pane (server only).
+//
+// RELEASE 17: a tiered file is a RELATIVE SYMLINK in `data/<id>/` into
+// `channels/<slug>/media/`, and a dirent `isFile()` filter hides it — on a
+// tiered channel the audio would vanish from both pages. So links are listed
+// too, and BY FILE KIND:
+//
+// - a real file is statted directly (the corpus disk);
+// - a link is `lstat`ed for its time (the hook gives the link the file's own
+// times, so the corpus disk answers it) and its SIZE is asked of the media
+// drive — every media link of the directory in ONE `onDrive(mediaDir)` call
+// when the channel's media is relocated, so a drive that does not answer
+// costs one budget, not one per file, and a stalled one none at all. A
+// link whose target is not there (an unmounted or renamed-away drive) or
+// whose drive does not answer is listed with `size: null` — the file exists;
+// its bytes are not reachable now.
+//
+// The mover's tree walk (relocateDir.ts) keeps its own `isFile()`: over
+// `data/` it measures the corpus disk's text, over `media/` the real bytes.
+export async function loadVideoDirFiles(
+ videoDir: string,
+ // `config.mediaDir`, when the channel's media is relocated. Absent: an
+ // in-place `media/` is on the corpus disk and needs no watchdog.
+ mediaDir: string | undefined,
+): Promise<{ files: VideoFile[] }> {
+ let entries: Dirent[];
+ try {
+ entries = await readdir(videoDir, { withFileTypes: true });
+ } catch {
+ return { files: [] };
+ }
+ const files: VideoFile[] = [];
+ const links: Dirent[] = [];
+ for (const e of entries) {
+ if (e.name.includes(".tmp-")) continue;
+ if (e.isFile()) {
+ const s = await stat(path.join(videoDir, e.name)).catch(() => null);
+ if (s) files.push({ name: e.name, size: s.size, mtime: s.mtimeMs });
+ } else if (e.isSymbolicLink()) {
+ links.push(e);
+ }
+ }
+ if (links.length > 0) {
+ const sizes = new Map<string, number>();
+ const statLinks = async (kind: "media" | "other") => {
+ for (const e of links) {
+ const isMedia = isTierable(e.name);
+ if ((kind === "media") !== isMedia) continue;
+ const s = await stat(path.join(videoDir, e.name)).catch(() => null);
+ if (s?.isFile()) sizes.set(e.name, s.size);
+ }
+ };
+ // A link that is not a media name is not the tier's (nothing makes one);
+ // it is statted directly, like the files.
+ await statLinks("other");
+ try {
+ await (mediaDir ? onDrive(mediaDir, () => statLinks("media")) : statLinks("media"));
+ } catch {
+ /* the drive did not answer: the sizes stay unknown */
+ }
+ for (const e of links) {
+ const l = await lstat(path.join(videoDir, e.name)).catch(() => null);
+ if (!l) continue;
+ files.push({
+ name: e.name,
+ size: sizes.get(e.name) ?? null,
+ mtime: l.mtimeMs,
+ });
+ }
+ }
+ files.sort((a, b) => a.name.localeCompare(b.name));
+ return { files };
+}
diff --git a/editor/app/channels/[slug]/page.tsx b/editor/app/channels/[slug]/page.tsx
@@ -635,7 +635,11 @@ export default async function ChannelDetailPage({
// and stated in the action too, because a disabled button is a courtesy
// and the server is the guard. "Busy" is jobs AND in-flight auto-queue
// units, which make no job record — see lib/mediaBusy.ts.
- const busy = channelMediaBusyReason(slug, "moving its media");
+ // The move holds the media writers only (release 17): a digest may
+ // run during it.
+ const busy = channelMediaBusyReason(slug, "moving its media", {
+ mediaOnly: true,
+ });
const blockedReason =
busy ??
(marker
@@ -658,7 +662,10 @@ export default async function ChannelDetailPage({
// From the loaded snapshot, not a walk. Null (rendered "—") when the
// snapshot predates the field or does not exist: a 0 would claim a
// measurement nobody took.
- mediaBytes={snapshot.totalAudioBytes ?? null}
+ mediaBytes={snapshot.totalMediaBytes ?? null}
+ // The text tier (release 17). Null on a report written before
+ // it: unknown, never 0.
+ textBytes={snapshot.totalTextBytes ?? null}
// The clips/ share of the same snapshot — the number the evict
// card acts on. Null, not 0, when the field is absent: the
// snapshot may predate it.
diff --git a/editor/app/channels/[slug]/storageActions.ts b/editor/app/channels/[slug]/storageActions.ts
@@ -16,7 +16,7 @@
// because the in-memory registry keys by slug and those jobs would be orphaned
// by the move. A relocation has the same hazard with a sharper edge — a download
// or a transcribe running against `data/` WHILE its bytes are being copied out
-// would write into the directory the swap is about to replace, and the verify
+// would write into the `media/` the swap is about to replace, and the verify
// would then fail (which is the safe outcome) or the write would be lost (which
// is not). The check is cheap and refuses early, before any bytes move.
//
@@ -32,11 +32,12 @@ import type { StreamActionResult } from "yt-dlp-transcript-common/jobs/streamCom
import {
clearRelocationMarker,
readRelocationMarker,
- relocatedDataDir,
} from "yt-dlp-transcript-common/lib/channelMedia";
+import { relocatedMediaDir } from "yt-dlp-transcript-common/lib/mediaTier-server";
import {
previewRelocation,
relocationRootProblem,
+ tierMigrationRefusal,
type RelocationPreview,
} from "yt-dlp-transcript-common/controller/relocateChannelMedia";
import { enqueueRelocation } from "../lib/relocationJob";
@@ -49,10 +50,12 @@ import { channelMediaBusyReason } from "../lib/mediaBusy";
export type PreviewRelocationResult =
{ ok: true; preview: RelocationPreview } | { ok: false; error: string };
-// Read-only: one tree walk of the channel's data dir plus two statfs calls. It
-// runs INLINE rather than as a job because its whole purpose is to answer a
-// question the operator is holding a form open for; a queued job with a log
-// would be a worse way to show two numbers.
+// One tree walk of the channel's `media/` plus two statfs calls — and, the
+// first time for a classic channel, the tiering the job's preflight would do
+// (same-disk renames, idempotent: release 17's ruling). It runs INLINE rather
+// than as a job because its whole purpose is to answer a question the operator
+// is holding a form open for; a queued job with a log would be a worse way to
+// show two numbers.
export async function previewRelocationAction(
slug: string,
dest: MoveDestination,
@@ -93,7 +96,9 @@ export async function relocateChannelMediaAction(
root,
});
if (rootProblem) return { ok: false, error: rootProblem };
- const refusal = channelMediaBusyReason(slug, "moving its media");
+ const refusal = channelMediaBusyReason(slug, "moving its media", {
+ mediaOnly: true,
+ });
if (refusal) return { ok: false, error: refusal };
return enqueueRelocation({ slug, direction: "out", root });
}
@@ -112,7 +117,7 @@ export async function relocateChannelMediaAction(
//
// So this enqueues the SAME relocation job with the SAME direction, and the
// controller does the resuming. What it adds is the root, which the marker does
-// not carry: a marker records `<root>/<slug>/data`, so the root is its
+// not carry: a marker records `<root>/<slug>/media`, so the root is its
// grandparent. That inversion is checked rather than assumed — if rebuilding
// the target from the derived root does not give back the marker's own target
// (a hand-edited marker, a slug with a separator in it, a future layout), this
@@ -160,18 +165,23 @@ async function resumeRelocation(
const refusal = channelMediaBusyReason(
slug,
reconcile ? "reconciling its move" : "resuming its move",
+ { mediaOnly: true },
);
if (refusal) return { ok: false, error: refusal };
+ // A TIER MIGRATION'S MARKER (release 17) is not this panel's to resume:
+ // the migration runs with the editor stopped and resumes itself.
+ const migrating = tierMigrationRefusal(slug, marker);
+ if (migrating) return { ok: false, error: migrating };
if (marker.direction === "back") {
return enqueueRelocation({ slug, direction: "back", reconcile });
}
const root = path.dirname(path.dirname(marker.target));
- if (marker.target !== relocatedDataDir(root, slug)) {
+ if (marker.target !== relocatedMediaDir(root, slug)) {
return {
ok: false,
error:
`The relocation marker points at ${marker.target}, which is not ` +
- `<root>/${slug}/data — the destination root cannot be recovered from ` +
+ `<root>/${slug}/media — the destination root cannot be recovered from ` +
`it. Clear the marker and start the move again.`,
};
}
@@ -181,7 +191,9 @@ async function resumeRelocation(
export async function moveChannelMediaBackAction(
slug: string,
): Promise<StreamActionResult> {
- const refusal = channelMediaBusyReason(slug, "moving its media back");
+ const refusal = channelMediaBusyReason(slug, "moving its media back", {
+ mediaOnly: true,
+ });
if (refusal) return { ok: false, error: refusal };
return enqueueRelocation({ slug, direction: "back" });
}
@@ -207,8 +219,16 @@ export async function clearRelocationMarkerAction(
const refusal = channelMediaBusyReason(
slug,
"clearing its relocation marker",
+ { mediaOnly: true },
);
if (refusal) return { ok: false, error: refusal };
+ // AN INTERRUPTED TIER MIGRATION'S MARKER is never cleared from here: it is
+ // the phase the migration resumes from (the same sentence Resume gives).
+ const migrating = tierMigrationRefusal(
+ slug,
+ await readRelocationMarker(getPaths(), slug),
+ );
+ if (migrating) return { ok: false, error: migrating };
try {
await clearRelocationMarker(getPaths(), slug);
} catch (e) {
diff --git a/editor/app/channels/[slug]/videos/[id]/components/cards/FilesList.tsx b/editor/app/channels/[slug]/videos/[id]/components/cards/FilesList.tsx
@@ -41,7 +41,8 @@ export function FilesList({
{f.name}
</span>
<span className="text-xs text-muted-foreground">
- {formatBytes(f.size)} · {new Date(f.mtime).toLocaleString()}
+ {f.size === null ? "— (media drive not reachable)" : formatBytes(f.size)}{" "}
+ · {new Date(f.mtime).toLocaleString()}
</span>
</div>
{kind === "audio" || kind === "video" ? (
diff --git a/editor/app/channels/[slug]/videos/[id]/components/cards/videoFiles.ts b/editor/app/channels/[slug]/videos/[id]/components/cards/videoFiles.ts
@@ -5,7 +5,10 @@ import {
export type VideoFile = {
name: string;
- size: number;
+ // NULL for a tiered file (a link into `channels/<slug>/media/`, release 17)
+ // whose media drive is unmounted or not answering: the file exists, its size
+ // is not knowable now. Rendered "—", never 0.
+ size: number | null;
mtime: number;
};
diff --git a/editor/app/channels/[slug]/videos/[id]/page.tsx b/editor/app/channels/[slug]/videos/[id]/page.tsx
@@ -2,8 +2,6 @@ import type { Metadata } from "next";
import Link from "next/link";
import { notFound } from "next/navigation";
import path from "node:path";
-import { readdir, stat } from "node:fs/promises";
-import type { Dirent } from "node:fs";
import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels";
import { loadDownloadOutcome } from "yt-dlp-transcript-common/lib/downloadOutcome-server";
import { loadAvailability } from "yt-dlp-transcript-common/lib/availability-server";
@@ -42,7 +40,8 @@ import { TagsPanel } from "./components/TagsPanel";
import { MetadataHistoryDetails } from "./components/MetadataHistoryDetails";
import { loadVideoTags } from "./lib/videoTags";
import { loadVideoOperationPanels } from "./lib/videoOperationPanels";
-import { channelMediaStall } from "yt-dlp-transcript-common/lib/channelMedia";
+import { channelTextStall } from "yt-dlp-transcript-common/lib/channelMedia";
+import { loadVideoDirFiles } from "../../lib/videoDirFiles";
import {
isDriveNotAnswering,
onDrive,
@@ -51,26 +50,15 @@ import { MediaNotAnswering } from "../../components/MediaNotAnswering";
export const dynamic = "force-dynamic";
+// The directory's files, tiered links included (release 17), each media
+// link's size asked of the media drive. See lib/videoDirFiles.ts.
async function loadVideoDir(
slug: string,
videoId: string,
+ mediaDir: string | undefined,
): Promise<{ files: VideoFile[] }> {
const dir = path.join(getPaths().channelsDir, slug, "data", videoId);
- let entries: Dirent[];
- try {
- entries = await readdir(dir, { withFileTypes: true });
- } catch {
- return { files: [] };
- }
- const files: VideoFile[] = [];
- for (const e of entries) {
- if (!e.isFile()) continue;
- if (e.name.includes(".tmp-")) continue;
- const s = await stat(path.join(dir, e.name));
- files.push({ name: e.name, size: s.size, mtime: s.mtimeMs });
- }
- files.sort((a, b) => a.name.localeCompare(b.name));
- return { files };
+ return loadVideoDirFiles(dir, mediaDir);
}
// metadata.info.json when the video was downloaded, else the channel's
@@ -86,9 +74,11 @@ export async function generateMetadata({
params: Promise<{ slug: string; id: string }>;
}): Promise<Metadata> {
const { slug, id } = await params;
- // The title is read off the drive; a drive that is not answering is not
- // asked, and one that does not answer within the budget
- // (`storage.health.budgetMs`, 3 s by default) is given up on.
+ // The title is TEXT, on the corpus disk (release 17). Only a channel still on
+ // the retired whole-directory layout has its text on another drive: that
+ // drive is not asked while it is not answering, and one that does not answer
+ // within the budget (`storage.health.budgetMs`, 3 s by default) is given up
+ // on.
const drive = (await readChannelConfig(getPaths(), slug))?.dataDir?.trim();
let subject = id;
try {
@@ -110,18 +100,24 @@ export default async function VideoDetailPage({
const { slug, id } = await params;
const config = await readChannelConfig(getPaths(), slug);
if (!config) notFound();
- // Everything below reads the video's directory on the channel's drive; a
- // drive that is not answering is not read. See MediaNotAnswering.
- const stall = channelMediaStall(config);
+ // Everything below reads the video's TEXT, which is on the corpus disk
+ // (release 17): a stalled MEDIA drive does not hold this page — its audio
+ // is listed with its size unknown, and the file route answers 503 for it.
+ // Only a legacy channel (text on its retired `dataDir`) is held here. See
+ // MediaNotAnswering.
+ const stall = channelTextStall(config);
if (stall) {
return <MediaNotAnswering slug={slug} stall={stall} what="This video's page" />;
}
- // EVERY READ BELOW IS OF THIS VIDEO'S DIRECTORY, on the channel's drive when
- // it is relocated, so they go through the watchdog as one unit: a drive that
- // has not answered them within the budget (3 s by default) is marked stalled
- // and the page says so.
+ const mediaDir = config.mediaDir?.trim() || undefined;
+ // On a legacy channel EVERY READ BELOW is of a directory on its retired
+ // drive, so they go through the watchdog as one unit: a drive that has not
+ // answered them within the budget (3 s by default) is marked stalled and the
+ // page says so. On every other channel they read the corpus disk directly;
+ // the media links' sizes go through the media drive's watchdog on their own
+ // (lib/videoDirFiles.ts).
const loadAll = async () => {
- const dirData = await loadVideoDir(slug, id);
+ const dirData = await loadVideoDir(slug, id, mediaDir);
const meta = await loadMeta(slug, id);
const videoDir = path.join(getPaths().channelsDir, slug, "data", id);
const downloadOutcome = await loadDownloadOutcome(videoDir);
diff --git a/editor/app/channels/[slug]/videos/[id]/videoActions.ts b/editor/app/channels/[slug]/videos/[id]/videoActions.ts
@@ -1,7 +1,7 @@
"use server";
import path from "node:path";
-import { readdir, readFile, rm, stat } from "node:fs/promises";
+import { lstat, readdir, readFile, rm, stat } from "node:fs/promises";
import { revalidatePath } from "next/cache";
import { safeRevalidate } from "../../../../lib/safeRevalidate";
import { redirect } from "next/navigation";
@@ -38,9 +38,11 @@ import {
} from "yt-dlp-transcript-common/controller/keepVideosMatching";
import {
ChannelMediaUnreachableError,
+ channelTextStall,
inspectChannelMedia,
} from "yt-dlp-transcript-common/lib/channelMedia";
import { onDrive } from "yt-dlp-transcript-common/lib/storageHealth";
+import { isTierable } from "yt-dlp-transcript-common/lib/mediaTier";
import { setExcludedFromTruncatedCheck } from "yt-dlp-transcript-common/lib/excludeTruncatedCheck-server";
import { pruneFailedTranscriptions } from "yt-dlp-transcript-common/controller/failedTranscriptions";
import { transcodeAudio } from "yt-dlp-transcript-common/controller/transcode";
@@ -499,10 +501,54 @@ export async function deleteVideoFileAction(
if (!target) {
return { ok: false, error: `Refusing to delete suspicious filename "${filename}"` };
}
+ // A TIERED file's stat — a link (the `lstat`) with a tierable name — is
+ // asked of the media tier's drive, through its watchdog (release 17); a
+ // real file, `source-media.*` included, is on the corpus disk (review N7).
+ //
+ // A LEGACY channel (review R1) is asked nothing here: its whole `data/` is a
+ // link onto the retired drive, so the `lstat` would reach it — refused while
+ // that drive is known not to answer, and otherwise read directly as before
+ // (it has no tiered links).
+ const config = await readChannelConfig(getPaths(), slug);
+ if (channelTextStall(config)) {
+ return {
+ ok: false,
+ error: `${filename} not deleted: this channel's drive is not answering.`,
+ };
+ }
+ const linked = config?.dataDir?.trim()
+ ? false
+ : await lstat(target)
+ .then((l) => l.isSymbolicLink())
+ .catch(() => false);
+ const mediaDrive =
+ linked && isTierable(path.basename(target))
+ ? config?.mediaDir?.trim()
+ : undefined;
let s;
try {
- s = await stat(target);
+ s = await (mediaDrive
+ ? onDrive(mediaDrive, () => stat(target))
+ : stat(target));
} catch {
+ // A TIERED FILE WHOSE DRIVE IS NOT THERE: removing the link alone would
+ // orphan its bytes on that drive, so nothing is removed.
+ if (linked && mediaDrive) {
+ return {
+ ok: false,
+ error:
+ `${filename} is on this channel's media drive, which is not mounted ` +
+ `or not answering — nothing was deleted.`,
+ };
+ }
+ // A dangling link on an in-place `media/`: its bytes are already gone, so
+ // the link is all there is to remove (review N8).
+ if (linked) {
+ await removeMediaFile(videoDir, path.basename(target));
+ revalidatePath(`/channels/${slug}/videos/${videoId}`);
+ requestChannelSnapshot(getPaths(), slug);
+ return { ok: true };
+ }
return { ok: false, error: `File not found: ${filename}` };
}
if (!s.isFile()) {
diff --git a/editor/app/channels/[slug]/videos/page.tsx b/editor/app/channels/[slug]/videos/page.tsx
@@ -1,6 +1,4 @@
import path from "node:path";
-import { readdir, stat } from "node:fs/promises";
-import type { Dirent } from "node:fs";
import type { Metadata } from "next";
import { notFound } from "next/navigation";
import { isSocialChannel } from "yt-dlp-transcript-common/lib/channelConfig";
@@ -36,7 +34,8 @@ import { computeVideoRows, readDataDirVideoIds } from "../lib/videoRowsServer";
import { normalizeBuckets } from "yt-dlp-transcript-common/views/pipeline/stageStatus";
import { attachCuratedTags } from "../lib/videoTagRows";
import { readChannelVideoTitles } from "yt-dlp-transcript-common/controller/videoTitles";
-import { channelMediaStall } from "yt-dlp-transcript-common/lib/channelMedia";
+import { channelTextStall } from "yt-dlp-transcript-common/lib/channelMedia";
+import { loadVideoDirFiles } from "../lib/videoDirFiles";
import {
isDriveNotAnswering,
onDrive,
@@ -45,26 +44,14 @@ import { MediaNotAnswering } from "../components/MediaNotAnswering";
export const dynamic = "force-dynamic";
+// The selected video's files, tiered links included (release 17): a dirent
+// `isFile()` filter hides them. See lib/videoDirFiles.ts.
async function loadVideoDir(
channelDataDir: string,
videoId: string,
+ mediaDir: string | undefined,
): Promise<{ files: VideoFile[] }> {
- const dir = path.join(channelDataDir, videoId);
- let entries: Dirent[];
- try {
- entries = await readdir(dir, { withFileTypes: true });
- } catch {
- return { files: [] };
- }
- const files: VideoFile[] = [];
- for (const e of entries) {
- if (!e.isFile()) continue;
- if (e.name.includes(".tmp-")) continue;
- const s = await stat(path.join(dir, e.name));
- files.push({ name: e.name, size: s.size, mtime: s.mtimeMs });
- }
- files.sort((a, b) => a.name.localeCompare(b.name));
- return { files };
+ return loadVideoDirFiles(path.join(channelDataDir, videoId), mediaDir);
}
export async function generateMetadata({
@@ -106,10 +93,12 @@ export default async function ChannelVideosPage({
// A social channel has posts, not videos — there is no data directory to list
// and nothing here would render. 404 rather than an empty workspace.
if (isSocialChannel(config)) notFound();
- // THE LIST IS READ OFF THE DRIVE (a readdir of data/, a head read per title,
- // the selected video's files), so a drive that is not answering is not read:
- // the page says so instead. See MediaNotAnswering.
- const stall = channelMediaStall(config);
+ // THE LIST IS TEXT (a readdir of data/, a head read per title, the selected
+ // video's files), on the corpus disk since release 17: a stalled MEDIA drive
+ // does not hold it. Only a legacy channel's text is on another drive, and a
+ // drive that is not answering is not read: the page says so instead. See
+ // MediaNotAnswering.
+ const stall = channelTextStall(config);
if (stall) {
return <MediaNotAnswering slug={slug} stall={stall} what="The video list" />;
}
@@ -144,11 +133,12 @@ export default async function ChannelVideosPage({
);
const channelDataDir = path.join(paths.channelsDir, slug, "data");
- // THE READS OF THE DRIVE go through the watchdog when the channel is
- // relocated: a drive that has not answered them within the budget
- // (`storage.health.budgetMs`, 3 s by default) is marked stalled and the page
- // says so instead (see MediaNotAnswering).
+ // THE READS OF A LEGACY CHANNEL'S DRIVE go through the watchdog: a drive
+ // that has not answered them within the budget (`storage.health.budgetMs`,
+ // 3 s by default) is marked stalled and the page says so instead (see
+ // MediaNotAnswering). Every other channel's text is on the corpus disk.
const drive = config.dataDir?.trim();
+ const mediaDir = config.mediaDir?.trim() || undefined;
const onMedia = <T,>(call: () => Promise<T>): Promise<T> =>
drive ? onDrive(drive, call) : call();
let channelDataDirIds: string[];
@@ -221,7 +211,7 @@ export default async function ChannelVideosPage({
let loadedVideo;
try {
loadedVideo = await onMedia(() => Promise.all([
- loadVideoDir(channelDataDir, selectedVideoId),
+ loadVideoDir(channelDataDir, selectedVideoId, mediaDir),
// Already read for the list — the title map covers every row.
Promise.resolve(titles.get(selectedVideoId)?.title ?? null),
loadDownloadOutcome(videoDir),
diff --git a/editor/app/channels/bulkStorageActions.ts b/editor/app/channels/bulkStorageActions.ts
@@ -47,10 +47,9 @@ import {
// so the bar renders it the same way they do.
export type BulkRelocateResult = QueueOutcome;
-// Follows the link, deliberately: for a relocated channel `data/` is a symlink
-// and what matters is whether the thing it points at is there. (This path only
-// reaches it for a channel inspect() already called in-place, so in practice it
-// is a real directory or nothing.)
+// Follows a link, deliberately. This path only reaches it for a channel
+// inspect() already called in-place, whose `data/` is a real directory on the
+// corpus disk or nothing.
async function isDirectory(p: string): Promise<boolean> {
try {
return (await stat(p)).isDirectory();
@@ -100,7 +99,7 @@ export async function bulkRelocateChannelMediaAction(
}
// Jobs AND in-flight auto-queue units — the lanes write into `data/`
// and make no job record. See lib/mediaBusy.ts.
- const busy = channelMediaBusyReason(slug);
+ const busy = channelMediaBusyReason(slug, undefined, { mediaOnly: true });
if (busy) return busy;
const media = await inspectChannelMedia(paths, slug, config);
// NOTHING TO MOVE IS A SKIP, NOT A JOB. A channel that has downloaded
@@ -116,6 +115,11 @@ export async function bulkRelocateChannelMediaAction(
if (media.marker) {
return `a relocation (${media.marker.direction}) to ${media.marker.target} is already in flight`;
}
+ // THE RETIRED LAYOUT (release 17) is migrated, never moved: the skip
+ // names the command.
+ if (media.status === "legacy") {
+ return `cannot be moved — ${media.detail ?? "its media layout is the retired whole-directory one"}`;
+ }
// ALREADY RELOCATED IS A SKIP, NOT A FAILURE — including to a DIFFERENT
// root. Selecting the whole page and pressing Move is the ordinary way
// this gets used, and the channels already on the platter are exactly the
@@ -123,7 +127,7 @@ export async function bulkRelocateChannelMediaAction(
if (media.relocated) {
return `already relocated to ${media.target ?? "another root"}`;
}
- // Containment is per-channel because the target is: `<root>/<slug>/data`
+ // Containment is per-channel because the target is: `<root>/<slug>/media`
// can resolve into one channel's directory and not another's.
return relocationRootProblem({ paths, slug, root: chosen });
},
diff --git a/editor/app/channels/components/ChannelForm.tsx b/editor/app/channels/components/ChannelForm.tsx
@@ -717,11 +717,11 @@ export function ChannelForm({
/>
{/* READ-ONLY, AND NOT AN INPUT — the one field on this page that is a
record of the disk rather than an instruction to it.
- `config.dataDir` is written ONLY by the relocate job, on success,
+ `config.mediaDir` is written ONLY by the relocate job, on success,
after the bytes are copied, verified and the symlink is in place.
An editable text box here would let the two disagree with a
keystroke: type a path nothing was moved to and every reader
- follows a `data/` link that still points somewhere else, which
+ follows a `media` link that still points somewhere else, which
inspectChannelMedia reports as `inconsistent` and refuses to guess
past. So the move is the only writer, and this line just says what
it wrote. (It is also not in CHANNEL_FORM_FIELDS, so saving this
@@ -732,14 +732,17 @@ export function ChannelForm({
aria-label="media location"
className="font-mono text-xs break-all rounded border border-border bg-muted px-2 py-1"
>
- {c?.dataDir?.trim()
- ? c.dataDir.trim()
- : "In the channel directory (data/)"}
+ {c?.mediaDir?.trim()
+ ? c.mediaDir.trim()
+ : c?.dataDir?.trim()
+ ? `${c.dataDir.trim()} (the retired whole-directory layout — run archilyzer storage migrate-tier)`
+ : "In the channel directory (media/)"}
</span>
<span className="text-xs text-muted-foreground">
- Where this channel’s downloaded media actually lives. Change it
- from the Storage panel, which copies and verifies the bytes before
- recording anything here.
+ Where this channel’s big files — the audio and the raw live
+ chat — actually live; its text is always in the channel directory.
+ Change it from the Storage panel, which copies and verifies the
+ bytes before recording anything here.
</span>
</div>
</Section>
diff --git a/editor/app/channels/components/ChannelVolumeBar.tsx b/editor/app/channels/components/ChannelVolumeBar.tsx
@@ -28,7 +28,7 @@ import { LOCATION_FILTER_PARAM } from "yt-dlp-transcript-common/views/storage";
// thing here that stays local.
export type ChannelVolume = {
- // A location id, "internal" for the corpus volume, or "" for a dataDir under
+ // A location id, "internal" for the corpus volume, or "" for a mediaDir under
// a root nobody named.
id: string;
label: string;
diff --git a/editor/app/channels/lib/mediaBusy.ts b/editor/app/channels/lib/mediaBusy.ts
@@ -27,6 +27,7 @@ import {
channelWriters,
describeChannelWriter,
} from "yt-dlp-transcript-common/controller/channelWriters";
+import { channelMediaWriters } from "yt-dlp-transcript-common/controller/relocateChannelMedia";
// A sentence naming what is holding the channel, or null when nothing is. The
// caller supplies the verb (`"moving its media"`) so the same reason reads as
@@ -39,11 +40,19 @@ import {
// and its job's first step ask — with queued jobs counted here, because this is
// the courtesy before anything is enqueued. A lane's download unit is also a
// registry job; it is counted once, as the unit.
+//
+// `mediaOnly` (release 17): a MOVE of the media tier is held only by the
+// channel's media writers — a digest may run during it (`channelMediaWriters`,
+// the same list the move's preview and its first step ask). A rename or a
+// delete moves or removes the whole channel and asks every writer.
export function channelMediaBusyReason(
slug: string,
what?: string,
+ opts: { mediaOnly?: boolean } = {},
): string | null {
- const writers = channelWriters(slug, { includeQueued: true });
+ const writers = opts.mediaOnly
+ ? channelMediaWriters(slug, { includeQueued: true })
+ : channelWriters(slug, { includeQueued: true });
if (writers.length === 0) return null;
const jobs = writers.filter((w) => w.source === "job").length;
const units = writers.length - jobs;
diff --git a/editor/app/channels/lib/relocationJob.ts b/editor/app/channels/lib/relocationJob.ts
@@ -109,7 +109,10 @@ export async function enqueueRelocation(opts: {
// fail the verify with 131 GB correctly on the far side.
(result.retried
? " (a change made during the copy was settled by a second mirror pass)"
- : ""),
+ : "") +
+ // A classic channel's first move tiers it in place first (release
+ // 17): its big files renamed into `media/` and linked from `data/`.
+ (result.tiered > 0 ? ` (${result.tiered} file(s) tiered first)` : ""),
);
// No snapshot regen — deliberately, and `relocate-channel-media` is in
// NO_REGEN_KINDS so the central hook does not arm one either. The move
diff --git a/editor/app/channels/page.tsx b/editor/app/channels/page.tsx
@@ -252,7 +252,7 @@ export default async function ChannelsPage({
// in-place one and names the location from the volume, which is the
// same prefix match the Location column shows — one answer, not two.
media: mediaBySlug.get(stat.slug) ?? null,
- volume: channelVolumeOf(brief?.config.dataDir, locations),
+ volume: channelVolumeOf(brief?.config, locations),
priority: {
tier: tierOf(priority, stat.slug),
rank: rankOf(priority, stat.slug),
diff --git a/editor/app/components/MediaLocationBadge.tsx b/editor/app/components/MediaLocationBadge.tsx
@@ -10,19 +10,25 @@ import type { ChannelRowMedia } from "yt-dlp-transcript-common/views/channelRow"
// the filesystem; the inspect() call that produces the location happens on the
// server, once per row, and only its result travels.
//
-// WHAT THE SIX STATUSES LOOK LIKE, and why there are only three appearances:
+// WHAT THE SEVEN STATUSES LOOK LIKE, and why there are only three appearances:
//
// in-place → NOTHING. The overwhelming majority of channels are in place,
// and a badge on every row saying "normal" is noise that makes
// the two that matter harder to see, not easier.
// ok → neutral. Relocated and reachable is a fact worth stating (the
// bytes are not on the corpus disk) but it is not a problem.
-// everything → red. unreachable, in-transition, inconsistent and stalled are
-// else all "do not trust what this channel's dirs say right now": the
-// first because the drive is not mounted, the second because a
-// move is half-done, the third because disk and config disagree
-// and nothing here is willing to guess which one is right, the
-// fourth because the drive is mounted and not answering.
+// everything → red. unreachable, in-transition, inconsistent, stalled and
+// else legacy are all "do not trust what this channel's media says
+// right now": the first because the drive is not mounted, the
+// second because a move is half-done, the third because disk and
+// config disagree and nothing here is willing to guess which one
+// is right, the fourth because the drive is mounted and not
+// answering, the fifth because the channel is still on the
+// retired whole-directory layout and is held until
+// `archilyzer storage migrate-tier` runs (release 17).
+//
+// Release 17: "media" is the media TIER — `channels/<slug>/media`, the big
+// files. A channel's text is on the corpus disk whatever this badge says.
//
// The `detail` string is the operator's prose from inspect() — the drive path,
// the phase, the disagreement — and it goes on `title` so a row badge carries
@@ -54,8 +60,9 @@ const LABELS: Record<ChannelMediaStatus, string | null> = {
"in-transition": "Media moving",
inconsistent: "Media inconsistent",
stalled: "Media not answering",
- // Release 17: the retired whole-directory layout (`archilyzer storage
- // migrate-tier`). Added by slice T1 so the status union stays exhaustive.
+ // Release 17: the retired whole-directory layout, held until
+ // `archilyzer storage migrate-tier` runs. Danger, like every non-ok status;
+ // the detail on the title names the command.
legacy: "Media layout retired",
};
@@ -76,7 +83,7 @@ const SHORT_STATUS: Record<ChannelMediaStatus, string | null> = {
// caller that could not inspect). Both are the same instruction to a renderer.
//
// `locationLabel` IS THE NAME OF THE STORAGE LOCATION the channel's media sits
-// on — `locationOfDataDir(config.dataDir, settings.storage.locations)`, resolved
+// on — `locationOfDataDir(config.mediaDir, settings.storage.locations)`, resolved
// on the SERVER where the settings are, and passed down as a string. Undefined
// when it sits on none: a root the operator typed by hand, or a corpus with no
// locations configured.
diff --git a/editor/app/page.tsx b/editor/app/page.tsx
@@ -100,7 +100,7 @@ export default async function Dashboard({
bands: [],
priority: neutralChannelPriority(),
media,
- volume: channelVolumeOf(r.channel.config.dataDir, locations),
+ volume: channelVolumeOf(r.channel.config, locations),
// The named location, matched against the configured roots from the
// media TARGET, as this table always has. A pure prefix match, never a
// probe.
diff --git a/editor/app/storage/actions.ts b/editor/app/storage/actions.ts
@@ -190,7 +190,7 @@ export async function editStorageLocationAction(
}
// DELETING A LOCATION MOVES NOTHING, and that is exactly why it is refused
-// while channels are on it: their `config.dataDir` would keep naming an
+// while channels are on it: their `config.mediaDir` would keep naming an
// absolute path on a disk nothing in the corpus remembers the name of, which is
// the situation this page exists to end.
export async function deleteStorageLocationAction(
diff --git a/editor/app/storage/buildStorage.ts b/editor/app/storage/buildStorage.ts
@@ -50,14 +50,40 @@ export async function buildStorage(): Promise<StorageRowsPayload> {
// snapshot predates `totalMediaBytes` contributes to `unknownBytes` and the
// row says so rather than under-reporting.
const briefs = await listChannelBriefs(paths);
+ // THE THREE TIERS OFF EACH REPORT (release 17): the media tier (what a
+ // location holds), the text tier and the clip windows (on the corpus volume
+ // whatever the media's location). A report written before release 17 has no
+ // text figure: unknown, never 0.
const mediaBytes: Record<string, number | undefined> = {};
- // The `clips/` share, off the same snapshot. Absent for a report written
- // before the field existed — the same set `mediaBytes` is absent for, which
- // is why the rollup needs no second "unknown" counter.
const clipsBytes: Record<string, number | undefined> = {};
+ const textBytes: Record<string, number | undefined> = {};
+ const corpus = { textBytes: 0, clipsBytes: 0, unknown: 0, legacy: 0 };
for (const b of briefs) {
mediaBytes[b.slug] = b.snapshot?.totalMediaBytes;
clipsBytes[b.slug] = b.snapshot?.totalClipsBytes;
+ textBytes[b.slug] = b.snapshot?.totalTextBytes;
+ // A LEGACY channel's text and clips are not on the corpus volume at all
+ // (they are on its retired `dataDir`, and its report cannot be refreshed
+ // until it is migrated): counted as "to migrate", never added here and
+ // never "unmeasured" (review N6).
+ if (b.config.dataDir?.trim()) {
+ corpus.legacy += 1;
+ continue;
+ }
+ const text = b.snapshot?.totalTextBytes;
+ const clips = b.snapshot?.totalClipsBytes;
+ if (typeof text === "number") corpus.textBytes += text;
+ if (typeof clips === "number") corpus.clipsBytes += clips;
+ // Unmeasured on the corpus volume: a report with no text figure — except
+ // an in-place channel whose MEDIA is unmeasured too, which the internal
+ // rollup already counts (one channel, one "unmeasured").
+ const inPlace = !b.config.mediaDir?.trim();
+ if (
+ typeof text !== "number" &&
+ !(inPlace && typeof b.snapshot?.totalMediaBytes !== "number")
+ ) {
+ corpus.unknown += 1;
+ }
}
const configs = briefs.map((b) => ({ slug: b.slug, config: b.config }));
const [probes, rollups, udisksctl, freeOnCorpus] = await Promise.all([
@@ -68,6 +94,7 @@ export async function buildStorage(): Promise<StorageRowsPayload> {
configs,
mediaBytes,
clipsBytes,
+ textBytes,
includeInternal: true,
}),
// Memoised per binary path inside storageVolumes, so this is one
@@ -141,6 +168,7 @@ export async function buildStorage(): Promise<StorageRowsPayload> {
internal: {
root: paths.channelsDir,
...(Number.isFinite(freeOnCorpus) ? { freeBytes: freeOnCorpus } : {}),
+ corpus,
},
defaultLocationId: settings.storage.defaultLocationId,
probes,
diff --git a/editor/app/storage/components/StorageLocationsTable.tsx b/editor/app/storage/components/StorageLocationsTable.tsx
@@ -233,10 +233,22 @@ function LocationCard({
<dt className="text-muted-foreground">Media</dt>
<dd aria-label="location media bytes">
{row.bytesText}
- {/* THE CLIPS SHARE, inside the Media figure and never beside it as a
- second total. A fetched window lives under `data/<id>/clips/`, so
- it is already in the number above; what this adds is that some of
- it is a CACHE nothing prunes. Absent when there are none. */}
+ {/* THE INTERNAL ROW'S BREAKDOWN (release 17): every channel's text
+ and clip windows stay on the corpus volume, plus the media of the
+ channels in place. A location row's figure is its media tier
+ alone, and carries no breakdown. */}
+ {row.tiersText && (
+ <span
+ aria-label="location tier bytes"
+ className="block text-xs text-muted-foreground"
+ >
+ {row.tiersText}
+ </span>
+ )}
+ {/* THE CLIPS SHARE, inside the figure and never beside it as a
+ second total: on the internal row (clips/ is never tiered), it is
+ already in the number above; what this adds is that some of it is
+ a CACHE nothing prunes. Absent when there are none. */}
{row.clipsText && (
<span
aria-label="location clip bytes"
@@ -374,8 +386,8 @@ function LocationCard({
<h3 className="text-sm font-semibold">Re-point</h3>
<p className="text-xs text-muted-foreground">
The volume is mounted at a different place than this location’s
- root. Re-pointing rewrites each channel’s <code>data/</code>{" "}
- symlink and its <code>dataDir</code>, then the root.{" "}
+ root. Re-pointing rewrites each channel’s <code>media</code>{" "}
+ symlink and its <code>mediaDir</code>, then the root.{" "}
<strong>No bytes move</strong> — the media is already there.
</p>
<StreamActionLog
diff --git a/editor/app/storage/page.tsx b/editor/app/storage/page.tsx
@@ -23,13 +23,14 @@ export default async function StoragePage() {
</div>
<p className="text-sm text-muted-foreground max-w-3xl">
- A storage location is a named place a channel’s media may live —
- usually a second drive. A channel is on a location when its{" "}
- <code>dataDir</code> is under that location’s root; nothing is
- tagged, so moving a channel on or off one is a move, not a setting. When
- a drive comes back at a different mountpoint, <strong>re-point</strong>{" "}
- the location: it rewrites every channel’s symlink and{" "}
- <code>dataDir</code> and moves no bytes. Move media onto a location from
+ A storage location is a named place a channel’s media — its big
+ files, the audio and the raw live chat — may live, usually a second
+ drive; a channel’s text never leaves the corpus volume. A channel
+ is on a location when its <code>mediaDir</code> is under that
+ location’s root; nothing is tagged, so moving a channel on or off
+ one is a move, not a setting. When a drive comes back at a different
+ mountpoint, <strong>re-point</strong> the location: it rewrites every
+ channel’s symlink and <code>mediaDir</code> and moves no bytes. Move media onto a location from
a channel’s <Link href="/channels" className="underline">Storage
panel</Link>.
</p>
diff --git a/editor/e2e/channel-storage.spec.ts b/editor/e2e/channel-storage.spec.ts
@@ -3,6 +3,8 @@ import {
lstat,
mkdir,
readdir,
+ readlink,
+ rename,
rm,
stat,
symlink,
@@ -21,11 +23,18 @@ import {
resetData,
resolvePath,
writeChannelConfig,
+ writeDigestVideo,
writeSettings,
} from "./helpers";
// MOVING A CHANNEL'S MEDIA TO ANOTHER DIRECTORY, AND BACK.
//
+// RELEASE 17: what moves is the MEDIA TIER — `channels/<slug>/media`, the big
+// files (the audio, the raw live chat), each reached from `data/<id>/<name>`
+// by a relative link. The text never leaves `data/` on the corpus disk. A
+// classic channel (its audio still a real file in `data/<id>/`) is tiered by
+// the move's preview and preflight first.
+//
// The thing this spec is really pinning is the claim the whole design rests on:
// that a relocated channel is indistinguishable from an in-place one to every
// reader. So the assertions after the move are deliberately NOT about the move —
@@ -52,6 +61,28 @@ const SLUG = "test-youtube";
const VIDEO = "20240101_test1234567";
const dataDir = () => resolvePath(`test-transcripts/channels/${SLUG}/data`);
+const mediaLink = () => resolvePath(`test-transcripts/channels/${SLUG}/media`);
+const videoFile = (name: string) => join(dataDir(), VIDEO, name);
+
+// A small audio file — the route streams bytes, it does not decode them.
+const AUDIO_BYTES = 4096;
+const AUDIO = "ID3" + "x".repeat(AUDIO_BYTES - 3);
+
+// THE TIERED LAYOUT, BY HAND: the bytes in `<mediaDir>/<id>/<name>` and the
+// RELATIVE link `data/<id>/<name> -> ../../media/<id>/<name>` — what the hook
+// leaves behind (lib/mediaTier-server.ts). `mediaDir` is the channel's real
+// `media/` (tiered in place) or a relocated target the `media` link points at.
+async function tierByHand(
+ mediaDir: string,
+ files: Record<string, string>,
+): Promise<void> {
+ await mkdir(join(mediaDir, VIDEO), { recursive: true });
+ for (const [name, content] of Object.entries(files)) {
+ await writeFile(join(mediaDir, VIDEO, name), content);
+ await rm(videoFile(name), { force: true });
+ await symlink(join("..", "..", "media", VIDEO, name), videoFile(name));
+ }
+}
// Wait until the channel has no running or queued job. Polled from the same
// endpoint /jobs draws, because the guard reads the same registry.
@@ -80,17 +111,26 @@ async function quiet(page: Page): Promise<void> {
test("relocate a channel's media to another root, and move it back", async ({
page,
}, testInfo) => {
- test.setTimeout(90_000);
+ test.setTimeout(120_000);
await resetData("one-youtube-channel-with-data");
await generateReport(page, SLUG);
await quiet(page);
const root = testInfo.outputPath("media-root");
await mkdir(root, { recursive: true });
- const target = join(root, SLUG, "data");
+ const target = join(root, SLUG, "media");
+ // A CLASSIC CHANNEL: its audio and its raw live chat are real files in
+ // `data/<id>/`, which is how every channel downloaded before release 17 is.
+ await writeFile(videoFile("audio.mp3"), AUDIO);
+ await writeFile(videoFile("transcript.live_chat.json"), "[]");
// --- before -------------------------------------------------------------
await page.goto(channelStage(SLUG, "storage"));
- await expect(page.getByLabel("media path")).toHaveText(dataDir());
+ // Two rows under one heading: the media tier (not there yet — in place) and
+ // the text, which never moves.
+ await expect(page.getByLabel("media path")).toHaveText(
+ `${mediaLink()} (in place)`,
+ );
+ await expect(page.getByLabel("text path")).toContainText(dataDir());
// In place draws no badge at all — a badge on every channel saying "normal"
// is what makes the one that matters hard to find.
await expect(page.getByLabel(/^media location:/)).toHaveCount(0);
@@ -114,8 +154,15 @@ test("relocate a channel's media to another root, and move it back", async ({
await page.getByRole("button", { name: "Preview", exact: true }).click();
const preview = page.getByLabel("relocation preview");
await expect(preview).toBeVisible({ timeout: 15_000 });
- // The fixture is two files in one video dir.
+ // The media tier only: the audio and the raw live chat — not the transcript
+ // or the metadata, which stay.
await expect(page.getByLabel("bytes to move")).toContainText("2 file(s)");
+ // THE PREVIEW TIERED THE CLASSIC CHANNEL FIRST, on this disk, and says so.
+ await expect(page.getByLabel("tiered first")).toContainText(
+ "2 file(s) tiered first",
+ );
+ expect((await lstat(videoFile("audio.mp3"))).isSymbolicLink()).toBe(true);
+ expect((await lstat(mediaLink())).isDirectory()).toBe(true);
// --- the move -----------------------------------------------------------
await expect(moveButton).toBeEnabled();
@@ -136,39 +183,59 @@ test("relocate a channel's media to another root, and move it back", async ({
/Copying… .* · \d+ % · /,
);
- // data/ is a symlink now, and config.json records the target — written only
- // by the job, on success, after the copy verified.
- expect((await lstat(dataDir())).isSymbolicLink()).toBe(true);
+ // `media` is ONE symlink now, `data/` is still a real directory, and each big
+ // file in it is a link into `media/` — config.json records the target,
+ // written only by the job, on success, after the copy verified.
+ expect((await lstat(mediaLink())).isSymbolicLink()).toBe(true);
+ expect(await readlink(mediaLink())).toBe(target);
+ expect((await lstat(dataDir())).isDirectory()).toBe(true);
+ expect((await lstat(dataDir())).isSymbolicLink()).toBe(false);
+ expect((await lstat(videoFile("audio.mp3"))).isSymbolicLink()).toBe(true);
expect(
- (
- await readJson<{ dataDir?: string }>(
- `test-transcripts/channels/${SLUG}/config.json`,
- )
- ).dataDir,
- ).toBe(target);
+ (await lstat(videoFile("transcript.live_chat.json"))).isSymbolicLink(),
+ ).toBe(true);
+ expect((await lstat(videoFile("transcript.en.vtt"))).isFile()).toBe(true);
+ expect(await existsAbs(join(target, VIDEO, "audio.mp3"))).toBe(true);
+ const config = await readJson<{ mediaDir?: string; dataDir?: string }>(
+ `test-transcripts/channels/${SLUG}/config.json`,
+ );
+ expect(config.mediaDir).toBe(target);
+ expect(config.dataDir).toBe(undefined);
// The source was reclaimed: no parked copy left holding a second copy of the
- // channel on the volume the move exists to free, and no marker.
+ // media on the volume the move exists to free, and no marker.
const siblings = await readdir(
resolvePath(`test-transcripts/channels/${SLUG}`),
);
- expect(siblings.filter((n) => n.startsWith("data."))).toEqual([]);
+ expect(siblings.filter((n) => n.startsWith("media."))).toEqual([]);
expect(
await pathExists(`test-transcripts/channels/${SLUG}/.relocating.json`),
).toBe(false);
// --- NOTHING ELSE NOTICED ----------------------------------------------
// The videos list is read off data/ through the same joined path it always
- // was, and resolves through the link.
- await page.goto(channelVideos(SLUG));
+ // was; the selected video lists the tiered audio beside the transcript.
+ await page.goto(channelVideos(SLUG, { video: VIDEO }));
await expect(page.getByText(VIDEO).first()).toBeVisible();
+ await expect(page.getByLabel("file name audio.mp3")).toBeVisible();
+ await expect(page.getByLabel("file name transcript.en.vtt")).toBeVisible();
// So does the per-file route, which resolves channelsDir/<slug>/data/<id>/
- // and has no idea any of this happened.
- const file = await page.request.get(
- `${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/transcript.en.vtt`,
- );
+ // and has no idea any of this happened: the transcript off the corpus disk,
+ // the audio streamed through its link onto the other drive.
+ const fileUrl = (name: string) =>
+ `${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/${name}`;
+ const file = await page.request.get(fileUrl("transcript.en.vtt"));
expect(file.status()).toBe(200);
expect(await file.text()).toContain("WEBVTT");
+ const audio = await page.request.get(fileUrl("audio.mp3"));
+ expect(audio.status()).toBe(200);
+ expect(audio.headers()["content-type"]).toBe("audio/mpeg");
+ expect((await audio.body()).length).toBe(AUDIO_BYTES);
+ const ranged = await page.request.get(fileUrl("audio.mp3"), {
+ headers: { range: "bytes=0-99" },
+ });
+ expect(ranged.status()).toBe(206);
+ expect((await ranged.body()).toString()).toBe(AUDIO.slice(0, 100));
// --- the badge ----------------------------------------------------------
await page.goto("/channels");
@@ -176,6 +243,34 @@ test("relocate a channel's media to another root, and move it back", async ({
page.getByLabel(/^media location: Media relocated/),
).toBeVisible();
+ // --- THE MEDIA DRIVE GOES AWAY; THE TEXT DOES NOT ------------------------
+ // Renamed away (the buildStats.test.ts trick): the `media` link dangles, as
+ // it does with the drive unmounted. The video page still reads — its text is
+ // on the corpus disk — the audio is listed with its size unknown, the file
+ // route answers 503 with a retry for it and 200 for the transcript, and the
+ // rack says the media is held.
+ const away = join(root, `${SLUG}.away`);
+ await rename(join(root, SLUG), away);
+ await forgetCaches();
+ await page.goto(`/channels/${SLUG}/videos/${VIDEO}`);
+ await expect(page.getByLabel("file name transcript.en.vtt")).toBeVisible();
+ await expect(page.getByLabel("file name audio.mp3")).toBeVisible();
+ await expect(page.getByText("media drive not reachable").first()).toBeVisible();
+ const text = await page.request.get(fileUrl("transcript.en.vtt"));
+ expect(text.status()).toBe(200);
+ const held = await page.request.get(fileUrl("audio.mp3"));
+ expect(held.status()).toBe(503);
+ expect(held.headers()["retry-after"]).toBe("15");
+ await page.goto("/channels");
+ const chip = page.getByLabel(`media hold for ${SLUG}`);
+ await expect(chip).toContainText("media held");
+ await page.goto(channelStage(SLUG, "storage"));
+ await expect(page.getByLabel("media hold", { exact: true })).toContainText(
+ "Its text stays readable",
+ );
+ await rename(away, join(root, SLUG));
+ await forgetCaches();
+
// --- back, WHICH IS A DESTINATION AND NOT A SECOND BUTTON ----------------
// "Move back in place" was its own section with its own button; it is now
// the one destination the select offers while the media is on a location,
@@ -200,19 +295,24 @@ test("relocate a channel's media to another root, and move it back", async ({
{ timeout: 60_000 },
);
- // A real directory again, the config field gone, and the target reclaimed.
- expect((await lstat(dataDir())).isDirectory()).toBe(true);
+ // `media/` a REAL directory on the corpus disk again, the config field gone,
+ // the target reclaimed — and the per-file link untouched, resolving here now.
+ expect((await lstat(mediaLink())).isDirectory()).toBe(true);
+ expect((await lstat(mediaLink())).isSymbolicLink()).toBe(false);
+ expect((await lstat(videoFile("audio.mp3"))).isSymbolicLink()).toBe(true);
expect(
(
- await readJson<{ dataDir?: string }>(
+ await readJson<{ mediaDir?: string }>(
`test-transcripts/channels/${SLUG}/config.json`,
)
- ).dataDir,
+ ).mediaDir,
).toBe(undefined);
- const back = await page.request.get(
- `${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/transcript.en.vtt`,
- );
+ expect(await existsAbs(target)).toBe(false);
+ const back = await page.request.get(fileUrl("transcript.en.vtt"));
expect(back.status()).toBe(200);
+ const backAudio = await page.request.get(fileUrl("audio.mp3"));
+ expect(backAudio.status()).toBe(200);
+ expect((await backAudio.body()).length).toBe(AUDIO_BYTES);
});
test("the Configure form shows the media location read-only", async ({
@@ -223,7 +323,7 @@ test("the Configure form shows the media location read-only", async ({
// exact: the Storage panel's badge is "media location: …", and a substring
// match would find either.
const line = page.getByLabel("media location", { exact: true });
- await expect(line).toHaveText("In the channel directory (data/)");
+ await expect(line).toHaveText("In the channel directory (media/)");
// Not an input: it is a record of what is on disk, and the only writer is a
// move that succeeded. A text box here would let config and disk disagree
// with a keystroke.
@@ -233,8 +333,8 @@ test("the Configure form shows the media location read-only", async ({
// THE BULK MOVE, from /channels, with the cold root coming out of Settings.
//
// Two channels are selected and exactly one moves. The other is a channel that
-// is ALREADY relocated, built directly on disk — an absolute `data` symlink plus
-// `config.dataDir`, which is precisely what a finished move leaves behind —
+// is ALREADY relocated, built directly on disk — an absolute `media` symlink
+// plus `config.mediaDir`, which is precisely what a finished move leaves behind —
// rather than by running a second relocation first. A real move here would cost
// a second rsync, a second job wait and a second 60s timeout to assert a skip
// that is decided before any byte is read; what is under test is the skip, and
@@ -280,16 +380,24 @@ test("the /channels bulk move queues one job per channel and skips the rest", as
});
const PRE = "pre-moved";
- const preTarget = join(root, PRE, "data");
- await mkdir(join(preTarget, "20240102_pre1234567"), { recursive: true });
+ const preTarget = join(root, PRE, "media");
+ const preVideo = "20240102_pre1234567";
+ await mkdir(join(preTarget, preVideo), { recursive: true });
+ await writeFile(join(preTarget, preVideo, "audio.mp3"), AUDIO);
+ const preData = resolvePath(`test-transcripts/channels/${PRE}/data/${preVideo}`);
+ await mkdir(preData, { recursive: true });
await writeFile(
- join(preTarget, "20240102_pre1234567", "transcript.en.vtt"),
+ join(preData, "transcript.en.vtt"),
"WEBVTT\n\n00:00.000 --> 00:01.000\nhello\n",
);
- await writeChannelConfig(PRE, { dataDir: preTarget });
+ await symlink(
+ join("..", "..", "media", preVideo, "audio.mp3"),
+ join(preData, "audio.mp3"),
+ );
+ await writeChannelConfig(PRE, { mediaDir: preTarget });
await symlink(
preTarget,
- resolvePath(`test-transcripts/channels/${PRE}/data`),
+ resolvePath(`test-transcripts/channels/${PRE}/media`),
);
await page.goto("/channels");
@@ -341,30 +449,31 @@ test("the /channels bulk move queues one job per channel and skips the rest", as
await expect(badges.first()).toHaveText("on Cold");
await expect(badges.nth(1)).toHaveText("on Cold");
- // The moved channel: config records the target under the tmp root, `data/` is
- // a link, and the videos list still lists the video through it.
+ // The moved channel: config records the target under the tmp root, `media` is
+ // a link, `data/` is still real, and the videos list lists the video.
expect(
(
- await readJson<{ dataDir?: string }>(
+ await readJson<{ mediaDir?: string }>(
`test-transcripts/channels/${SLUG}/config.json`,
)
- ).dataDir,
- ).toBe(join(root, SLUG, "data"));
- expect((await lstat(dataDir())).isSymbolicLink()).toBe(true);
+ ).mediaDir,
+ ).toBe(join(root, SLUG, "media"));
+ expect((await lstat(mediaLink())).isSymbolicLink()).toBe(true);
+ expect((await lstat(dataDir())).isSymbolicLink()).toBe(false);
await page.goto(channelVideos(SLUG));
await expect(page.getByText(VIDEO).first()).toBeVisible();
// The skipped channel was not touched: same target, still a link, no marker.
expect(
(
- await readJson<{ dataDir?: string }>(
+ await readJson<{ mediaDir?: string }>(
`test-transcripts/channels/${PRE}/config.json`,
)
- ).dataDir,
+ ).mediaDir,
).toBe(preTarget);
expect(
(
- await lstat(resolvePath(`test-transcripts/channels/${PRE}/data`))
+ await lstat(resolvePath(`test-transcripts/channels/${PRE}/media`))
).isSymbolicLink(),
).toBe(true);
expect(
@@ -514,7 +623,7 @@ test("the Storage panel moves to a location picked by name", async ({
await expect(destination).toContainText("Cold");
await destination.selectOption("cold");
- const target = join(cold, SLUG, "data");
+ const target = join(cold, SLUG, "media");
// EXACT. The panel has a second preview since the clip-window card joined it
// ("Preview eviction"), and getByRole's name match is a case-insensitive
// SUBSTRING by default — so the bare name resolves to two buttons and the
@@ -533,10 +642,10 @@ test("the Storage panel moves to a location picked by name", async ({
});
expect(
(
- await readJson<{ dataDir?: string }>(
+ await readJson<{ mediaDir?: string }>(
`test-transcripts/channels/${SLUG}/config.json`,
)
- ).dataDir,
+ ).mediaDir,
).toBe(target);
// AND THE BADGE READS THE NAME. `on Cold`, not sixty columns of absolute
@@ -566,14 +675,18 @@ test("Resume move finishes an interrupted move and clears its marker", async ({
test.setTimeout(90_000);
await resetData("one-youtube-channel-with-data");
const root = testInfo.outputPath("resume-root");
- const target = join(root, SLUG, "data");
+ const target = join(root, SLUG, "media");
+ // The channel was tiered in place by the move's preflight before it was
+ // killed: its two big files in `media/`, linked from `data/`.
+ await tierByHand(mediaLink(), {
+ "audio.mp3": AUDIO,
+ "transcript.live_chat.json": "[]",
+ });
// The partial copy: the video dir is there with ONE of its two files.
await mkdir(join(target, VIDEO), { recursive: true });
await copyFile(
- resolvePath(
- `test-transcripts/channels/${SLUG}/data/${VIDEO}/transcript.en.vtt`,
- ),
- join(target, VIDEO, "transcript.en.vtt"),
+ join(mediaLink(), VIDEO, "audio.mp3"),
+ join(target, VIDEO, "audio.mp3"),
);
// AND A STALE SCRATCH DIR the source no longer has — the 2026-10-01 case: a
// transcriber's `.audio.mp3.parakeet/` copied mid-transcription, then deleted
@@ -592,6 +705,7 @@ test("Resume move finishes an interrupted move and clears its marker", async ({
direction: "out",
startedAt: new Date().toISOString(),
phase: "copy",
+ scope: "media",
},
null,
2,
@@ -624,25 +738,29 @@ test("Resume move finishes an interrupted move and clears its marker", async ({
expect(
await pathExists(`test-transcripts/channels/${SLUG}/.relocating.json`),
).toBe(false);
- expect((await lstat(dataDir())).isSymbolicLink()).toBe(true);
+ expect((await lstat(mediaLink())).isSymbolicLink()).toBe(true);
expect(
(
- await readJson<{ dataDir?: string }>(
+ await readJson<{ mediaDir?: string }>(
`test-transcripts/channels/${SLUG}/config.json`,
)
- ).dataDir,
+ ).mediaDir,
).toBe(target);
- // And the bytes are readable through the same URL as ever.
+ // And the bytes are readable through the same URLs as ever.
const file = await page.request.get(
`${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/transcript.en.vtt`,
);
expect(file.status()).toBe(200);
+ const audio = await page.request.get(
+ `${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/audio.mp3`,
+ );
+ expect(audio.status()).toBe(200);
// The stale scratch dir is gone from the copy, and the copy is the source's
- // two files and nothing else.
+ // two media files and nothing else.
expect(await existsAbs(scratch)).toBe(false);
expect((await readdir(join(target, VIDEO))).sort()).toEqual([
- "metadata.info.json",
- "transcript.en.vtt",
+ "audio.mp3",
+ "transcript.live_chat.json",
]);
});
@@ -721,11 +839,13 @@ test("a move that starts while a job writes into the channel refuses, naming the
`Nothing has been touched.`,
);
- // NOTHING WAS TOUCHED: no marker, no copy, the media a real directory.
+ // NOTHING WAS TOUCHED: no marker, no copy, no `media` made, the text a real
+ // directory.
expect(
await pathExists(`test-transcripts/channels/${SLUG}/.relocating.json`),
).toBe(false);
expect(await existsAbs(join(root, SLUG))).toBe(false);
+ expect(await existsAbs(mediaLink())).toBe(false);
expect((await lstat(dataDir())).isDirectory()).toBe(true);
// And the panel says it before anyone clicks: the move is blocked, and the
@@ -814,7 +934,7 @@ test("the rack and the Storage panel show the hold while a marker stands, and it
await resetData("one-youtube-channel-with-data");
await generateReport(page, SLUG);
await quiet(page);
- const target = join(testInfo.outputPath("hold-root"), SLUG, "data");
+ const target = join(testInfo.outputPath("hold-root"), SLUG, "media");
await writeFile(
resolvePath(`test-transcripts/channels/${SLUG}/.relocating.json`),
JSON.stringify({
@@ -822,6 +942,7 @@ test("the rack and the Storage panel show the hold while a marker stands, and it
direction: "out",
startedAt: new Date().toISOString(),
phase: "copy",
+ scope: "media",
}),
);
await forgetCaches();
@@ -841,6 +962,8 @@ test("the rack and the Storage panel show the hold while a marker stands, and it
await expect(hold).toContainText(
"when the move completes, or its marker is cleared below",
);
+ // A MEDIA move holds the media only (release 17).
+ await expect(hold).toContainText("Its text stays readable");
// Abandoned: the marker goes, and the hold with it. The click is retried
// until the marker is gone: one that lands before hydration does nothing.
@@ -869,11 +992,16 @@ test("Reconcile and resume settles an extra and a changed file on the destinatio
test.setTimeout(90_000);
await resetData("one-youtube-channel-with-data");
const root = testInfo.outputPath("reconcile-root");
- const target = join(root, SLUG, "data");
+ const target = join(root, SLUG, "media");
+ // Tiered in place by the interrupted move's preflight.
+ await tierByHand(mediaLink(), {
+ "audio.mp3": AUDIO,
+ "transcript.live_chat.json": '[{"fresh":true}]',
+ });
await mkdir(join(target, VIDEO), { recursive: true });
// The full copy, timestamps and all, as a copy pass leaves it…
- for (const name of ["metadata.info.json", "transcript.en.vtt"]) {
- const src = join(dataDir(), VIDEO, name);
+ for (const name of ["audio.mp3", "transcript.live_chat.json"]) {
+ const src = join(mediaLink(), VIDEO, name);
await copyFile(src, join(target, VIDEO, name));
const { atime, mtime } = await stat(src);
await utimes(join(target, VIDEO, name), atime, mtime);
@@ -882,7 +1010,13 @@ test("Reconcile and resume settles an extra and a changed file on the destinatio
const scratch = join(target, VIDEO, ".audio.mp3.parakeet");
await mkdir(scratch, { recursive: true });
await writeFile(join(scratch, "meta.json"), "{}");
- await writeFile(join(target, VIDEO, "transcript.en.vtt"), "WEBVTT\n\nstale\n");
+ // A different LENGTH as well as different words: rsync's quick check
+ // compares size and a one-second mtime, and both files were written within
+ // the same second.
+ await writeFile(
+ join(target, VIDEO, "transcript.live_chat.json"),
+ '[{"stale":true,"from":"an older copy"}]',
+ );
await writeFile(
resolvePath(`test-transcripts/channels/${SLUG}/.relocating.json`),
JSON.stringify({
@@ -890,6 +1024,7 @@ test("Reconcile and resume settles an extra and a changed file on the destinatio
direction: "out",
startedAt: new Date().toISOString(),
phase: "copy",
+ scope: "media",
}),
);
@@ -907,20 +1042,22 @@ test("Reconcile and resume settles an extra and a changed file on the destinatio
await expect(out).toContainText(
/Reconciling: the destination copy differs from the source — \d+ extra on the destination \([^)]*\.audio\.mp3\.parakeet/,
);
- await expect(out).toContainText(/changed \([^)]*transcript\.en\.vtt/);
+ await expect(out).toContainText(
+ /changed \([^)]*transcript\.live_chat\.json/,
+ );
// Finished: link, config, no marker; the copy is the source's two files.
expect(
await pathExists(`test-transcripts/channels/${SLUG}/.relocating.json`),
).toBe(false);
- expect((await lstat(dataDir())).isSymbolicLink()).toBe(true);
+ expect((await lstat(mediaLink())).isSymbolicLink()).toBe(true);
expect(await existsAbs(scratch)).toBe(false);
expect((await readdir(join(target, VIDEO))).sort()).toEqual([
- "metadata.info.json",
- "transcript.en.vtt",
+ "audio.mp3",
+ "transcript.live_chat.json",
]);
const file = await page.request.get(
- `${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/transcript.en.vtt`,
+ `${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/transcript.live_chat.json`,
);
expect(file.status()).toBe(200);
expect(await file.text()).not.toContain("stale");
@@ -986,8 +1123,10 @@ test("a move to an unmounted root refuses before it creates anything", async ({
// NOTHING WAS CREATED on the way to finding out — that IS the bug.
expect(await pathExists(root)).toBe(false);
expect(await pathExists(join(root, SLUG))).toBe(false);
- // The media is still a real directory in the corpus, unmoved.
+ // The channel is untouched in the corpus — not even tiered: the root is
+ // refused before the preview's tiering runs.
expect((await lstat(dataDir())).isDirectory()).toBe(true);
+ expect(await existsAbs(mediaLink())).toBe(false);
});
// EVICTING THIS CHANNEL'S FETCHED CLIP WINDOWS, from the panel that already
@@ -1094,13 +1233,13 @@ test("Sync all skips a channel whose media drive is not mounted", async ({
test.setTimeout(90_000);
await resetData("one-youtube-channel-with-data");
await writeSettings({ minFreeDiskGB: 0 });
- // A relocation whose drive went away: the link and the config agree with each
- // other and with nothing on disk. Deliberately NOT created — an unmounted
- // mountpoint whose parent is missing too is the honest version.
- const target = join(testInfo.outputPath("never-mounted"), SLUG, "data");
- await rm(dataDir(), { recursive: true, force: true });
- await symlink(target, dataDir());
- await writeChannelConfig(SLUG, { dataDir: target });
+ // A relocation whose drive went away: the `media` link and the config agree
+ // with each other and with nothing on disk. Deliberately NOT created — an
+ // unmounted mountpoint whose parent is missing too is the honest version.
+ // The text stays in `data/` on the corpus disk, as it always does now.
+ const target = join(testInfo.outputPath("never-mounted"), SLUG, "media");
+ await symlink(target, mediaLink());
+ await writeChannelConfig(SLUG, { mediaDir: target });
await page.goto("/channels");
await page.getByRole("button", { name: "sync every channel" }).click();
@@ -1115,3 +1254,104 @@ test("Sync all skips a channel whose media drive is not mounted", async ({
new RegExp(`${SLUG}: media unreachable:.*drive not mounted`),
);
});
+
+// THE RETIRED LAYOUT (release 17). A channel whose whole `data/` was moved
+// before the media tier — an absolute `data` link and `config.dataDir`, the old
+// mover's work — is `legacy`: its text is on the far drive too, so it is held
+// by both guards until `archilyzer storage migrate-tier` brings the text home.
+// The row says "Media layout retired", the Storage panel refuses to move it
+// with the same sentence, and a media job refused on it names the command.
+test("a legacy channel reads Media layout retired, and a media job's refusal names migrate-tier", async ({
+ page,
+}, testInfo) => {
+ test.setTimeout(90_000);
+ await resetData("one-youtube-channel-with-data");
+ await writeSettings({ minFreeDiskGB: 0 });
+ const target = join(testInfo.outputPath("legacy-root"), SLUG, "data");
+ await mkdir(join(target, VIDEO), { recursive: true });
+ for (const name of await readdir(join(dataDir(), VIDEO))) {
+ await copyFile(join(dataDir(), VIDEO, name), join(target, VIDEO, name));
+ }
+ await rm(dataDir(), { recursive: true, force: true });
+ await symlink(target, dataDir());
+ await writeChannelConfig(SLUG, { dataDir: target });
+ await forgetCaches();
+
+ await page.goto("/channels");
+ await expect(
+ page.getByLabel(/^media location: Media layout retired/),
+ ).toBeVisible();
+
+ await page.goto(channelStage(SLUG, "storage"));
+ await expect(page.getByLabel("move back refused")).toContainText(
+ `run archilyzer storage migrate-tier ${SLUG}`,
+ );
+
+ await page.goto("/channels");
+ await page.getByRole("button", { name: "sync every channel" }).click();
+ const result = page.getByLabel("sync all result");
+ await expect(result).toContainText(/Queued 0 . skipped 1/, {
+ timeout: 15_000,
+ });
+ await expect(result).toHaveAttribute(
+ "title",
+ new RegExp(`${SLUG}: .*archilyzer storage migrate-tier ${SLUG}`),
+ );
+});
+
+// THE TEXT OUTLIVES ITS MEDIA DRIVE (release 17): a digest reads and writes
+// the text tier, so it runs on a channel whose media is relocated onto a drive
+// that is not there — the job is asked the text guard, not the media one, and
+// the digest lane makes the same decision (autoRunner.ts isChannelHeldForLane,
+// pinned in its unit tests). The ollama stub answers, as in digest.spec.ts.
+test("a digest runs on a channel whose media drive is away", async ({
+ page,
+}, testInfo) => {
+ test.setTimeout(120_000);
+ const CHANNEL = "digest-channel";
+ const DIGEST_VIDEO = "digestvid0001";
+ await resetData(null);
+ await writeSettings({
+ adminTitle: "Test Admin",
+ maxTranscriptPageBytes: 8388608,
+ sleepBetweenDownloadsSeconds: 0,
+ minFreeDiskGB: 0,
+ digest: {
+ localAppId: "ollama-direct",
+ remoteAppId: "claude-code",
+ sections: ["chapters"],
+ },
+ });
+ // Relocated media, renamed away: the `media` link dangles; `data/` is real.
+ const target = join(testInfo.outputPath("away-root"), CHANNEL, "media");
+ await writeChannelConfig(CHANNEL, { mediaDir: target });
+ await writeDigestVideo({ channelSlug: CHANNEL, videoId: DIGEST_VIDEO });
+ await symlink(
+ target,
+ resolvePath(`test-transcripts/channels/${CHANNEL}/media`),
+ );
+ await symlink(
+ join("..", "..", "media", DIGEST_VIDEO, "audio.mp3"),
+ resolvePath(
+ `test-transcripts/channels/${CHANNEL}/data/${DIGEST_VIDEO}/audio.mp3`,
+ ),
+ );
+ await forgetCaches();
+
+ await generateReport(page, CHANNEL);
+ await page.goto("/channels");
+ await expect(page.getByLabel(`media hold for ${CHANNEL}`)).toContainText(
+ "media held",
+ );
+ await page.goto(channelStage(CHANNEL, "digest"));
+ await page.getByRole("button", { name: "Digest channel" }).click();
+ await expect(page.getByLabel("Digest channel output")).toContainText(
+ "Digest batch:",
+ { timeout: 60_000 },
+ );
+ expect(
+ await pathExists(
+ `test-transcripts/channels/${CHANNEL}/data/${DIGEST_VIDEO}/ai-digest.json`,
+ ),
+ ).toBe(true);
+});
diff --git a/editor/e2e/channels-storage-columns.spec.ts b/editor/e2e/channels-storage-columns.spec.ts
@@ -1,4 +1,4 @@
-import { mkdir, rename, symlink } from "node:fs/promises";
+import { mkdir, readdir, rename, symlink, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { test, expect } from "@playwright/test";
import { baseUrl } from "./baseUrl";
@@ -32,21 +32,47 @@ import {
// 549d).
const SLUG = "test-youtube";
+const VIDEO = "20240101_test1234567";
+
+// THE SIZE COLUMN IS THE MEDIA TIER (release 17): the audio and the raw live
+// chat, what a move carries. The fixture has none, so one is written.
+async function giveAudio(slug: string, id: string, bytes = 4096) {
+ await writeFile(
+ resolvePath(`test-transcripts/channels/${slug}/data/${id}/audio.mp3`),
+ "x".repeat(bytes),
+ );
+}
-// Put a channel's media on `root` the way a finished relocation leaves it: the
-// real directory on the "drive", an absolute symlink at channels/<slug>/data,
-// and config.dataDir naming the target. Built directly rather than by running a
-// move — what is under test is the COLUMN, and a real rsync buys nothing here.
+// Put a channel's media tier on `root` the way a finished relocation leaves it
+// (release 17): each audio file moved to `<root>/<slug>/media/<id>/` with its
+// relative link left in `data/<id>/`, an absolute symlink at
+// channels/<slug>/media, and config.mediaDir naming the target. Built directly
+// rather than by running a move — what is under test is the COLUMN, and a real
+// rsync buys nothing here.
async function relocateOnDisk(slug: string, root: string): Promise<string> {
const channelDir = resolvePath(`test-transcripts/channels/${slug}`);
- const target = join(root, slug, "data");
- await mkdir(join(root, slug), { recursive: true });
- await rename(join(channelDir, "data"), target);
- await symlink(target, join(channelDir, "data"));
+ const target = join(root, slug, "media");
+ await mkdir(target, { recursive: true });
+ for (const id of await readdir(join(channelDir, "data"))) {
+ const names = await readdir(join(channelDir, "data", id)).catch(
+ () => [] as string[],
+ );
+ if (!names.includes("audio.mp3")) continue;
+ await mkdir(join(target, id), { recursive: true });
+ await rename(
+ join(channelDir, "data", id, "audio.mp3"),
+ join(target, id, "audio.mp3"),
+ );
+ await symlink(
+ join("..", "..", "media", id, "audio.mp3"),
+ join(channelDir, "data", id, "audio.mp3"),
+ );
+ }
+ await symlink(target, join(channelDir, "media"));
const config = await readJson<Record<string, unknown>>(
`test-transcripts/channels/${slug}/config.json`,
);
- await writeChannelConfig(slug, { ...config, dataDir: target });
+ await writeChannelConfig(slug, { ...config, mediaDir: target });
await fetch(`${baseUrl}/api/test/invalidate-cache`).catch(() => {});
return target;
}
@@ -66,6 +92,7 @@ test("the Location and Size columns say which disk and how much", async ({
defaultLocationId: "cold",
},
});
+ await giveAudio(SLUG, VIDEO);
await generateReport(page, SLUG);
await page.goto("/channels");
@@ -73,8 +100,8 @@ test("the Location and Size columns say which disk and how much", async ({
// is on the corpus volume, and that is a fact about it, not an absence.
const location = page.getByLabel(`media location for ${SLUG}`);
await expect(location).toContainText("Internal");
- // The fixture is two small files, so the figure is not zero and not "—": the
- // report measured it.
+ // The fixture's audio is a small file, so the figure is not zero and not
+ // "—": the report measured it.
const size = page.getByLabel(`media size for ${SLUG}`);
await expect(size).not.toHaveText("—");
await expect(size).toContainText(/B|KB|MB/);
@@ -131,6 +158,8 @@ test("filter by volume, sort by size, and free up N GB", async ({
resolvePath(`test-transcripts/channels/${SECOND}/data/20240102_second0001`),
{ recursive: true },
);
+ await giveAudio(SLUG, VIDEO);
+ await giveAudio(SECOND, "20240102_second0001", 1024);
await writeSettings({
adminTitle: "Test Admin",
minFreeDiskGB: 0,
diff --git a/editor/e2e/storage-locations.spec.ts b/editor/e2e/storage-locations.spec.ts
@@ -26,8 +26,10 @@ import {
// back at a different mountpoint, ONE button makes every channel on it readable
// again, and it does so WITHOUT MOVING A BYTE. So the assertions after the
// re-point are deliberately not about the re-point — they are `readlink`,
-// `config.dataDir` and the transcript route serving the same URL it always did,
-// with the media now reached through a different path entirely.
+// `config.mediaDir` and the file route serving the same URLs it always did,
+// with the media now reached through a different path entirely. (Release 17:
+// what lives on a location is a channel's media tier, `channels/<slug>/media`;
+// its text never leaves `data/` on the corpus disk.)
//
// THE DISK IS A FAKE BINARY. `FINDMNT_BIN` and `UDISKSCTL_BIN` point at
// `e2e/fixtures/bin/fake-{findmnt,udisksctl}.mjs` (see editor/package.json), and
@@ -41,7 +43,7 @@ const VIDEO = "20240101_test1234567";
const UUID = "e2e-platter-uuid";
const channelDir = () => resolvePath(`test-transcripts/channels/${SLUG}`);
-const dataLink = () => join(channelDir(), "data");
+const mediaLink = () => join(channelDir(), "media");
const controlFile = () => resolvePath("test-transcripts/.fake-findmnt.json");
async function writeControl(control: {
@@ -53,22 +55,27 @@ async function writeControl(control: {
await fetch(`${baseUrl}/api/test/invalidate-cache`).catch(() => {});
}
-// Put the fixture channel's media on `root`, the way a finished relocation
-// leaves it: the real directory on the "drive", an absolute symlink at
-// channels/<slug>/data, and config.dataDir naming the target. Built directly
+// Put the fixture channel's media tier on `root`, the way a finished
+// relocation leaves it: the audio in `<root>/<slug>/media/<id>/` on the
+// "drive", its relative link in `data/<id>/`, an absolute symlink at
+// channels/<slug>/media, and config.mediaDir naming the target. Built directly
// rather than by running a move — what is under test here is the re-point, and
// a real rsync first would buy nothing but a minute.
async function relocateOnDisk(root: string): Promise<string> {
- const target = join(root, SLUG, "data");
- await mkdir(join(root, SLUG), { recursive: true });
- await rename(dataLink(), target);
- await symlink(target, dataLink());
+ const target = join(root, SLUG, "media");
+ await mkdir(join(target, VIDEO), { recursive: true });
+ await writeFile(join(target, VIDEO, "audio.mp3"), "ID3audio");
+ await symlink(
+ join("..", "..", "media", VIDEO, "audio.mp3"),
+ join(channelDir(), "data", VIDEO, "audio.mp3"),
+ );
+ await symlink(target, mediaLink());
const config = await readJson<Record<string, unknown>>(
`test-transcripts/channels/${SLUG}/config.json`,
);
await writeFile(
join(channelDir(), "config.json"),
- JSON.stringify({ ...config, dataDir: target }, null, 2) + "\n",
+ JSON.stringify({ ...config, mediaDir: target }, null, 2) + "\n",
);
await fetch(`${baseUrl}/api/test/invalidate-cache`).catch(() => {});
return target;
@@ -138,7 +145,7 @@ test("the page lists a location with its status, counts and refusals", async ({
// DELETE IS REFUSED WHILE ANYBODY LIVES THERE. Deleting the location moves no
// bytes — which is exactly why it is refused: it would erase the only record
- // of which disk that channel's absolute dataDir belongs to.
+ // of which disk that channel's absolute mediaDir belongs to.
await expect(cold.getByLabel("delete cold")).toBeDisabled();
await expect(cold.getByLabel("delete withheld")).toContainText(
"1 channel(s) still have their media under",
@@ -223,7 +230,7 @@ test("a volume that came up somewhere else is re-pointed in one click", async ({
// --- the disk comes back somewhere else ---------------------------------
// The media moves with it, because it IS the disk. Nothing in the corpus
- // changes: config.dataDir and the symlink still name the old mountpoint,
+ // changes: config.mediaDir and the symlink still name the old mountpoint,
// which is precisely the breakage this page exists for.
await rename(join(rootA, SLUG), join(rootB, SLUG));
await rm(rootA, { recursive: true, force: true });
@@ -235,7 +242,7 @@ test("a volume that came up somewhere else is re-pointed in one click", async ({
// is intact and its target is gone, which is the whole of the breakage. A
// stat here follows the link and reports "no data/ at all", which is the
// reading this module exists to stop anybody making.
- expect(await readlink(dataLink())).toBe(targetA);
+ expect(await readlink(mediaLink())).toBe(targetA);
await page.goto("/storage");
// The probe memo is ten seconds wide and keyed by id+root, neither of which
@@ -261,17 +268,17 @@ test("a volume that came up somewhere else is re-pointed in one click", async ({
});
// The link and the config now name the new mountpoint...
- const targetB = join(rootB, SLUG, "data");
+ const targetB = join(rootB, SLUG, "media");
expect(targetA).not.toBe(targetB);
await expect
- .poll(async () => readlink(dataLink()), { timeout: 30_000 })
+ .poll(async () => readlink(mediaLink()), { timeout: 30_000 })
.toBe(targetB);
expect(
(
- await readJson<{ dataDir?: string }>(
+ await readJson<{ mediaDir?: string }>(
`test-transcripts/channels/${SLUG}/config.json`,
)
- ).dataDir,
+ ).mediaDir,
).toBe(targetB);
// ...and so does the location.
const settings = await readJson<{
@@ -282,12 +289,19 @@ test("a volume that came up somewhere else is re-pointed in one click", async ({
);
// NOTHING ELSE NOTICED. The per-file route joins channelsDir/<slug>/data/<id>
- // exactly as it always has and has no idea any of this happened.
+ // exactly as it always has and has no idea any of this happened: the
+ // transcript off the corpus disk, the audio through its link and the
+ // re-pointed `media` onto the drive at its new mountpoint.
const file = await page.request.get(
`${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/transcript.en.vtt`,
);
expect(file.status()).toBe(200);
expect(await file.text()).toContain("WEBVTT");
+ const audio = await page.request.get(
+ `${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/audio.mp3`,
+ );
+ expect(audio.status()).toBe(200);
+ expect(await audio.text()).toBe("ID3audio");
// And the page agrees: the channel is reachable again.
await page.goto("/storage");
diff --git a/plans/FACTS.md b/plans/FACTS.md
@@ -3917,6 +3917,72 @@ as they were measured.
`channelTextStall` on the retired `dataDir` only. `readChannelStat` and the recency tail reads pass
a drive only for a legacy channel. `rootOfUnknownPath` strips `<root>/<slug>/media` and `/data`.
+
+### Release 17 slice T2 — the mover over `media/`, and the surfaces (2026-10-02)
+
+Supersedes, where they differ, the mover and storage-location facts above (the whole-`data/` move,
+`relocatedDataDir`, `config.dataDir` as the location key).
+- **The mover's unit is `channels/<slug>/media`** (`controller/relocateChannelMedia.ts`). Out: the
+ job's first step asks the writers, then a classic channel is tiered in place
+ (`tierChannelMedia(paths, slug, { createMediaDir: true })`, only in the `copy` phase — a resumed run
+ tiers nothing, its marker stands), then `media/` is measured, copied to `<root>/<slug>/media`
+ (`relocatedMediaDir`), mirrored and verified, renamed to `media.relocated-<ts>`, replaced by ONE
+ absolute link, and `config.mediaDir` written. Back: `<root>/<slug>/media` → `media.incoming` → the
+ link swapped for the real dir, `mediaDir` unset; the per-file links are never rewritten. The marker
+ carries `scope: "media"` (and `readDirMarker` now keeps a `scope` it reads). Refused before anything:
+ a `legacy` channel (a `data` link or `config.dataDir`, the sentence names `archilyzer storage
+ migrate-tier <slug>`) and a marker with `scope: "tier-migration"`. `rootOfRelocatedMediaDir`
+ replaces `rootOfRelocatedDataDir`; `relocatedDataDir` (`lib/channelMedia.ts`) is kept only as the
+ retired shape for the migration and the tests. The containment check resolves the deepest EXISTING
+ ancestor of `<root>/<slug>/media` (it usually does not exist yet), so a `<slug>` level linked back
+ into the channel dir is still refused before the preview tiers anything.
+- **The preview tiers too** — an in-place channel with a `data/`, no marker — and reports
+ `tieredFirst`; a second preview tiers nothing. The result carries `tiered`, and the job's done line
+ says "(N file(s) tiered first)".
+- **A move is held by the channel's MEDIA writers only.** `channelMediaWriters(slug, opts)`
+ (`relocateChannelMedia.ts`) = `channelWriters` minus every job whose kind is not `needsMedia` —
+ except `relocate-channel-media` itself (its kind is not `needsMedia`, and must not be, but it is the
+ thing writing into `media/`) — minus the digest lane's units. The job's first step (ignoring its own
+ kind), the third ask under the marker, the preview, and the editor's
+ `channelMediaBusyReason(slug, what, { mediaOnly: true })` for Move, Move back, Resume, Reconcile,
+ Clear marker, the bulk move and the panel's blocked line ask it. Rename and delete still ask every
+ writer.
+- **Location membership keys on `config.mediaDir`, or a legacy channel's retired `dataDir`**:
+ `channelsOnLocation` (with `textBytes`, `unknownTextBytes` and a `legacy` count inside
+ `unreachable`), `channelVolumeOf(config, locations)` (now takes the config), the storage watch (one
+ auto-pause per channel, as before), `deleteChannel` (removes both targets). The re-point rewrites the
+ `media` link and `mediaDir`, and a legacy channel on the same location the RETIRED way — its `data`
+ link and `dataDir` to `relocatedDataDir(newRoot, slug)`, in the same ledger and rollback
+ (`RepointPreflight.legacy`), so `dataDir` is always `<root>/<slug>/data` for the tier migration. The
+ rename moves a convention-shaped `<root>/<slug>/media` and re-points the `media` link; it REFUSES a
+ legacy channel (`migrate-tier`), for the same invariant. `tierMigrationRefusal` (exported from the
+ mover) is the one sentence the job, the preview, Resume/Reconcile and Clear marker refuse a
+ `scope: "tier-migration"` marker with.
+- **`/storage` bytes.** A location row's figure is its channels' `totalMediaBytes` (the media tier),
+ with no clips (`clipsBytes` 0 there); the internal row is in-place media + every channel's
+ `totalTextBytes` + `totalClipsBytes` (`internal.corpus`, folded in `editor/app/storage/buildStorage.ts`;
+ a report with no text figure counts unmeasured, once per channel; a legacy channel is left out of the
+ sums and named "legacy (n to migrate)") with the breakdown in `tiersText`
+ (`aria-label="location tier bytes"`). `channelsText` adds "(n to migrate)".
+- **The two video pages read the text on the corpus disk.** They gate on `channelTextStall` (a legacy
+ channel only), not `channelMediaStall`, and list a directory through ONE loader,
+ `loadVideoDirFiles(videoDir, mediaDir)` (`editor/app/channels/[slug]/lib/videoDirFiles.ts`): links
+ are listed (a dirent `isFile()` hid them), a link's time is its `lstat` (the hook copies the file's
+ times onto it), and the media links' sizes are asked of the media drive in ONE `onDrive(mediaDir)`
+ call; unreachable → `VideoFile.size: null`, rendered "— (media drive not reachable)".
+- **The file route picks the drive by where the file is**: a link (`lstat`) with an `isTierable` name
+ → `mediaDir`; anything else, `source-media.*` included, the corpus disk (a legacy channel's retired
+ `dataDir` for both). It answers **503 with `retry-after: 15`** when the stat of such a link fails AND
+ the channel has `mediaDir` — a tiered file whose drive is away; a dangling link on an in-place
+ `media/` is a missing file, 404. `deleteVideoFileAction` refuses the first (unlinking alone would
+ orphan the bytes) and removes the second.
+- **The live-chat archive (export build) reads no raw replay while the channel's media is not
+ `ok`/`in-place`**: it stages the cues on disk as they are (`isLiveChatCuesFresh` `lstat`s the link)
+ and logs how many are older than their replay and how many videos with a raw replay and no cues it
+ left out; it never drops the channel.
+- **`noCorpusWalkInRenderPaths.test.ts` greps render-path files for the walkers' NAMES**, comments
+ included: naming `measureTree` in a comment of a file a page imports fails it.
+
## Channel priority (verified 2026-09-11) — one tier per channel, four compiled trees
Branch `channel-priority/s5`, off S0's `28bfee3`, merging `s1`–`s4` and closing the twelve
diff --git a/plans/release-17.md b/plans/release-17.md
@@ -1710,4 +1710,207 @@ no-subs-fallback.spec rate-limit.spec` (`$T/RL-specs-merge.txt`): run 10 (`RL-e2
- An H1 failure records `ytdlpExitCode: 0` with an `error` (the one attempt that does; the job log
line says why).
+### Slice T2, as shipped — the mover over `media/`, and the surfaces (2026-10-02)
+
+Branch `r17/media-tier-mover` off `main` `a395aaa1` (U1, XP, D0 and T1 merged), worktree
+`~/Projects/r12-paths-fix` (editor 5001, test 5011, export 5010), one Opus implementer, beside U2 and RL.
+Scratch files `T2-*` in the job's `tmp`. The plan is "The model (A′)" §5 and §6, the `loadVideoDir` change
+in §2, the `onDrive`-by-file-kind editor changes in §4, and the T2 row; T1's "left for T2" items are below.
+
+**What it does.**
+- **The mover carries `channels/<slug>/media`, never `data/`** (`relocateChannelMedia.ts`). Out: the job's
+ first step asks the writers, then a classic channel is tiered in place (`tierChannelMedia(paths, slug,
+ { createMediaDir: true })` — same-disk renames and links; only in the copy phase: a resumed run's marker
+ stands and the hook writes nothing under one), then `media/` is measured, copied to
+ `<root>/<slug>/media`, mirrored, verified, parked as `media.relocated-<ts>`, replaced by ONE absolute
+ link, and `config.mediaDir` written. Back: the target → `media.incoming` → the link swapped for a real
+ `media/` on the corpus disk, `mediaDir` unset, the target reclaimed; not one per-file link is touched in
+ either direction. The marker keeps its shape plus `scope: "media"` (`readDirMarker` now keeps a `scope`
+ it reads, so a resume rewrites what it found). The space check is `media/`'s bytes plus the margin.
+ Refused with nothing touched: a `legacy` channel (`… cannot be moved: its media layout is the retired
+ whole-directory one — run archilyzer storage migrate-tier <slug>.`), out, back and in the preview; a
+ marker whose `scope` is `tier-migration`. `relocationRootProblem`, `assertRelocationRootPresent` and
+ `rootOfRelocatedMediaDir` (renamed from `…DataDir`) work on `<root>/<slug>/media`; `relocateDir.ts`'s
+ names follow. The result carries `tiered`; the job's done line says "(N file(s) tiered first)".
+- **The preview tiers an in-place classic channel too** (with a `data/`, no marker, no `mediaDir`) and
+ reports `tieredFirst`, shown as "N file(s) tiered first"; a second preview tiers nothing.
+- **A move is held by the channel's media writers only.** `channelMediaWriters` (new, in the mover) is
+ `channelWriters` minus every job whose kind is not `needsMedia` — except `relocate-channel-media`, the
+ move itself — and minus the digest lane's units. The job's first step and the ask under the marker, the
+ preview, and the editor's `channelMediaBusyReason(slug, what, { mediaOnly: true })` for Move, Move
+ back, Resume, Reconcile, Clear marker, the bulk move and the panel's blocked line ask it; rename and
+ delete still ask every writer.
+- **Locations key on `mediaDir`** (a legacy channel on its retired `dataDir`): `channelsOnLocation`
+ (new `textBytes`, `unknownTextBytes`, `legacy`), `channelVolumeOf(config, …)`, the storage watch (one
+ auto-pause per channel, as before), the re-point (rewrites the `media` link and `mediaDir`; since the
+ review a legacy channel on the same location is re-pointed the retired way — see "Review"), the
+ rename (a convention-shaped `<root>/<slug>/media` moves and the `media` link is re-pointed; the
+ per-file links are relative and move with the channel dir; since the review a legacy channel's rename
+ is refused), `deleteChannel` (removes `mediaDir`, and a legacy channel's `dataDir`).
+- **`/storage`**: a location row's figure is its channels' media tier, with no clips; the internal row is
+ in-place media + every channel's text + every channel's clip windows (`internal.corpus`, folded in
+ `buildStorage.ts`), with the breakdown "text N + clips N on the corpus volume, plus N media of in-place
+ channels" (`aria-label="location tier bytes"`); a report with no `totalTextBytes` counts unmeasured,
+ once per channel; `legacy` counts as unreachable with "(n to migrate)". The page's and the re-point's
+ prose name `mediaDir`. SETTINGS.md regenerated (one field doc names `config.mediaDir`).
+- **The Storage panel**: two rows under one heading — `media path` (the target, or `<channel>/media (in
+ place)`) with `media bytes` (`totalMediaBytes`, the tier), and `text path` with `text bytes`
+ (`totalTextBytes`); `free on media volume` kept; the buttons keep their names; the hold sentence says
+ "Its text stays readable: the video pages, the index and the digests go on." (or that the text is held
+ too, when it is); the stale-marker paragraph says the media lanes skip the channel. A legacy channel's
+ Move back is refused with the migrate-tier sentence (`move back refused`). Resume refuses a
+ tier-migration marker. The badge's `legacy` entry is "Media layout retired" (danger; T1 added it for
+ tsc). The Configure form's read-only line shows `mediaDir`, a legacy `dataDir` with the command, or "In
+ the channel directory (media/)".
+- **The two video pages read the text on the corpus disk**: they gate on `channelTextStall` (a legacy
+ channel only), and list a directory through one loader, `loadVideoDirFiles(videoDir, mediaDir)`
+ (`editor/app/channels/[slug]/lib/videoDirFiles.ts`, replacing both pages' `isFile()` loaders): links are
+ listed, a link's time is its `lstat`, the media links' sizes come from ONE `onDrive(mediaDir)` call, and
+ an unreachable one is `size: null` ("— (media drive not reachable)"). The file route chooses the drive
+ by `classifyEntry(name)` and answers **503 with `retry-after: 15`** for a link whose target is not
+ there. A file delete refuses a tiered file whose drive is away (removing the link alone would orphan the
+ bytes).
+
+**T1's "left for T2", answered.**
+- `loadVideoDir` and `videos/page.tsx` filtered `isFile()`: replaced by `loadVideoDirFiles` (above).
+- The `/storage` rollups counted clips inside media bytes: a location row has none now; the internal row
+ holds every channel's clips and text.
+- The mover writes `scope: "media"` on every marker.
+- `relocatedDataDir` stays exported, documented as the retired shape (the migration and the tests build
+ it); nothing moves a channel to it.
+- The hook's marker re-check before the name changes: verified in `mediaTier-server.ts` (`markerStands`
+ is asked again after the bytes land; a move that began meanwhile leaves the file real). Corpus-wide
+ callers carry no slug, so the hook asks the marker beside the `media` it is writing into — as T1
+ shipped it; nothing to add.
+- `archiveLiveChat` read a stale raw replay with no channel guard. Ruling taken here: the build asks the
+ channel's media once (`inspectChannelMedia`, fresh); while it is not `ok`/`in-place` it reads no raw
+ replay and stages the cues already on disk (the freshness check `lstat`s the link), logging how many
+ are older than their replay; a video with no cues yet is left out of that build. Not a refusal: a
+ build never fails over a media drive.
+
+**Small edits outside the T2 row** (each needed by the surfaces above): `common/lib/channelMedia.ts`
+(the `relocatedDataDir` comment), `common/lib/mediaTier-server.ts` and `common/lib/savedVideoStore.ts`
+(comments naming the retired shape), `common/jobs/jobKinds.ts` (one comment), `common/lib/
+storageLocations.ts` (comments and one field doc), `editor/app/channels/lib/mediaBusy.ts` (`mediaOnly`),
+`editor/app/channels/[slug]/videos/[id]/components/cards/{videoFiles.ts,FilesList.tsx}` (`size: null`),
+`editor/app/channels/components/{ChannelForm.tsx,ChannelVolumeBar.tsx}`, `editor/app/{page.tsx,channels/
+page.tsx}` (`channelVolumeOf(config)`), `editor/app/storage/{page.tsx,actions.ts,components/
+StorageLocationsTable.tsx}` (prose, the tier line), `common/controller/archiveLiveChat.ts` (T1's item 4).
+No helper was added to T1's `mediaTier-server.ts` or `channelMedia.ts`.
+
+**Deviations from the plan** (one sentence each):
+1. `channelMediaWriters` lives in the mover rather than as T1's `channelWriters(…, { mediaOnly })` alone:
+ `relocate-channel-media` is not `needsMedia`, so `mediaOnly` dropped a running move and the panel
+ would have offered Clear marker, and the preview a second move, over it.
+2. The preview tiers only an in-place channel (no `mediaDir`): a relocated channel's tier is on the far
+ drive, and a preview copies nothing there.
+3. Locations, the volume column, the watch and `deleteChannel` place a legacy channel by its retired
+ `dataDir`, so `/storage` can count it "(n to migrate)" on the drive it is actually on.
+4. The containment check resolves the deepest EXISTING ancestor of the target (it used `realpath` or a
+ lexical path): `<root>/<slug>/media` usually does not exist yet, so a `<slug>` level linked back into
+ the channel dir was caught only by the mirror's direction check, after the preflight had tiered.
+5. "The digest lane still picks the channel" is pinned in e2e as a digest JOB that runs on a channel
+ whose media drive is away (the lane's decision is T1's `isChannelHeldForLane`, pinned in its unit
+ tests): the fixture has no digest-lane work without a policy and a model.
+6. Two new e2e cases rather than more steps in the main one: the legacy channel (badge, panel refusal,
+ Sync all's skip naming `migrate-tier`) and the digest.
+
+**Commits**
+
+| Commit | What |
+|---|---|
+| `6407b697` | `common:` the mover over `media/` (tier first, `media.*` parked names, `scope`, `mediaDir`, legacy and tier-migration refused), `relocateDir.ts` names, the re-point, rename, `deleteChannel`, the watch and the rollups on `mediaDir`, `/storage`'s tiers and "(n to migrate)", `channelVolumeOf(config)`; the 28 T1 skips rebased |
+| `c93048a7` | `editor:` the Storage panel's two rows, the badge, Resume/bulk/job wording, `loadVideoDirFiles`, the file route's drive by kind and its 503, the guarded file delete, the Configure line |
+| `438911c5` | `common:` the live-chat archive reads no raw replay while the media is away |
+| `e4cab18e` | `common, editor:` `channelMediaWriters` — a move is held by the media writers only; the `/storage` prose |
+| `f93caa69` | `editor:` e2e rebased on the media tier, two new cases |
+| `8d9d11f9` | `editor:` the loader's comment names no corpus walker (`noCorpusWalkInRenderPaths`) |
+| `12339ae2` | `common, editor:` the re-point refuses a legacy channel naming `migrate-tier`; the reconcile spec's changed file differs in length |
+| `46f6b51e` | merge `main` `0a62bf74` (U2, the deck's finale) — clean |
+| this commit | `plans:` this section, FACTS "Release 17 slice T2", the editor changelog |
+
+#### Gates (logs `$T/T2-*.log`)
+
+- **tsc** (all workspaces) clean at every commit and on the merged tree.
+- **common:** before the merge **2,608 passed, 1 failed, 0 skipped** (2,609) — the failure was this slice's
+ (`noCorpusWalkInRenderPaths`: a comment named the walker), fixed in `8d9d11f9`; on the merged tree
+ **2,610 passed, 0 failed, 0 skipped** (2,610). **The 28 tests T1 skipped all run** (none skipped
+ anywhere): `relocateChannelMedia.test.ts` 13, `renameChannel.test.ts` 1, `storageLocations.test.ts` 7,
+ `storageWatch.test.ts` 7 (T1's record lists 7 there; 28 in all). New: `relocateChannelMedia.test.ts`
+ 40 → 44 (a channel tiered in place moves without tiering; a legacy channel refused out, back and in the
+ preview; a tier-migration marker refused; `channelMediaWriters`) plus the rebased preview case (tiers,
+ idempotent) and back case (a real `media/`, the link untouched, the next move tiers nothing);
+ `storageLocations.test.ts` 14 → 16 (a legacy channel counted unreachable and to migrate with the tier
+ sums; the re-point refuses it); `channels.test.ts` 10 → 11 (a legacy delete); `views/storage.test.ts`
+ 16 → 18; `channelRow.test.ts` rebased.
+- **Editor unit:** 109/109 (before and after the merge).
+- **test:scripts:** before the merge 394 passed, 2 skipped (396); on the merged tree 461 passed, 2
+ skipped (463 — U2 and the deck's tests came with `main`).
+- **Build:** the capped editor build (`systemd-run --scope -p MemoryMax=6G`): exit 0, 39 s before the
+ merge, 53 s on the merged tree.
+- **e2e** (from the worktree root, `$T/T2-specs.txt`: channel-storage, storage-locations,
+ channels-storage-columns, bulk-actions, maybe-missing, video-page, fetch-window, channel-rename):
+ run 1 (before the merge) **67 passed, 1 failed, 4.9 min** — the reconcile case's "changed" file had the
+ same size as the source's and was written in the same second, so rsync's quick check called it
+ unchanged (a fixture fault, fixed in `12339ae2`); run 2, `channel-storage.spec.ts` alone, **15 passed,
+ 0 failed, 2.1 min**; run 3 on the merged tree (`46f6b51e` + this record) **68 passed, 0 failed, 3.9 min** (after a few minutes in the queue behind another worktree's suite).
+- **Privacy gate:** 0 added lines carry the user or host name (`git diff main`, counts only; the one file the whole-file grep names is `plans/FACTS.md`, with the same count as on `main`). No identifier ends in the refused parent suffix.
+- **Numbers tool:** none.
+
+#### Found and left
+
+- `HELD_REASON.inconsistent` (`lib/channelMediaHold.ts`, T1's) still says "its data link and its config
+ disagree"; since release 17 it is the media link. Wording only.
+- A move interrupted DURING its preflight's tiering leaves no marker; the rerun tiers the rest. A file a
+ writer finishes while a marker stands stays real in `data/` (on the corpus disk) until the next hook
+ sweep after the move tiers it onto the far drive — by design (the hook writes nothing under a marker).
+- `removeVideoDirMedia` still cannot clear `media/<id>/` while the drive is unmounted (T1's note); the
+ video page's file delete now refuses that case for one file, and the directory delete already did.
+- For T3: the migration writes `mediaDir` and unsets `dataDir`; everything in this slice reads a channel
+ with `mediaDir` and no `dataDir` as relocated on the new layout, and a channel with both as legacy.
+
+#### Review (SHIP AFTER FIXES) and the fixes
+
+The review (`$T/T2-review.md`) found one HIGH, three LOWs and five NITs. Rulings: the live-chat archive
+publishes the cues on disk while a channel's media is away and never drops the channel; legacy channels
+stay placed by their retired `dataDir`; the digest job stands for the digest lane in e2e.
+
+| Finding | Fix |
+|---|---|
+| H1 a legacy channel refused the re-point of its whole location | `22bdf3d7`: the preflight lists a legacy channel (its `data/` a link) with the others and `RepointPreflight.legacy` names it; the job re-points it the retired way in the same ledger and rollback — its `data` link and `dataDir` to `relocatedDataDir(newRoot, slug)` after checking that tree exists — so `dataDir` stays `<root>/<slug>/data` for T3. A recorded `dataDir` over a real `data/` is refused by name. Tests: one legacy and one `mediaDir` channel on one location, both re-pointed; a legacy tree missing under the new root is named and nothing moves. |
+| L2 a legacy channel's rename left `<root>/<oldSlug>/data` | `b9de2318`: refused before anything moves, with the `migrate-tier` sentence. Test. |
+| L3 Clear marker removed a tier-migration marker | `0e53239a`: `tierMigrationRefusal` is exported and Clear marker, Resume/Reconcile, the preview and the job give its one sentence. |
+| L4 videos with a raw replay and no cues dropped silently | `ec3a7e59`: counted in the build log ("N video(s) with a raw replay and no cues left out of this build"); the channel is never dropped. |
+| N5 the job tiered before checking the destination's identity | `0e53239a`: `assertRelocationRootPresent` runs before the preflight tier (and again before the mkdir). |
+| N6 legacy channels counted "unmeasured" on the corpus row | `250c02fa`: left out of the corpus volume's sums and named "legacy (n to migrate)" in its breakdown. Test. |
+| N7 the drive was chosen by `classifyEntry === "media"` | `7069e5b8`: a file is on the media tier only when its `lstat` is a link and its name `isTierable` (`source-media.*` never is), in the file route, the file delete and the directory loader. |
+| N8 a dangling link on an in-place `media/` answered 503 forever | `7069e5b8`: 503 only when the channel's media is relocated (`mediaDir`); otherwise 404, and the file delete removes the dangling link. |
+| N9 the preview did not refuse a tier-migration marker | `0e53239a`: it does, and tiers nothing. Test. |
+
+**Gates after the fixes** (logs `$T/T2-regate.log`, `$T/T2-e2e-4.log`, `$T/T2-regate2.log`,
+`$T/T2-e2e-5.log`):
+- Before the merge, at `250c02fa`: tsc clean; the four mover/storage test files (`relocateChannelMedia`,
+ `storageLocations`, `renameChannel`, `storageWatch`) 88/88; common 2,613 passed, 0 failed, 0 skipped;
+ editor unit 109/109; e2e (`channel-storage`, `storage-locations`, `channel-rename`, `video-page`,
+ `$T/T2-specs-review.txt`) **46 passed, 0 failed, 3.5 min**.
+- **Merge of `main` `ce9ec612`** (slice RL and a plans commit) at `a001f17e`: two conflicts, both
+ appends — `editor/CHANGELOG.md` keeps every `[Unreleased]` bullet, this file keeps U1, U2, XP, D0, T1,
+ RL, then T2 under `## Record`. On the merged tree: tsc clean; **common 2,653 passed, 0 failed, 0
+ skipped** (2,653); editor unit 109/109; e2e (the same four specs) **46 passed, 0 failed, 3.3 min**.
+- Not re-run: the capped build and test:scripts (no package, route or script surface changed by the
+ fixes; tsc covers the editor's types). Privacy: 0 added lines carry the user or host name.
+- From this point a commit's `Co-Authored-By` names the model that wrote it (Opus); the earlier
+ commits are not rewritten.
+
+**Re-review (SHIP AFTER FIXES, no further round):**
+- R1 the file route and the file delete `lstat`ed before asking the text stall → `bb2c259f`:
+ `channelTextStall` first (no I/O), and no `lstat` on a channel with `dataDir` (a legacy channel's
+ `data/` is a link onto the retired drive and holds no tiered links).
+- NIT the mixed re-point rollback → `37fe4644`: media channel first then the legacy one fails, and the
+ reverse; both restored, no `media` link invented, settings written 0 times (`storageLocations.test.ts`
+ 19/19).
+- R2 gates at `37fe4644`: tsc clean; **common 2,655 passed, 0 failed, 0 skipped**; capped editor build
+ exit 0, 44 s; e2e (`channels-storage-columns`, `bulk-actions`, `maybe-missing`, `fetch-window`)
+ **22 passed, 0 failed, 1.7 min**.
+
## Rollout
