commit caac7c6a6c02a33ed03a83d5b7bcb4440fe65ef6
parent 0fd12e4122724c390aa0f16b3b9ccec84775ed51
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 1 Oct 2026 17:15:44 -0400
common: the media tier's model — mediaDir, legacy, the text guard, lanes and builds by tier
config.json gains mediaDir (CHANNEL.md regenerated); dataDir is documented
retired and still parsed. inspectChannelMedia describes channels/<slug>/media
+ mediaDir, adds `legacy` (a data link or a recorded dataDir) and a text
record; assertChannelTextReadable beside the media guard; isTextHeld and
HELD_REASON.legacy; channelMediaStall keys on mediaDir, channelTextStall on
the retired dataDir. The index and stats builds, the snapshot, normalize,
shards and clip eviction read the text tier only; the digest lane and the
digest batch are held only by the text hold; fourteen kinds flip needsMedia
-> needsText and runManagedFunction asks the text guard for them;
channelWriters takes mediaOnly. The snapshot's byte figures split into media,
text and clips, its links statted once per video through the watchdog.
The mover/re-point/rename/storage-watch cases that build the retired layout
are skipped with a reason until slice T2 rebases them.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
36 files changed, 1339 insertions(+), 479 deletions(-)
diff --git a/CHANNEL.md b/CHANNEL.md
@@ -6,9 +6,9 @@ One channel of the corpus, persisted to `transcripts/channels/<slug>/config.json
`handling` is the one required key: a file without a valid one is not a channel. The smallest channel is `{ "handling": "youtube", "url": "https://www.youtube.com/@example" }`.
-Every other key is optional and has NO default of its own: an absent key means whatever its description says — for the per-channel overrides, inherit the global setting of the same name; for `name`, `url`, `dataDir`, `subLangs` and the sync-state stamps, simply unset. So an ill-typed or out-of-range value is not coerced — it is DROPPED, as if the file did not spell it. Unknown keys (including the retired `excludeFromSync`, now a paused `sync` tier in the channel-priority document) are dropped by every read and every write.
+Every other key is optional and has NO default of its own: an absent key means whatever its description says — for the per-channel overrides, inherit the global setting of the same name; for `name`, `url`, `mediaDir`, `subLangs` and the sync-state stamps, simply unset. So an ill-typed or out-of-range value is not coerced — it is DROPPED, as if the file did not spell it. Unknown keys (including the retired `excludeFromSync`, now a paused `sync` tier in the channel-priority document) are dropped by every read and every write.
-The three **sync state** keys are not configuration: the sync, sweep and download passes stamp them, the channel form never does, and they live in the same file on purpose. Writers after creation PATCH (`patchChannelConfig`): each re-reads the file at the moment it writes and changes only its own keys, so a stamp and a form save made at once in the editor both land. The two exceptions write a whole config, and only when there is no readable file to patch: a media move and a channel rename record `dataDir` from their own copy of the config.
+The three **sync state** keys are not configuration: the sync, sweep and download passes stamp them, the channel form never does, and they live in the same file on purpose. Writers after creation PATCH (`patchChannelConfig`): each re-reads the file at the moment it writes and changes only its own keys, so a stamp and a form save made at once in the editor both land. The two exceptions write a whole config, and only when there is no readable file to patch: a media move and a channel rename record `mediaDir` from their own copy of the config.
Regenerate this file with `pnpm --filter yt-dlp-transcript-common exec tsx bin/file-schemas-docs.ts`.
@@ -27,7 +27,8 @@ Regenerate this file with `pnpm --filter yt-dlp-transcript-common exec tsx bin/f
| `keepLatest` | config | Keep-latest window: the newest N videos (by upload date) are protected from the Clean-audio sweep AND have their source video persisted to the saved-video store. 0 or absent = disabled; positives clamp to [1, 100000]. A kept video later found deleted at the source is pinned permanently via the do-not-clean marker. |
| `extractionMode` | config | `"ytdlp"` (default — yt-dlp's own `-x --audio-format` postprocessor, no source container kept) or `"app"` (yt-dlp downloads the source container and the app runs ffmpeg). The keep-latest persistence rule forces `"app"` for the videos it persists. |
| `savedVideosDir` | config | Per-channel override for the saved-video store root: this channel's persisted source videos live under `<savedVideosDir>/<slug>/<videoId>/`. Trimmed; blank = the global store. |
-| `dataDir` | config | Where this channel's media ACTUALLY lives when relocated to another drive: the absolute path `channels/<slug>/data` is a symlink to. Absent = in place. Written ONLY by the relocate / re-point jobs on success — a record of what is on disk, never free text, because a value that disagrees with the link is an "inconsistent" channel every guard refuses. |
+| `dataDir` | config | RETIRED (release 17). The whole-directory layout's record: the absolute path `channels/<slug>/data` was a symlink to. Still parsed for one release so a write never erases it: a channel that carries it — or whose `data/` is a link — is `legacy`, and every media job, lane and build holds it until `archilyzer storage migrate-tier <slug>` moves its text back and its media into `mediaDir`. Never written by anything but that migration, which removes it. |
+| `mediaDir` | config | Where this channel's big files live when relocated: `channels/<slug>/media` is a symlink to it, `<root>/<slug>/media`. Absent = in place. Written only by relocate / re-point / the tier migration — a record of what is on disk, never free text, because a value that disagrees with the link is an "inconsistent" channel every media guard refuses. The text (`data/`) never moves. |
| `ytdlpExtraArgs` | config | Extra yt-dlp arguments, appended verbatim. Must be an array of strings or it is dropped. |
| `subLangs` | config | yt-dlp `--sub-langs` value for caption downloads. |
| `lastSyncedAt` | sync state | SYNC STATE. When the channel last synced (ISO time). Stamped by every sync, and by a social fetch; read by the scheduler's cadence gate. |
diff --git a/common/bin/doctor.test.ts b/common/bin/doctor.test.ts
@@ -237,9 +237,16 @@ test("an unmounted drive is a warning, not an empty channel and not a failure",
const c = checkout();
const chan = path.join(c.paths.channelsDir, "moved");
mkdirSync(chan, { recursive: true });
- const target = path.join(c.root, "elsewhere", "moved", "data");
- writeFileSync(path.join(chan, "config.json"), JSON.stringify({ dataDir: target }));
- symlinkSync(target, path.join(chan, "data")); // the target does not exist
+ const target = path.join(c.root, "elsewhere", "moved", "media");
+ writeFileSync(path.join(chan, "config.json"), JSON.stringify({ mediaDir: target }));
+ mkdirSync(path.join(chan, "data"));
+ symlinkSync(target, path.join(chan, "media")); // the target does not exist
+ // A channel on the retired whole-directory layout is named with its way out.
+ const old = path.join(c.paths.channelsDir, "retired");
+ mkdirSync(old, { recursive: true });
+ const oldTarget = path.join(c.root, "elsewhere", "retired", "data");
+ writeFileSync(path.join(old, "config.json"), JSON.stringify({ dataDir: oldTarget }));
+ symlinkSync(oldTarget, path.join(old, "data"));
// No workers: a `{}` file would synthesize whisper workers, whose engine
// this machine does not have — a real failure, and not this test's.
writeFileSync(c.paths.settingsFile, JSON.stringify({ workers: [] }));
@@ -250,6 +257,8 @@ test("an unmounted drive is a warning, not an empty channel and not a failure",
const media = r.checks.find((x) => x.id === "media")!;
assert.equal(media.status, "warn");
assert.match(media.detail, /moved: unreachable/);
+ const retired = r.checks.filter((x) => x.id === "media").map((x) => x.detail).join("\n");
+ assert.match(retired, /retired: legacy — .*archilyzer storage migrate-tier retired/);
assert.equal(r.ok, true, renderDoctorReport(r));
assert.deepEqual(tree(c.root), before);
});
diff --git a/common/bin/run-operation.test.ts b/common/bin/run-operation.test.ts
@@ -140,12 +140,13 @@ test("the media guard refuses an unmounted channel before any job exists", { tim
const slug = "moved";
const channelDir = path.join(getPaths().channelsDir, slug);
await mkdir(channelDir, { recursive: true });
- const target = path.join(ROOT, "platter", slug, "data"); // never created
+ const target = path.join(ROOT, "platter", slug, "media"); // never created
await writeFile(
path.join(channelDir, "config.json"),
- JSON.stringify({ handling: "transcribe", url: "https://example.com/m", dataDir: target }),
+ JSON.stringify({ handling: "transcribe", url: "https://example.com/m", mediaDir: target }),
);
- await symlink(target, path.join(channelDir, "data"));
+ await mkdir(path.join(channelDir, "data"));
+ await symlink(target, path.join(channelDir, "media"));
const before = jobRecords();
const { o, out } = capture();
const code = await runOperation({ operation: "diarization", channel: slug, ids: [] }, { out });
diff --git a/common/controller/autoRunner.ts b/common/controller/autoRunner.ts
@@ -89,7 +89,7 @@ import {
readRelocationMarker,
type ChannelMediaStatus,
} from "../lib/channelMedia";
-import { isMediaHeld, mediaHoldText } from "../lib/channelMediaHold";
+import { isMediaHeld, isTextHeld, mediaHoldText } from "../lib/channelMediaHold";
import {
type ChannelPriority,
type FocusSummary,
@@ -637,8 +637,19 @@ async function buildChannelWork(
// drive, a link and a config that disagree. It lifts by itself — the next
// tick after the marker is removed (a move completed or abandoned; the
// movers forget the memo) or the drive is back reads the channel again.
+ //
+ // THE DIGEST LANE READS TEXT (release 17): it is held only by the text
+ // hold — a `legacy` channel, or a `data/` it cannot read — so a digest
+ // runs on a channel whose media is moving, stalled or unmounted. The
+ // transcription, download and backfill lanes open the big files and keep
+ // the media hold.
const location = media[i];
- if (location && isMediaHeld(location.status)) {
+ const held = location
+ ? kind === "digest"
+ ? isTextHeld(location.status) || !location.text.readable
+ : isMediaHeld(location.status)
+ : false;
+ if (location && held) {
noteSkippedForMedia(slug, location.status, location.detail);
continue;
}
@@ -1894,8 +1905,12 @@ async function runLoop(
// lane it retires nothing — runOperationPick owns the (operation, video)
// keys and never ran — which is equally fine, because guard 2 has taken
// the channel off the list before the next tick could offer it again.
+ //
+ // NOT THE DIGEST LANE (release 17): a media move leaves the text where it
+ // is, and a digest writes only text. A tier migration rebuilds `data/`
+ // itself, and holds it too.
const marker = await readRelocationMarker(paths, channelSlug);
- if (marker) {
+ if (marker && (kind !== "digest" || marker.scope === "tier-migration")) {
onLog(
`Auto-${kind}: skipping ${channelSlug}/${pick.videoId}, ` +
`${mediaHoldText("in-transition")} — a relocation ` +
diff --git a/common/controller/buildIndex.test.ts b/common/controller/buildIndex.test.ts
@@ -1,15 +1,23 @@
// Integration: the index build's HOLD, through the REAL buildIndex, over a temp
// corpus.
//
-// A channel's `data/` may be an absolute symlink to another drive (AGENTS.md,
-// "A channel's `data/` may live on another drive"). With that drive unmounted
-// the link dangles, and the index build used to read the channel as having no
-// videos: it removed every record the channel had, and the site built next
-// published the channel as gone. These cases pin the hold that replaced it:
-// the channel is not rescanned, and its records and shared pages are kept as
-// they are; a FULL rebuild with a channel held refuses unless
-// ARCHILYZER_INDEX_ALLOW_HELD is set; and all of it undoes itself when the
-// drive is back. The stats build's twin is buildStats.test.ts case (i).
+// A channel's `data/` used to be an absolute symlink to another drive (the
+// retired whole-directory layout). With that drive unmounted the link
+// dangled, and the index build used to read the channel as having no videos:
+// it removed every record the channel had, and the site built next published
+// the channel as gone. These cases pin the hold that replaced it: the channel
+// is not rescanned, and its records and shared pages are kept as they are; a
+// FULL rebuild with a channel held refuses unless ARCHILYZER_INDEX_ALLOW_HELD
+// is set; and all of it undoes itself when the text is readable again. The
+// stats build's twin is buildStats.test.ts case (i).
+//
+// RELEASE 17: only the TEXT holds the index. A relocated channel's text stays
+// on the corpus disk and only its media (`channels/<slug>/media`) is on the
+// drive, so an unmounted MEDIA drive holds nothing here (case (j)). What holds
+// is a text tier that cannot be read — here, the channel put back on the
+// retired layout with its drive away (`unmount`), which reads `legacy` — and
+// what lifts it is the text home again (`remount`, which is what
+// `archilyzer storage migrate-tier` does).
//
// Run with: node_modules/.bin/tsx --test common/controller/buildIndex.test.ts
@@ -83,7 +91,9 @@ let afterStat: ((p: string) => void) | null = null;
const fsCjs = req("node:fs") as Record<string, unknown>;
const fspCjs = req("node:fs/promises") as Record<string, unknown>;
const WRITES = ["writeFile", "appendFile", "rename", "mkdir", "rm", "rmdir", "unlink", "copyFile", "cp", "symlink", "link", "utimes", "truncate", "mkdtemp", "chmod"];
- const TWO_PATHS = new Set(["rename", "copyFile", "cp", "symlink", "link"]);
+ // A symlink's first argument is its TARGET (relative to the link, and never
+ // written); only the link itself, the second, is.
+ const TWO_PATHS = new Set(["rename", "copyFile", "cp", "link"]);
const opensForWrite = (flags: unknown) =>
(typeof flags === "string" && /[wa+]/.test(flags)) ||
(typeof flags === "number" && (flags & 3) !== 0);
@@ -94,7 +104,8 @@ let afterStat: ((p: string) => void) | null = null;
if (typeof fn !== "function") return;
mod[name] = function (this: unknown, ...args: unknown[]) {
if (mode === "write" || opensForWrite(args[1])) {
- const ps = TWO_PATHS.has(name.replace(/Sync$/, "")) ? [args[0], args[1]] : [args[0]];
+ const base = name.replace(/Sync$/, "");
+ const ps = base === "symlink" ? [args[1]] : TWO_PATHS.has(base) ? [args[0], args[1]] : [args[0]];
for (const a of ps) {
const p = asPath(a);
if (p !== null) writes.push(path.resolve(p));
@@ -200,21 +211,40 @@ function resetCorpus(channels: string[] = [CHANNEL, DRIVE_CHANNEL]): void {
});
}
-// A channel whose data/ is a relocated symlink, the way the editor's Storage
-// panel leaves it: channels/<slug>/data -> <MEDIA>/<slug>/data, with
-// config.dataDir recording the target. d1 carries a subtitle track.
+// A channel whose MEDIA is relocated (release 17): its text in a real data/
+// on the corpus disk, channels/<slug>/media -> <MEDIA>/<slug>/media, with
+// config.mediaDir recording the target. d1 carries a subtitle track.
+const DRIVE_MEDIA = () => path.join(MEDIA, DRIVE_CHANNEL, "media");
+const driveDataLink = () => path.join(paths.channelsDir, DRIVE_CHANNEL, "data");
+const AWAY_TEXT = () => path.join(AWAY, DRIVE_CHANNEL, "data");
function seedDriveChannel(titles: Record<string, string> = {}): void {
- const target = path.join(MEDIA, DRIVE_CHANNEL, "data");
- mkdirSync(target, { recursive: true });
- writeChannel(DRIVE_CHANNEL, { dataDir: target });
- symlinkSync(target, path.join(paths.channelsDir, DRIVE_CHANNEL, "data"));
+ mkdirSync(DRIVE_MEDIA(), { recursive: true });
+ writeChannel(DRIVE_CHANNEL, { mediaDir: DRIVE_MEDIA() });
+ symlinkSync(DRIVE_MEDIA(), path.join(paths.channelsDir, DRIVE_CHANNEL, "media"));
seedVideo("d1", DRIVE_CHANNEL, { subs: true, title: titles.d1 });
seedVideo("d2", DRIVE_CHANNEL, { title: titles.d2 });
}
-// Unmount: the link now dangles, exactly as an absent USB drive leaves it.
-const unmount = () => renameSync(MEDIA, AWAY);
-const remount = () => renameSync(AWAY, MEDIA);
+// The text goes away: the channel is on the retired whole-directory layout
+// (`data` an absolute link to <MEDIA>/<slug>/data, `dataDir` recorded) with its
+// drive not mounted — the text moved to AWAY, the link dangling. `rename`
+// keeps every mtime, as `rsync -a` would.
+const unmount = () => {
+ mkdirSync(path.dirname(AWAY_TEXT()), { recursive: true });
+ renameSync(driveDataLink(), AWAY_TEXT());
+ const target = path.join(MEDIA, DRIVE_CHANNEL, "data");
+ symlinkSync(target, driveDataLink());
+ writeChannel(DRIVE_CHANNEL, { dataDir: target });
+};
+// The text home again, in a real data/ (what `migrate-tier` leaves).
+const remount = () => {
+ rmSync(driveDataLink());
+ renameSync(AWAY_TEXT(), driveDataLink());
+ writeChannel(DRIVE_CHANNEL, { mediaDir: DRIVE_MEDIA() });
+};
+// The MEDIA drive alone goes away and comes back.
+const unmountMedia = () => renameSync(MEDIA, `${MEDIA}-unmounted`);
+const remountMedia = () => renameSync(`${MEDIA}-unmounted`, MEDIA);
async function runIndex(log: string[] = []) {
const res = await buildIndex({ paths, onLog: (s) => log.push(s) });
@@ -327,7 +357,7 @@ test("(a) an unmounted drive: the channel's records, transcript pages and subs s
assert.ok(line, log.join("\n"));
assert.match(
line,
- /its media is not reachable \(drive not mounted\?\), on location "USB drive"; its 2 indexed video\(s\) are kept as they are, not rescanned/,
+ /its media layout is the retired whole-directory one \(run archilyzer storage migrate-tier\), on location "USB drive"; its 2 indexed video\(s\) are kept as they are, not rescanned/,
);
assert.ok(!line.includes(ROOT), line);
assert.ok(
@@ -387,7 +417,7 @@ test("(c) a full rebuild with a channel held refuses without the override, and h
err.message,
new RegExp(
`must be rebuilt in full \\(index schema ${current - 1} -> ${current}\\), but 1 channel\\(s\\) cannot be read: ` +
- `drive-channel \\(its media is not reachable \\(drive not mounted\\?\\), on location "USB drive"\\)`,
+ `drive-channel \\(its media layout is the retired whole-directory one \\(run archilyzer storage migrate-tier\\), on location "USB drive"\\)`,
),
);
// Why, the ways out (mounting first), and the override by name; no path.
@@ -438,7 +468,7 @@ test("(c) a full rebuild with a channel held refuses without the override, and h
assert.ok(
log.some(
(l) =>
- l.startsWith(`Channel ${DRIVE_CHANNEL}: its media is not reachable`) &&
+ l.startsWith(`Channel ${DRIVE_CHANNEL}: its media layout is the retired`) &&
l.includes(`held under ${INDEX_ALLOW_HELD_ENV}: this full rebuild cleared its index records`),
),
log.join("\n"),
@@ -629,12 +659,36 @@ test("(i) the drive lost MID-WALK: the second look holds the channel instead of
assert.deepEqual(sharedSubs(), subsBefore);
assert.ok(
log.some((l) =>
- l.startsWith(`Channel ${DRIVE_CHANNEL}: its media is not reachable (drive not mounted?), on location "USB drive"; its 4 indexed video(s) are kept`),
+ l.startsWith(`Channel ${DRIVE_CHANNEL}: its media layout is the retired whole-directory one (run archilyzer storage migrate-tier), on location "USB drive"; its 4 indexed video(s) are kept`),
),
log.join("\n"),
);
});
+test("(j) release 17: an unmounted MEDIA drive does not hold the index — the text is read", async () => {
+ resetCorpus();
+ seedVideo("local");
+ seedDriveChannel();
+ // d2's audio is tiered: a relative link into the channel's media/.
+ mkdirSync(path.join(DRIVE_MEDIA(), "d2"), { recursive: true });
+ writeFileSync(path.join(DRIVE_MEDIA(), "d2", "audio.mp3"), "AUDIO");
+ symlinkSync("../../media/d2/audio.mp3", path.join(videoDir("d2", DRIVE_CHANNEL), "audio.mp3"));
+ await runIndex();
+ assert.deepEqual(indexed(), [...DRIVE_VIDEOS, `${CHANNEL}/local`]);
+
+ unmountMedia();
+ try {
+ seedVideo("d3", DRIVE_CHANNEL); // arrived meanwhile, text on the corpus disk
+ const { res, log } = await runIndex();
+ assert.deepEqual(res.heldChannels, [], log.join("\n"));
+ assert.equal(res.added, 1);
+ assert.equal(res.removed, 0);
+ assert.deepEqual(indexed(), [...DRIVE_VIDEOS, `${DRIVE_CHANNEL}/d3`, `${CHANNEL}/local`]);
+ } finally {
+ remountMedia();
+ }
+});
+
test("(z) no write this file caused landed outside its temp root", () => {
// LMDB writes natively, past the spy: its file must be under the root too.
assert.ok(paths.lmdbPath.startsWith(ROOT + path.sep), paths.lmdbPath);
diff --git a/common/controller/buildIndex.ts b/common/controller/buildIndex.ts
@@ -99,7 +99,6 @@ import {
HELD_WAYS_OUT,
describeHeld,
heldReason,
- isMediaHeld,
} from "../lib/channelMediaHold";
import { resolveChannelGroupId } from "../lib/channelGroups";
import type { Paths } from "../lib/paths";
@@ -342,14 +341,20 @@ async function scanSource(
// and routing it through the video scan would only ever produce noise. Its
// posts tree is built from the JSONL shards further down.
if (isSocialChannel(cfg)) continue;
- // An unmounted drive is not an empty channel (lib/channelMedia.ts): the
- // readdir below would fail, and every record the channel has would be
+ // An unreadable text tier is not an empty channel (lib/channelMedia.ts):
+ // the readdir below would fail, and every record the channel has would be
// removed as gone.
+ //
+ // THE TEXT GUARD, NOT THE MEDIA ONE (release 17): the index reads the text
+ // tier only — presence of a media file is a name in the one readdir, and
+ // every stat is a sidecar — so a moving, stalled or unmounted MEDIA drive
+ // never holds it. Held: a `legacy` channel (its text is on the far drive),
+ // a `data/` that is not a directory, a tier migration in flight.
const media = await inspectChannelMedia({ channelsDir }, ch.name, cfg, {
fresh: true,
});
- if (isMediaHeld(media.status)) {
- held.set(ch.name, heldReason(media, cfg.dataDir, locations));
+ if (!media.text.readable) {
+ held.set(ch.name, heldReason(media, cfg.mediaDir ?? cfg.dataDir, locations));
continue;
}
const dataDir = path.join(channelDir, "data");
@@ -358,9 +363,9 @@ async function scanSource(
videoEntries = await readdir(dataDir, { withFileTypes: true });
} catch (err) {
const code = errCode(err);
- // No data/ at all on a channel whose media was never moved: it has
- // downloaded nothing yet (or its media was deleted), and it IS empty.
- if (code === "ENOENT" && media.status === "in-place") {
+ // No data/ at all: the text tier is on the corpus disk, so the channel
+ // has downloaded nothing yet (or its files were deleted), and it IS empty.
+ if (code === "ENOENT") {
log(`Channel ${ch.name}: no data/ directory; indexed as a channel with no videos.`);
continue;
}
@@ -461,14 +466,14 @@ async function scanSource(
held.set(ch.name, `a video in its data directory could not be read (${readFailure})`);
continue;
}
- // Asked again after the walk: a drive that went away DURING it leaves the
- // videos after that point missing from this scan, which would remove them.
- // Three syscalls a channel.
+ // Asked again after the walk: a text tier that became unreadable DURING it
+ // (a tier migration that started) leaves the videos after that point
+ // missing from this scan, which would remove them. A few syscalls a channel.
const after = await inspectChannelMedia({ channelsDir }, ch.name, cfg, {
fresh: true,
});
- if (isMediaHeld(after.status)) {
- held.set(ch.name, heldReason(after, cfg.dataDir, locations));
+ if (!after.text.readable) {
+ held.set(ch.name, heldReason(after, cfg.mediaDir ?? cfg.dataDir, locations));
continue;
}
for (const e of channelLive) live.push(e);
diff --git a/common/controller/buildStats.test.ts b/common/controller/buildStats.test.ts
@@ -455,19 +455,22 @@ test("(h) a video the index skipped is not announced as pending on every run", a
assert.equal(res.notIndexable, 1, "the undated one stays, and is said as such");
});
-// A second channel whose data/ is a relocated symlink, the way the editor's
-// Storage panel leaves it: channels/<slug>/data -> <root>/<slug>/data, with
-// config.dataDir recording the target.
-function seedDriveChannel(): { target: string } {
- const target = path.join(ROOT, "media", DRIVE_CHANNEL, "data");
- mkdirSync(target, { recursive: true });
+// A second channel whose MEDIA is relocated (release 17): its text in a real
+// data/ on the corpus disk, channels/<slug>/media -> <root>/<slug>/media, with
+// config.mediaDir recording the target.
+function driveConfig(extra: Record<string, unknown>): void {
writeJson(path.join(paths.channelsDir, DRIVE_CHANNEL, "config.json"), {
handling: "youtube",
name: "Drive Channel",
url: "https://www.youtube.com/@drive/videos",
- dataDir: target,
+ ...extra,
});
- symlinkSync(target, path.join(paths.channelsDir, DRIVE_CHANNEL, "data"));
+}
+function seedDriveChannel(): { target: string } {
+ const target = path.join(ROOT, "media", DRIVE_CHANNEL, "media");
+ mkdirSync(target, { recursive: true });
+ driveConfig({ mediaDir: target });
+ symlinkSync(target, path.join(paths.channelsDir, DRIVE_CHANNEL, "media"));
for (const id of ["d1", "d2"]) {
seedVideo(id, "2026-07-11T11:00:00Z", {}, DRIVE_CHANNEL);
addCaptions(id, "2026-07-11T12:00:00Z", DRIVE_CHANNEL);
@@ -475,7 +478,45 @@ function seedDriveChannel(): { target: string } {
return { target };
}
-test("(i) an unmounted media drive keeps its channel's stats; a cache clear refuses", async () => {
+// The text goes away: the channel put on the retired whole-directory layout
+// (`data` an absolute link to <root>/<slug>/data, `dataDir` recorded) with its
+// drive not mounted — the text moved aside, the link dangling. And back.
+function retireAndUnmount(): void {
+ const data = path.join(paths.channelsDir, DRIVE_CHANNEL, "data");
+ const away = path.join(ROOT, "media-away", DRIVE_CHANNEL, "data");
+ mkdirSync(path.dirname(away), { recursive: true });
+ renameSync(data, away);
+ const target = path.join(ROOT, "media", DRIVE_CHANNEL, "data");
+ symlinkSync(target, data);
+ driveConfig({ dataDir: target });
+}
+function migrateHome(): void {
+ const data = path.join(paths.channelsDir, DRIVE_CHANNEL, "data");
+ rmSync(data);
+ renameSync(path.join(ROOT, "media-away", DRIVE_CHANNEL, "data"), data);
+ driveConfig({ mediaDir: path.join(ROOT, "media", DRIVE_CHANNEL, "media") });
+}
+
+test("(i2) release 17: an unmounted MEDIA drive does not hold the stats build", async () => {
+ resetCorpus();
+ seedVideo("local");
+ seedDriveChannel();
+ await runIndex();
+ await runStats();
+ const media = path.join(ROOT, "media");
+ renameSync(media, `${media}-unmounted`);
+ try {
+ const log: string[] = [];
+ const away = await runStats(log);
+ assert.deepEqual(away.res.heldChannels, [], log.join("\n"));
+ assert.equal(away.res.removed, 0);
+ assert.equal(statOf(away.byId, "d1").hasTranscript, true);
+ } finally {
+ renameSync(`${media}-unmounted`, media);
+ }
+});
+
+test("(i) a channel whose text cannot be read (the retired layout, drive away) keeps its stats; a cache clear refuses", async () => {
resetCorpus();
seedVideo("local");
seedDriveChannel();
@@ -490,8 +531,7 @@ test("(i) an unmounted media drive keeps its channel's stats; a cache clear refu
paths.settingsFile,
JSON.stringify({ storage: { locations: [{ id: "usb", label: "USB drive", root: media, autoRepoint: false }] } }),
);
- // Unmount: the link now dangles, exactly as an absent USB drive leaves it.
- renameSync(media, `${media}-away`);
+ retireAndUnmount();
const log: string[] = [];
const away = await runStats(log);
assert.equal(away.res.removed, 0, "not read as a channel with no videos");
@@ -499,7 +539,7 @@ test("(i) an unmounted media drive keeps its channel's stats; a cache clear refu
assert.deepEqual(away.res.heldChannels, [DRIVE_CHANNEL]);
assert.equal(statOf(away.byId, "d1").hasTranscript, true);
assert.ok(
- log.some((l) => l.startsWith(`Channel ${DRIVE_CHANNEL}: its media is not reachable`) && l.includes("its 2 cached stat(s) are kept")),
+ log.some((l) => l.startsWith(`Channel ${DRIVE_CHANNEL}: its media layout is the retired`) && l.includes("its 2 cached stat(s) are kept")),
log.join("\n"),
);
@@ -509,7 +549,7 @@ test("(i) an unmounted media drive keeps its channel's stats; a cache clear refu
await assert.rejects(runStats(), (err: Error) => {
assert.match(
err.message,
- /must be rebuilt .* cannot be read: drive-channel \(its media is not reachable \(drive not mounted\?\), on location "USB drive"\)/,
+ /must be rebuilt .* cannot be read: drive-channel \(its media layout is the retired whole-directory one \(run archilyzer storage migrate-tier\), on location "USB drive"\)/,
);
// The ways out, mounting first, and no path in the message.
assert.match(
@@ -522,7 +562,7 @@ test("(i) an unmounted media drive keeps its channel's stats; a cache clear refu
assert.equal(readStoredSchema(), STATS_SCHEMA_VERSION - 1);
assert.equal(countStats(), 3);
- renameSync(`${media}-away`, media);
+ migrateHome();
const back = await runStats();
assert.deepEqual(back.res.heldChannels, []);
assert.equal(readStoredSchema(), STATS_SCHEMA_VERSION);
diff --git a/common/controller/buildStats.ts b/common/controller/buildStats.ts
@@ -68,7 +68,6 @@ import {
HELD_WAYS_OUT,
describeHeld,
heldReason,
- isMediaHeld,
} from "../lib/channelMediaHold";
import { getSettings } from "../lib/settings";
import type { Paths } from "../lib/paths";
@@ -286,13 +285,15 @@ async function scanSource(
}
if (cfg.excludeFromBuild) continue;
channels.set(ch.name, cfg);
- // An unmounted drive is not an empty channel (lib/channelMedia.ts): the
- // readdir below would fail and every one of its stats would be removed.
+ // An unreadable text tier is not an empty channel (lib/channelMedia.ts):
+ // the readdir below would fail and every one of its stats would be removed.
+ // THE TEXT GUARD (release 17): the stats build reads the text tier only and
+ // is never held by the media tier — see buildIndex.ts's scanSource.
const media = await inspectChannelMedia({ channelsDir }, ch.name, cfg, {
fresh: true,
});
- if (isMediaHeld(media.status)) {
- held.set(ch.name, heldReason(media, cfg.dataDir, locations));
+ if (!media.text.readable) {
+ held.set(ch.name, heldReason(media, cfg.mediaDir ?? cfg.dataDir, locations));
continue;
}
const dataDir = path.join(channelDir, "data");
diff --git a/common/controller/channelSnapshot.test.ts b/common/controller/channelSnapshot.test.ts
@@ -12,7 +12,7 @@ import {
foldBucketLaneEntry,
generateChannelSnapshot,
} from "./channelSnapshot";
-import { ChannelMediaUnreachableError } from "../lib/channelMedia";
+import { ChannelTextUnreadableError } from "../lib/channelMedia";
import type { Paths } from "../lib/paths";
import {
emptyOperationCounts,
@@ -332,11 +332,13 @@ test("only the reachable diarization states will ever clear a hold", () => {
}
});
-test("generateChannelSnapshot refuses an unreachable channel rather than writing an empty snapshot", async () => {
+test("generateChannelSnapshot refuses a LEGACY channel rather than writing an empty snapshot", async () => {
// The readdir inside it swallows ENOENT as "no videos", so without the guard a
- // relocated channel whose drive is unmounted would publish a snapshot saying
+ // channel whose text is on an unmounted drive would publish a snapshot saying
// every video is undownloaded — and all four lanes read that as work to do.
// The throw is what makes the scheduler keep the last good snapshot.json.
+ // Since release 17 only the retired whole-directory layout puts the text on
+ // another drive, and the text guard refuses it whatever the drive is doing.
const dir = await mkdtemp(path.join(tmpdir(), "ttb-snap-media-"));
try {
const paths = { channelsDir: path.join(dir, "channels") } as Paths;
@@ -352,13 +354,52 @@ test("generateChannelSnapshot refuses an unreachable channel rather than writing
await assert.rejects(
() => generateChannelSnapshot(paths, "alpha"),
- ChannelMediaUnreachableError,
+ (err: unknown) =>
+ err instanceof ChannelTextUnreadableError && /migrate-tier alpha/.test(err.message),
);
} finally {
await rm(dir, { recursive: true, force: true });
}
});
+test("release 17: an unmounted MEDIA drive does not stop the snapshot; its media bytes are unknown", async () => {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-snap-tier-"));
+ try {
+ const paths = { channelsDir: path.join(dir, "channels") } as Paths;
+ const channelDir = path.join(paths.channelsDir, "alpha");
+ const videoDir = path.join(channelDir, "data", "v1");
+ await mkdir(videoDir, { recursive: true });
+ const target = path.join(dir, "platter", "alpha", "media");
+ await writeFile(
+ path.join(channelDir, "config.json"),
+ JSON.stringify({ handling: "youtube", url: "https://example.com/c", mediaDir: target }),
+ );
+ await writeFile(path.join(channelDir, "playlist"), "");
+ const meta = JSON.stringify({ id: "v1" });
+ await writeFile(path.join(videoDir, "metadata.info.json"), meta);
+ await writeFile(path.join(videoDir, "transcript.json"), "{}");
+ // The audio is tiered; the media link dangles (an unmounted drive).
+ await symlink(path.join("..", "..", "media", "v1", "audio.mp3"), path.join(videoDir, "audio.mp3"));
+ await symlink(target, path.join(channelDir, "media"));
+
+ const snap = await generateChannelSnapshot(paths, "alpha");
+ assert.equal(snap.totals.downloaded, 1, "the link is a name in the listing");
+ assert.equal(snap.totalMediaBytes, undefined, "unknown, never 0");
+ assert.equal(snap.totalAudioBytes, undefined);
+ assert.equal(snap.totalTextBytes, meta.length + 2);
+
+ // The drive back: the link's bytes are counted as media.
+ await mkdir(path.join(target, "v1"), { recursive: true });
+ await writeFile(path.join(target, "v1", "audio.mp3"), Buffer.alloc(300));
+ const back = await generateChannelSnapshot(paths, "alpha");
+ assert.equal(back.totalMediaBytes, 300);
+ assert.equal(back.totalAudioBytes, 300);
+ assert.equal(back.totalTextBytes, meta.length + 2);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+});
+
// ---------------------------------------------------------------------------
// The filtered channel, end to end through generateChannelSnapshot.
//
@@ -682,7 +723,7 @@ test("a chat already on disk stays a member after the mode is turned off", async
});
-test("a video dir's clips/ counts into totalMediaBytes AND into totalClipsBytes", async () => {
+test("a video dir's clips/ counts into totalClipsBytes, apart from the media and text tiers", async () => {
// `clips/` is the one subdirectory a video dir has, and until this it was
// counted by nothing: the loop did `if (!st.isFile()) continue` under a
// comment saying a video dir is flat, which the clip-window feature made
@@ -690,8 +731,9 @@ test("a video dir's clips/ counts into totalMediaBytes AND into totalClipsBytes"
// /storage and every relocation estimate under-reported a channel that had
// been walked by a report.
//
- // The two numbers OVERLAP on purpose: totalMediaBytes is every byte under
- // data/<id>/, totalClipsBytes is the "of which".
+ // Release 17: the three are SIBLINGS. totalMediaBytes is the media tier's
+ // (the tierable names — audio, the raw live chat), totalTextBytes the rest of
+ // data/<id>/, totalClipsBytes the clip cache, which is never tiered.
const dir = await mkdtemp(path.join(tmpdir(), "ttb-snap-clips-"));
try {
const paths = { channelsDir: path.join(dir, "channels") } as Paths;
@@ -714,7 +756,8 @@ test("a video dir's clips/ counts into totalMediaBytes AND into totalClipsBytes"
const snap = await generateChannelSnapshot(paths, "alpha");
assert.equal(snap.totalClipsBytes, 57);
- assert.equal(snap.totalMediaBytes, 100 + meta.length + 57);
+ assert.equal(snap.totalMediaBytes, 100);
+ assert.equal(snap.totalTextBytes, meta.length);
} finally {
await rm(dir, { recursive: true, force: true });
}
diff --git a/common/controller/channelSnapshot.ts b/common/controller/channelSnapshot.ts
@@ -1,7 +1,7 @@
import { writeJsonAtomic } from "../lib/jsonFile-server";
import path from "node:path";
import type { Dirent } from "node:fs";
-import { readdir, readFile, stat } from "node:fs/promises";
+import { lstat, readdir, readFile, stat } from "node:fs/promises";
import pLimit from "p-limit";
import { readArchive } from "../lib/archive";
import {
@@ -21,7 +21,8 @@ import {
type Availability,
} from "../lib/availability";
import { resolveCookiePolicy } from "../lib/cookiePolicy";
-import { assertChannelMediaReachable } from "../lib/channelMedia";
+import { assertChannelTextReadable } from "../lib/channelMedia";
+import { isTierable } from "../lib/mediaTier";
import { isDriveNotAnswering, onDrive } from "../lib/storageHealth";
import { CLIPS_DIR_NAME } from "../lib/clipWindow";
import { getSettings } from "../lib/settings";
@@ -349,21 +350,31 @@ export type ChannelSnapshot = {
// must default to 0 — and MUST render that as "—", not "0", because a zero
// here would claim a measurement nobody took.
totalAudioBytes?: number;
- // EVERY byte under `data/<id>/` for every video — audio, transcripts, cues,
- // metadata, thumbnails, a persisted container. The figure `/storage` and the
- // `/channels` Size column are priced in, and the one a relocation carries;
+ // THE MEDIA TIER's bytes (release 17): every file `isTierable` names — the
+ // audio and the raw live-chat replay — whether it is a link into
+ // `channels/<slug>/media/` already or still a real file a move's preflight
+ // would tier. The figure a media move carries and a media location holds;
// `totalAudioBytes` is a fraction of it and is about what a CLEANUP could
- // reclaim, which is a different question.
+ // reclaim, which is a different question. (Before release 17 this was every
+ // byte under `data/<id>/`; the text is now `totalTextBytes`.)
//
// Optional, and the distinction is load-bearing: a snapshot written before
// this field existed lacks it, and a reader MUST render that as "size unknown
// until Refresh report", never as 0 — a zero would rank a 400 GB channel
- // bottom of a "free up N GB" list.
+ // bottom of a "free up N GB" list. ABSENT TOO when the channel's media tier
+ // could not be read (an unmounted, stalled or moving media drive): the
+ // snapshot is still written — its text is readable — and the bytes are
+ // unknown, never 0.
totalMediaBytes?: number;
- // OF WHICH: the bytes held by `data/<id>/clips/` — the clip windows umtool
- // and the video page fetch a few seconds at a time. A SUBSET of
- // `totalMediaBytes`, not a sibling of it: a window lives under the video dir,
- // `rsync -a` carries it with everything else, and a volume is holding it.
+ // THE TEXT TIER's bytes: everything else under `data/<id>/` — transcripts,
+ // cues, metadata, sidecars, thumbnails, a persisted source container not yet
+ // moved to the store, scratch — but not `clips/`. On the corpus disk always.
+ // Absent on a snapshot written before release 17: "unknown", never 0.
+ totalTextBytes?: number;
+ // The bytes held by `data/<id>/clips/` — the clip windows umtool and the
+ // video page fetch a few seconds at a time. A SIBLING of the two tiers since
+ // release 17 (before it, a subset of `totalMediaBytes`): `clips/` stays on
+ // the corpus disk and is never tiered.
//
// Split out because it is the one part of a channel's bytes that is a CACHE:
// nothing prunes a window, so a channel walked by many reports accumulates
@@ -735,8 +746,13 @@ export async function generateChannelSnapshot(
// the guard does not open config.json a second time for the one field it
// needs. One extra await in front of a function that then walks the whole
// channel.
+ //
+ // THE TEXT GUARD (release 17): the snapshot is a reading of the TEXT tier —
+ // names, sidecars, metadata — so a moving, stalled or unmounted MEDIA drive
+ // does not hold it (its media bytes are then unknown, below). Refused: a
+ // `legacy` channel, whose text is on the far drive.
const config = await readChannelConfig(paths, slug);
- await assertChannelMediaReachable(paths, slug, config);
+ const mediaLocation = await assertChannelTextReadable(paths, slug, config);
// A CHANNEL ON ANOTHER DRIVE IS WALKED THROUGH THE WATCHDOG
// (lib/storageHealth.ts `onDrive`): the data/ listing, the keep-latest keys and
@@ -748,9 +764,18 @@ export async function generateChannelSnapshot(
// scheduler keeps the last good snapshot.json, as on any failed refresh.
// The reconcile pass just below is sequential (one read at a time) and is
// not raced.
- const drive = config?.dataDir?.trim() || undefined;
- const through = <T>(read: () => Promise<T>): Promise<T> =>
- drive ? onDrive(drive, read) : read();
+ //
+ // SINCE RELEASE 17 THE TEXT IS NEVER ON ANOTHER DRIVE: the text guard above
+ // refuses the one layout where it was (`legacy`), so `through` reads
+ // directly. What may be on another drive is the media tier, and its stats
+ // are the only calls that go through the watchdog (per video, below).
+ const through = <T>(read: () => Promise<T>): Promise<T> => read();
+ const mediaDrive = config?.mediaDir?.trim() || undefined;
+ // Whether the media tier may be read at all this run: its links are statted
+ // only when the media is `ok` or `in-place`. Otherwise (unmounted, stalled,
+ // moving, inconsistent) no call is made to it and the bytes are unknown.
+ let mediaBytesKnown =
+ mediaLocation.status === "ok" || mediaLocation.status === "in-place";
// Heal any video dir that drifted from the canonical id layout before we read
// data/* (best-effort; never fail snapshot generation on a reconcile error).
@@ -845,54 +870,71 @@ export async function generateChannelSnapshot(
limit(() => through(async () => {
const dir = path.join(dataDir, id);
const files = await readVideoFiles(dir, { checkUntranscribable: true });
- // EVERY FILE IN THE DIR, STATTED ONCE, feeding two numbers.
+ // EVERY FILE IN THE DIR, `lstat`ED ONCE, feeding the byte figures.
//
// `audioSizes` is what it always was: the real audio files, keyed by
- // name, for the cleanup reclaim estimate. `mediaBytes` is new and is
- // every byte this video dir holds — audio, transcripts, sidecars,
- // thumbnails, a persisted container — because THAT is the number a
- // relocation moves and a volume holds, and the audio total is only a
- // fraction of it (a channel's transcripts, cues and metadata are not
- // free).
- //
- // NOT A SECOND WALK: the readdir is `files.entries`, already in hand,
- // and the audio stats this loop replaces were being paid anyway. What
- // it adds is a stat per NON-audio entry — six to ten per video, warm
- // inode cache, on a pass that already reads several sidecars per video.
+ // name, for the cleanup reclaim estimate. `mediaBytes` is the media
+ // tier's share (what `isTierable` names) and `textBytes` everything
+ // else; `clips/` (CLIPS_DIR_NAME, the fetched windows) is recursed ONE
+ // LEVEL into `clipsBytes`, apart from both — it is never tiered.
//
- // A VIDEO DIR IS NO LONGER FLAT, and exactly one subdirectory is the
- // reason: `clips/` (CLIPS_DIR_NAME), the fetched clip windows. It is
- // recursed ONE LEVEL — the windows and their `.json` sidecars are files,
- // and nothing writes a directory under it — and its bytes are counted
- // BOTH into `clipsBytes` and into `mediaBytes`. Every OTHER directory
- // still counts nothing: there is no other one today, and a blanket
- // recursion here would be the second walk this loop exists to avoid.
- //
- // Why clips count toward `mediaBytes` at all: `mediaBytes` is every
- // byte under `data/<id>/`, which is what the volume is holding and what
- // `rsync -a` carries in a relocation. A window was being left out of
- // both numbers while sitting on the platter.
+ // BY FILE KIND (release 17): a real file's size is its `lstat`, on the
+ // corpus disk, no watchdog. A LINK is a tiered media file whose bytes
+ // are on the media tier — possibly another drive — so the links are
+ // statted together, as ONE call through the watchdog per video
+ // (`onDrive(mediaDir)`), and only while the media is readable. A drive
+ // that does not answer makes the channel's media bytes unknown; the
+ // snapshot is still written.
const audioSet = new Set(files.audioFiles);
const audioSizes: Record<string, number> = {};
let mediaBytes = 0;
+ let textBytes = 0;
let clipsBytes = 0;
+ const linked: string[] = [];
for (const name of files.entries) {
try {
- const st = await stat(path.join(dir, name));
+ const st = await lstat(path.join(dir, name));
+ if (st.isSymbolicLink()) {
+ linked.push(name);
+ continue;
+ }
if (!st.isFile()) {
if (st.isDirectory() && name === CLIPS_DIR_NAME) {
- const bytes = await dirFileBytes(path.join(dir, name));
- clipsBytes += bytes;
- mediaBytes += bytes;
+ clipsBytes += await dirFileBytes(path.join(dir, name));
}
continue;
}
- mediaBytes += st.size;
+ if (isTierable(name)) mediaBytes += st.size;
+ else textBytes += st.size;
if (audioSet.has(name)) audioSizes[name] = st.size;
} catch {
// ignore — file vanished or is unreadable
}
}
+ if (linked.length > 0 && mediaBytesKnown) {
+ const statLinks = async () => {
+ const sizes: Array<[string, number]> = [];
+ for (const name of linked) {
+ try {
+ const st = await stat(path.join(dir, name));
+ if (st.isFile()) sizes.push([name, st.size]);
+ } catch {
+ // a dangling link: its bytes are not on the media tier
+ }
+ }
+ return sizes;
+ };
+ try {
+ const sizes = await (mediaDrive ? onDrive(mediaDrive, statLinks) : statLinks());
+ for (const [name, size] of sizes) {
+ mediaBytes += size;
+ if (audioSet.has(name)) audioSizes[name] = size;
+ }
+ } catch (err) {
+ if (!isDriveNotAnswering(err)) throw err;
+ mediaBytesKnown = false;
+ }
+ }
let nativeId: string | null = null;
if (wantNativeIndex && files.hasMeta) {
try {
@@ -962,6 +1004,7 @@ export async function generateChannelSnapshot(
backfill,
audioSizes,
mediaBytes,
+ textBytes,
clipsBytes,
nativeId,
availability,
@@ -1085,12 +1128,10 @@ export async function generateChannelSnapshot(
// hand on the perVideo entry, so this adds ZERO I/O to a pass that runs over
// ~79,000 videos.
let totalAudioBytes = 0;
- // Every byte under `data/`, not only the audio. The figure the storage
- // surfaces are priced in: how much a volume is holding for this channel, and
- // how much a move would carry.
+ // The media tier's bytes, the text tier's and the clip cache's — three
+ // siblings since release 17 (see the field comments).
let totalMediaBytes = 0;
- // The `clips/` share of the above. Counted in BOTH, deliberately: see the
- // field comment on `totalClipsBytes`.
+ let totalTextBytes = 0;
let totalClipsBytes = 0;
const heldAudioBytes = emptyHeldAudio();
const heldAudioCounts = emptyHeldAudio();
@@ -1108,6 +1149,7 @@ export async function generateChannelSnapshot(
files,
audioSizes,
mediaBytes,
+ textBytes,
clipsBytes,
backfill,
effectiveAvailability,
@@ -1119,6 +1161,7 @@ export async function generateChannelSnapshot(
if (isVideoTranscribed(files)) transcribed++;
if (isVideoDownloaded(files)) downloaded++;
totalMediaBytes += mediaBytes;
+ totalTextBytes += textBytes;
totalClipsBytes += clipsBytes;
// --- Hold attribution ---------------------------------------------------
@@ -1626,8 +1669,9 @@ export async function generateChannelSnapshot(
multipleAudioFormats: multipleAudioFormatsBytes,
foreignAudio: foreignAudioBytes,
},
- totalAudioBytes,
- totalMediaBytes,
+ // Unknown, never a partial sum, when the media tier could not be read.
+ ...(mediaBytesKnown ? { totalAudioBytes, totalMediaBytes } : {}),
+ totalTextBytes,
totalClipsBytes,
heldAudioBytes,
heldAudioCounts,
diff --git a/common/controller/channelWriters.test.ts b/common/controller/channelWriters.test.ts
@@ -214,3 +214,22 @@ test("through the live registry: cancel leaves it stopping, the job's end stamps
assert.equal(typeof record.endedAt, "number");
assert.deepEqual(channelWriters(slug), []);
});
+
+test("mediaOnly (release 17): a digest job and the digest lane are not media writers", () => {
+ const jobs = [
+ job({ id: "J1", kind: "digest-channel-local", channelSlug: "alpha" }),
+ job({ id: "J2", kind: "normalize-transcripts", channelSlug: "alpha" }),
+ job({ id: "J3", kind: "whisper-all", channelSlug: "alpha" }),
+ ];
+ const units = [
+ unit("digest", { videoId: "d1" }),
+ unit("transcription", { videoId: "t1" }),
+ ];
+ const all = channelWriters("alpha", { source: source(jobs, units) });
+ assert.equal(all.length, 5);
+ const media = channelWriters("alpha", { source: source(jobs, units), mediaOnly: true });
+ assert.deepEqual(
+ media.map((w) => (w.source === "job" ? w.jobId : `${w.lane}:${w.videoId}`)),
+ ["J3", "transcription:t1"],
+ );
+});
diff --git a/common/controller/channelWriters.ts b/common/controller/channelWriters.ts
@@ -4,7 +4,7 @@ import {
type JobStatus,
type JobTaskKind,
} from "../jobs/registry";
-import { jobKindLabel } from "../jobs/jobKinds";
+import { jobKindLabel, kindNeedsMedia } from "../jobs/jobKinds";
import { LANES, type AutoQueueKind } from "../lib/autoQueueTypes";
import { getAutoRunnerStatus, type AutoRunnerInFlight } from "./autoRunner";
@@ -82,6 +82,11 @@ export type ChannelWritersOptions = {
// `relocate-channel-media`: it is running on this channel's slug, it is the
// asker, and the relocation queue runs one move at a time.
ignoreKinds?: ReadonlyArray<string>;
+ // MEDIA WRITERS ONLY (release 17): a move of the media tier holds only the
+ // jobs that open or write a big file (`kindNeedsMedia`) and the lanes that
+ // do (every lane but digest). A digest, a normalize, an availability check
+ // reads and writes the text, which never moves, so it may run during a move.
+ mediaOnly?: boolean;
source?: ChannelWritersSource;
};
@@ -96,6 +101,7 @@ export function channelWriters(
const queued: ChannelWriter[] = [];
for (const j of source.jobs()) {
if (j.channelSlug !== slug || ignore.has(j.kind)) continue;
+ if (opts.mediaOnly && !kindNeedsMedia(j.kind)) continue;
// A CANCEL IS A REQUEST, NOT AN EXIT. The registry marks a running job
// `cancelled` the moment it is asked to stop, and stamps `endedAt` when
// its function has returned (registry.ts `finalize`). Until then it is
@@ -124,6 +130,7 @@ export function channelWriters(
const units: ChannelWriter[] = source
.units()
.filter(({ unit }) => unit.channelSlug === slug)
+ .filter(({ lane }) => !opts.mediaOnly || lane !== "digest")
.map(({ lane, unit }) => ({
source: "lane" as const,
lane,
diff --git a/common/controller/channels.ts b/common/controller/channels.ts
@@ -16,7 +16,7 @@ import {
readVideoFiles,
} from "../lib/videoStatus";
import { loadDigest } from "../lib/digest-server";
-import { channelMediaStall, readRelocationMarker } from "../lib/channelMedia";
+import { channelTextStall, readRelocationMarker } from "../lib/channelMedia";
import { isDriveNotAnswering, onDrive } from "../lib/storageHealth";
// TYPE-ONLY, and it must stay that way: ./channelSnapshot imports
// readChannelConfig from this module, and it drags in the snapshot generator's
@@ -88,8 +88,11 @@ async function hasDigestWithItems(videoDir: string): Promise<boolean> {
);
}
-// `drive` is the channel's configured target (`config.dataDir`) when its media
-// is on another drive. Then every read goes through `onDrive`: none while that
+// `drive` is the retired `config.dataDir` of a `legacy` channel, the one layout
+// whose TEXT is on another drive (release 17: a relocated channel's text stays
+// on the corpus disk, and this walk reads only names and text sidecars, so its
+// `mediaDir` is never a reason to wrap it — onDrive is keyed by file kind).
+// For a legacy channel every read goes through `onDrive`: none while that
// location is stalled, at most `inFlightPerLocation` (4) in flight on it, and
// one that has not answered within the budget (`storage.health.budgetMs`, 3 s
// by default) marks it stalled — and the walk answers null ("the drive did
@@ -212,8 +215,9 @@ export async function readChannelStat(
// one-second job-list poll asks this for every channel with a job listed, and
// on a stalled drive each readdir and stat in the walk would hold an I/O
// thread until the drive came back. No counts is what a caller already
- // handles (the row draws no progress bar).
- if (channelMediaStall(config)) return null;
+ // handles (the row draws no progress bar). Only a legacy channel's text is
+ // on a drive that can stall; a stalled MEDIA drive does not stop the count.
+ if (channelTextStall(config)) return null;
const channelDir = path.join(paths.channelsDir, slug);
const counts = await countDataFiles(
path.join(channelDir, "data"),
diff --git a/common/controller/evictClipWindows.test.ts b/common/controller/evictClipWindows.test.ts
@@ -117,21 +117,24 @@ test("a window whose sidecar was never written is still evictable", async () =>
});
});
-test("a channel whose media is in transition is SKIPPED, not evicted from", async () => {
- // `.relocating.json` means there may be two copies and the mover's verify
- // pass compares trees — deleting under it turns a copy that had verified into
- // a failed move of a channel that was fine.
+test("a tier migration in flight is SKIPPED; a media move is not (release 17)", async () => {
+ // A tier migration rebuilds `data/` itself (`scope: "tier-migration"`): its
+ // copy and verify compare the text trees, `clips/` included, so deleting
+ // under it turns a copy that had verified into a failed migration. A media
+ // move carries `media/` only — `clips/` is never in it — so eviction goes on.
await withTmp(async (paths) => {
const clipsDir = await seed(paths, "alpha", {
"10.00-40.00.mp4": { ageDays: 90, size: 1000 },
});
+ const marker = path.join(paths.channelsDir, "alpha", ".relocating.json");
await writeFile(
- path.join(paths.channelsDir, "alpha", ".relocating.json"),
+ marker,
JSON.stringify({
- target: "/mnt/platter/alpha/data",
+ target: "/mnt/platter/alpha/media",
direction: "out",
startedAt: new Date().toISOString(),
phase: "copy",
+ scope: "tier-migration",
}),
);
const r = await evictClipWindows({ paths, olderThanDays: 30 });
@@ -143,6 +146,21 @@ test("a channel whose media is in transition is SKIPPED, not evicted from", asyn
// A skip is the ANSWER, so it reaches the operator in the summary rather
// than being swallowed by a clean-looking zero.
assert.match(evictClipWindowsSummary(r), /Skipped: alpha:/);
+
+ await writeFile(
+ marker,
+ JSON.stringify({
+ target: "/mnt/platter/alpha/media",
+ direction: "out",
+ startedAt: new Date().toISOString(),
+ phase: "copy",
+ scope: "media",
+ }),
+ );
+ const moving = await evictClipWindows({ paths, olderThanDays: 30 });
+ assert.deepEqual(moving.skipped, []);
+ assert.equal(moving.windows, 1);
+ assert.equal(await exists(path.join(clipsDir, "10.00-40.00.mp4")), false);
});
});
@@ -215,7 +233,28 @@ test("an unmounted drive is a SKIP, never a clean eviction of nothing", async ()
assert.equal(r.channels, 0);
assert.equal(r.windows, 0);
assert.equal(r.skipped.length, 1);
- assert.match(r.skipped[0], /^alpha: media unreachable/);
+ // Release 17: a data link is the retired layout, `legacy`, with its way out.
+ assert.match(r.skipped[0], /^alpha: media legacy \(.*migrate-tier alpha\)/);
+ });
+});
+
+test("release 17: an unmounted MEDIA drive does not stop eviction — clips/ is on the corpus disk", async () => {
+ await withTmp(async (paths) => {
+ const clipsDir = await seed(paths, "alpha", {
+ "10.00-40.00.mp4": { ageDays: 90, size: 1000 },
+ });
+ const channelDir = path.join(paths.channelsDir, "alpha");
+ const target = path.join(paths.transcriptsDir, "platter", "alpha", "media");
+ await writeFile(
+ path.join(channelDir, "config.json"),
+ JSON.stringify({ url: "https://example.com/c", mediaDir: target }),
+ );
+ // The media link dangles: the media drive is not mounted.
+ await symlink(target, path.join(channelDir, "media"));
+ const r = await evictClipWindows({ paths, olderThanDays: 30 });
+ assert.deepEqual(r.skipped, []);
+ assert.equal(r.windows, 1);
+ assert.equal(await exists(path.join(clipsDir, "10.00-40.00.mp4")), false);
});
});
diff --git a/common/controller/evictClipWindows.ts b/common/controller/evictClipWindows.ts
@@ -82,24 +82,24 @@ async function evictChannel(
): Promise<void> {
const channelDir = path.join(opts.paths.channelsDir, slug);
- // AN UNMOUNTED DRIVE IS NOT AN EMPTY CHANNEL, and `inspectChannelMedia` is
- // the one module that can tell the two apart (AGENTS.md: a path that reads
+ // AN UNREADABLE TEXT TIER IS NOT AN EMPTY CHANNEL, and `inspectChannelMedia`
+ // is the one module that can tell the two apart (AGENTS.md: a path that reads
// `data/` guards there). Every enumerator else swallows ENOENT as "no
- // videos" — which here would report a clean eviction of zero bytes about a
- // platter full of windows, and an operator would read that as "nothing to
- // reclaim".
+ // videos" — which here would report a clean eviction of zero bytes, and an
+ // operator would read that as "nothing to reclaim".
//
- // `in-transition` is a refusal for a sharper reason than unreachability:
- // there may be two copies, `data/` may be a link whose target is half
- // written, and the mover's verify pass compares trees. Deleting under it
- // turns a copy that had verified into a failed move of a channel that was
- // fine. (The JOB also declares `needsMedia: true`, which covers a
- // single-channel run before it starts; this covers the corpus-wide one,
- // where there is no slug for that guard to check.)
+ // THE TEXT GUARD (release 17): `clips/` is on the corpus disk and is never
+ // tiered, so a channel whose MEDIA is moving, stalled or unmounted is evicted
+ // as usual — a media move carries `media/`, never `data/`. Skipped: a
+ // `legacy` channel (the retired whole-directory layout, its `data/` — clips
+ // included — on the far drive, where a move's verify compares trees) and a
+ // tier migration in flight, which rebuilds `data/` itself. (The JOB also
+ // declares `needsText`, which covers a single-channel run before it starts;
+ // this covers the corpus-wide one, where there is no slug for that guard.)
const media = await inspectChannelMedia(opts.paths, slug, undefined, {
fresh: true,
});
- if (media.status !== "ok" && media.status !== "in-place") {
+ if (!media.text.readable) {
out.skipped.push(
`${slug}: media ${media.status}${media.detail ? ` (${media.detail})` : ""} — nothing was touched`,
);
@@ -111,8 +111,8 @@ async function evictChannel(
try {
videoIds = await readdir(dataDir);
} catch {
- // Past the guard above this really is "no videos": a channel whose media
- // is in place and whose `data/` has never been created.
+ // Past the guard above this really is "no videos": a channel whose text
+ // tier is on the corpus disk and whose `data/` has never been created.
out.channels += 1;
return;
}
diff --git a/common/controller/normalizeAll.ts b/common/controller/normalizeAll.ts
@@ -9,7 +9,7 @@ import pLimit from "p-limit";
import { listChannelStatsFromDisk } from "./channels";
import { normalizeTranscript } from "./normalizeTranscript";
import type { Paths } from "../lib/paths";
-import { assertChannelMediaReachable } from "../lib/channelMedia";
+import { assertChannelTextReadable } from "../lib/channelMedia";
export type NormalizeAllOptions = {
paths: Paths;
@@ -59,8 +59,10 @@ export async function normalizeAllTranscripts(
// "this channel's drive is not mounted", and here the second reads as a
// clean run over zero videos that reports 0/0/0/0 and moves on. A sweep
// that silently skips a channel is worse than one that stops on it.
+ // THE TEXT GUARD (release 17): normalize reads and writes text only, so a
+ // moving, stalled or unmounted MEDIA drive does not stop it.
try {
- await assertChannelMediaReachable(opts.paths, ch.slug, ch.config);
+ await assertChannelTextReadable(opts.paths, ch.slug, ch.config);
} catch (err) {
log(`Normalize ${ch.slug}: SKIPPED — ${(err as Error).message}`);
result.failed++;
diff --git a/common/controller/operationBatch.ts b/common/controller/operationBatch.ts
@@ -40,6 +40,7 @@ import { getSettings, type SiteSettings } from "../lib/settings";
import { isGateHeld } from "../lib/pauseGates";
import {
assertChannelMediaReachable,
+ assertChannelTextReadable,
readRelocationMarker,
} from "../lib/channelMedia";
import { runPool } from "../jobs/concurrentRunner";
@@ -1590,7 +1591,16 @@ export async function runOperationBatch(
// of this function's callers are already behind guard 1 today; this is here so
// a fourth in-process caller that is not cannot quietly find "no candidates"
// on a channel whose drive is unmounted.
- await assertChannelMediaReachable(opts.paths, opts.channelSlug);
+ //
+ // BY LANE (release 17): the digest lane reads only the text tier, so it asks
+ // the text guard and runs while the channel's media is moving, stalled or
+ // unmounted; the backfill lane's operations open the audio and keep the
+ // media guard.
+ if (opts.lane === "digest") {
+ await assertChannelTextReadable(opts.paths, opts.channelSlug);
+ } else {
+ await assertChannelMediaReachable(opts.paths, opts.channelSlug);
+ }
const dataDir = path.join(opts.paths.channelsDir, opts.channelSlug, "data");
const allDirs = await readdir(dataDir).catch(() => [] as string[]);
const onDisk = new Set(allDirs);
diff --git a/common/controller/recencyIndex.ts b/common/controller/recencyIndex.ts
@@ -196,8 +196,12 @@ async function readTailUploadDate(file: string): Promise<string | null> {
// Date the ids in `wanted` from their on-disk metadata. Removes each id it
// keys from `wanted`, like interpolateFromPlaylist.
//
-// `drives` maps a channel whose media is on another drive to its configured
-// target. Its reads go through `onDrive`: none while that drive's location is
+// `drives` maps a channel whose TEXT is on another drive — a `legacy` channel,
+// the retired whole-directory layout, keyed by its `dataDir` — to that target.
+// Since release 17 a relocated channel's text (`metadata.info.json` here) is on
+// the corpus disk and only its media is on the far drive, so its reads pass no
+// drive at all: onDrive is keyed by file kind. A legacy channel's reads go
+// through `onDrive`: none while that drive's location is
// stalled (lib/storageHealth.ts) — each would hold an I/O thread until the drive
// came back, 32 at a time — at most `inFlightPerLocation` (4) in flight on it,
// and one that has not answered within the budget (3 s by default) marks it
@@ -349,7 +353,7 @@ export type BuildRecencyKeysArgs = {
// are on another drive (their reads go through the stall watchdog).
meta: ReadonlyArray<{
slug: string;
- config?: Pick<ChannelConfig, "dataDir"> | null;
+ config?: Pick<ChannelConfig, "dataDir" | "mediaDir"> | null;
}>;
// Every video id the caller might sort. Ids outside this set are not keyed.
candidateIds: ReadonlySet<string>;
@@ -508,6 +512,7 @@ export async function buildRecencyKeys({
if (owner && missing.size > 0) {
const drives = new Map<string, string>();
for (const m of meta) {
+ // The retired `dataDir` only: `mediaDir` holds no metadata.
const dir = m.config?.dataDir?.trim();
if (dir) drives.set(m.slug, dir);
}
diff --git a/common/controller/relocateChannelMedia.test.ts b/common/controller/relocateChannelMedia.test.ts
@@ -42,6 +42,11 @@ import {
} from "../lib/storageVolumes";
import { readChannelConfig } from "./channels";
+// RELEASE 17 SLICE T1 made a channel whose `data/` is a link (or whose config
+// carries `dataDir`) `legacy`; these cases still build that retired layout and
+// expect it to read `ok`. Slice T2 rebases them on `media/` and un-skips them.
+const T1_SKIP = "release 17 T2 rebases the mover on media/";
+
// Run with:
// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/relocateChannelMedia.test.ts
//
@@ -135,7 +140,7 @@ async function seed(
return channelDir;
}
-test("out: copies, links, records the target, keeps mtimes and reclaims the source", async () => {
+test("out: copies, links, records the target, keeps mtimes and reclaims the source", { skip: T1_SKIP }, async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500), "transcript.json": "{}" },
@@ -192,7 +197,7 @@ test("out: copies, links, records the target, keeps mtimes and reclaims the sour
});
});
-test("abort from an onLog hook leaves the source intact, and the rerun completes", async () => {
+test("abort from an onLog hook leaves the source intact, and the rerun completes", { skip: T1_SKIP }, async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "x".repeat(20000) },
@@ -329,7 +334,7 @@ test("a verify failure keeps the source and does not write the config", async ()
});
});
-test("back: restores a real directory, clears the config and reclaims the target", async () => {
+test("back: restores a real directory, clears the config and reclaims the target", { skip: T1_SKIP }, async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one", "transcript.json": "{}" },
@@ -557,7 +562,7 @@ async function copyTree(src: string, dest: string): Promise<void> {
}
}
-test("out @ swap: crash before the rename — the rerun re-verifies and completes", async () => {
+test("out @ swap: crash before the rename — the rerun re-verifies and completes", { skip: T1_SKIP }, async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
@@ -583,7 +588,7 @@ test("out @ swap: crash before the rename — the rerun re-verifies and complete
});
});
-test("out @ swap: crash after the config write — the first run's parked copy is still reclaimed", async () => {
+test("out @ swap: crash after the config write — the first run's parked copy is still reclaimed", { skip: T1_SKIP }, async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
@@ -621,7 +626,7 @@ test("out @ swap: crash after the config write — the first run's parked copy i
});
});
-test("out @ reclaim: the rerun sweeps every parked copy and clears the marker", async () => {
+test("out @ reclaim: the rerun sweeps every parked copy and clears the marker", { skip: T1_SKIP }, async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
@@ -832,7 +837,7 @@ async function pathIsThere(p: string): Promise<boolean> {
// the run copied the target to data.incoming, verified it, found data/ already a
// real dir, logged "the swap had completed", cleared the config, rm -r'd the
// target and then swept data.incoming. Three copies in, zero out.
-test("back: an inconsistent channel is refused, and the target keeps its bytes", async () => {
+test("back: an inconsistent channel is refused, and the target keeps its bytes", { skip: T1_SKIP }, async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one" },
@@ -886,7 +891,7 @@ test("back: an inconsistent channel is refused, and the target keeps its bytes",
// An `unreachable` channel (the drive is not mounted) is refused for the same
// reason: nothing can vouch for what the target holds, and the run ends by
// deleting it.
-test("back: an unreachable channel is refused", async () => {
+test("back: an unreachable channel is refused", { skip: T1_SKIP }, async () => {
await withTmp(async (paths, root) => {
await seed(paths, "alpha", { v1: { "audio.m4a": "one" } });
const target = relocatedDataDir(root, "alpha");
@@ -1020,7 +1025,7 @@ test("a relative root is refused by the preview, not only by the job", async ()
// `rsync -a` ahead of it. That is the shape that can see the difference: in the
// copy phase the transfer itself would have set the timestamps.
-test("out @ swap: a directory timestamp is settled by one more pass, not refused", async () => {
+test("out @ swap: a directory timestamp is settled by one more pass, not refused", { skip: T1_SKIP }, async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500), "transcript.json": "{}" },
@@ -1067,7 +1072,7 @@ test("out @ swap: a directory timestamp is settled by one more pass, not refused
// still the live directory, and one change gets one more mirror pass, exactly
// as in the copy phase. A second change is still a refusal ("a verify failure
// keeps the source …" above).
-test("out @ swap: a file the target is missing is mirrored by one more pass", async () => {
+test("out @ swap: a file the target is missing is mirrored by one more pass", { skip: T1_SKIP }, async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(500) },
@@ -1242,7 +1247,7 @@ test("a writer seen once the marker is written refuses, and a fresh move's marke
// on the destination that has since gone from the source. The resume used to
// copy everything else and refuse on the counts (1755 against 1750), and no
// rerun could settle it; the mirror pass now does.
-test("a resume with a stale extra dir on the destination completes", async () => {
+test("a resume with a stale extra dir on the destination completes", { skip: T1_SKIP }, async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v50t5yt: { "audio.mp3": "a".repeat(64), "transcript.json": "{}" },
@@ -1313,7 +1318,7 @@ test("back: a stale extra on the copy coming home is removed on resume", async (
// RECONCILE AND RESUME — the remediation (the ruling's last bullet). An extra
// file and a changed one on the destination: the job says what it found, by
// kind, makes the copy match the source and finishes the move.
-test("reconcile: an extra and a changed file on the destination are settled, and the move completes", async () => {
+test("reconcile: an extra and a changed file on the destination are settled, and the move completes", { skip: T1_SKIP }, async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(100), "transcript.json": '{"v":2}' },
@@ -1355,7 +1360,7 @@ test("reconcile: an extra and a changed file on the destination are settled, and
// A marker past the copy phase has nothing to reconcile: the run is a plain
// resume, and says so (the review's L3).
-test("reconcile: a marker past the copy phase resumes, and does not claim a reconcile", async () => {
+test("reconcile: a marker past the copy phase resumes, and does not claim a reconcile", { skip: T1_SKIP }, async () => {
await withTmp(async (paths, root) => {
const channelDir = await seed(paths, "alpha", {
v1: { "audio.m4a": "one".repeat(100) },
@@ -1412,6 +1417,7 @@ test("reconcile: with no marker there is nothing to reconcile", async () => {
const FAKE_FINDMNT = `#!/usr/bin/env node
import { readFileSync } from "node:fs";
import path from "node:path";
+
const control = JSON.parse(
readFileSync(path.join(import.meta.dirname, "control.json"), "utf8"),
);
diff --git a/common/controller/renameChannel.test.ts b/common/controller/renameChannel.test.ts
@@ -37,6 +37,12 @@ import {
RELOCATION_MARKER_FILENAME,
} from "../lib/channelMedia";
+// RELEASE 17 SLICE T1 made a channel whose `data/` is a link (or whose config
+// carries `dataDir`) `legacy`; these cases still build that retired layout and
+// expect it to read `ok`. Slice T2 rebases them on `media/` and un-skips them.
+const T1_SKIP = "release 17 T2 rebases the mover on media/";
+
+
// Run with:
// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/renameChannel.test.ts
@@ -150,7 +156,7 @@ test("renameChannel rejects invalid, same, and existing targets", async () => {
// --- relocated media ------------------------------------------------------
-test("rename re-points a convention-shaped relocated media dir", async () => {
+test("rename re-points a convention-shaped relocated media dir", { skip: T1_SKIP }, async () => {
await withPaths(async (paths) => {
const dir = path.dirname(paths.channelsDir);
const mediaRoot = path.join(dir, "platter");
diff --git a/common/controller/storageLocations.test.ts b/common/controller/storageLocations.test.ts
@@ -28,6 +28,12 @@ import {
} from "./storageLocations";
import { readChannelConfig } from "./channels";
+// RELEASE 17 SLICE T1 made a channel whose `data/` is a link (or whose config
+// carries `dataDir`) `legacy`; these cases still build that retired layout and
+// expect it to read `ok`. Slice T2 rebases them on `media/` and un-skips them.
+const T1_SKIP = "release 17 T2 rebases the mover on media/";
+
+
// Run with:
// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/storageLocations.test.ts
//
@@ -145,7 +151,7 @@ function loc(id: string, root: string, extra: Partial<StorageLocation> = {}): St
return { id, label: id, root, autoRepoint: false, ...extra };
}
-test("channelsOnLocation buckets ok / unreachable / moving and ignores channels elsewhere", async () => {
+test("channelsOnLocation buckets ok / unreachable / moving and ignores channels elsewhere", { skip: T1_SKIP }, async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
// Media dir never created: the link dangles, which is what an unmounted
@@ -180,7 +186,7 @@ test("channelsOnLocation buckets ok / unreachable / moving and ignores channels
});
});
-test("re-point rewrites both channels' links and configs, then the location", async () => {
+test("re-point rewrites both channels' links and configs, then the location", { skip: T1_SKIP }, async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
await seedRelocated(h, "beta", h.rootA);
@@ -227,7 +233,7 @@ test("re-point rewrites both channels' links and configs, then the location", as
});
});
-test("re-point refuses a target that has no media for a channel, naming the slug", async () => {
+test("re-point refuses a target that has no media for a channel, naming the slug", { skip: T1_SKIP }, async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
await seedRelocated(h, "beta", h.rootA);
@@ -263,7 +269,7 @@ test("re-point refuses a target that has no media for a channel, naming the slug
});
});
-test("a failure on the second channel rolls the first one back", async () => {
+test("a failure on the second channel rolls the first one back", { skip: T1_SKIP }, async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
await seedRelocated(h, "beta", h.rootA);
@@ -306,7 +312,7 @@ test("a failure on the second channel rolls the first one back", async () => {
});
});
-test("re-point refuses a busy channel and names it", async () => {
+test("re-point refuses a busy channel and names it", { skip: T1_SKIP }, async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
await mkdir(path.join(h.rootB, "alpha", "data"), { recursive: true });
@@ -324,7 +330,7 @@ test("re-point refuses a busy channel and names it", async () => {
});
});
-test("a rerun after a crash finishes the channels that were left", async () => {
+test("a rerun after a crash finishes the channels that were left", { skip: T1_SKIP }, async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
await seedRelocated(h, "beta", h.rootA);
@@ -375,7 +381,7 @@ test("a rerun after a crash finishes the channels that were left", async () => {
});
});
-test("a channel killed between its symlink and its config write is resumed, not refused", async () => {
+test("a channel killed between its symlink and its config write is resumed, not refused", { skip: T1_SKIP }, async () => {
await withTmp(async (h) => {
await seedRelocated(h, "alpha", h.rootA);
await seedRelocated(h, "beta", h.rootA);
diff --git a/common/controller/storageStall.test.ts b/common/controller/storageStall.test.ts
@@ -2,6 +2,14 @@
// storage location's drive in-process asks the health state first
// (lib/storageHealth.ts) and, on a stalled location, answers WITHOUT the call.
//
+// RELEASE 17, THE MEDIA TIER: a relocated channel's TEXT is on the corpus disk
+// and only its media is on the drive (`channels/<slug>/media` -> the drive,
+// one relative link per big file in `data/<id>/`). So a stalled drive holds
+// what opens a big file and nothing that reads text: the channel's counts,
+// its recency, its snapshot are read as usual, with no call on the drive. The
+// cases that need a channel whose TEXT is on the drive use the retired
+// whole-directory layout (`seedLegacy`), the one layout where it still is.
+//
// No test stalls a real drive. The drive here is an ordinary temp directory
// that answers every call at once; the health state is TOLD it is stalled.
// Every node:fs and node:fs/promises call this file's code makes is recorded
@@ -19,6 +27,8 @@ import {
existsSync,
mkdirSync,
mkdtempSync,
+ readFileSync,
+ renameSync,
rmSync,
symlinkSync,
writeFileSync,
@@ -31,6 +41,7 @@ import type { StorageLocation } from "../lib/storageLocations";
import {
assertChannelMediaReachable,
ChannelMediaUnreachableError,
+ ChannelTextUnreadableError,
CHANNEL_MEDIA_MEMO_MS,
clearRelocationMarker,
forgetChannelMedia,
@@ -128,30 +139,45 @@ const LOC: StorageLocation = {
root: DRIVE,
autoRepoint: false,
};
-const TARGET = path.join(DRIVE, SLUG, "data");
-const LINK = path.join(paths.channelsDir, SLUG, "data");
-const CONFIG = { dataDir: TARGET };
+// The media tier on the drive, and its one link: channels/<slug>/media.
+const TARGET = path.join(DRIVE, SLUG, "media");
+const LINK = path.join(paths.channelsDir, SLUG, "media");
+// The text, on the corpus disk; vid1's audio is a tiered link into media/.
+const DATA = path.join(paths.channelsDir, SLUG, "data");
+const TIERED = path.join(DATA, "vid1", "audio.mp3");
+const CONFIG = { mediaDir: TARGET };
+// The retired layout: data/ itself a link to the drive.
+const LEGACY_TARGET = path.join(DRIVE, SLUG, "data");
+const LEGACY_CONFIG = { dataDir: LEGACY_TARGET };
+let legacy = false;
const FINDMNT_MARK = path.join(ROOT, "findmnt-ran");
-function seed(): void {
- rmSync(CORPUS, { recursive: true, force: true });
- rmSync(DRIVE, { recursive: true, force: true });
- mkdirSync(path.join(TARGET, "vid1"), { recursive: true });
- writeFileSync(
- path.join(TARGET, "vid1", "metadata.info.json"),
- JSON.stringify({ id: "vid1", upload_date: "20260601" }),
- );
- writeFileSync(path.join(TARGET, "vid1", "transcript.en.vtt"), "WEBVTT\n");
- mkdirSync(path.join(paths.channelsDir, SLUG), { recursive: true });
+function writeConfig(extra: Record<string, unknown>): void {
writeFileSync(
path.join(paths.channelsDir, SLUG, "config.json"),
JSON.stringify({
handling: "youtube",
name: SLUG,
url: `https://www.youtube.com/@${SLUG}/videos`,
- dataDir: TARGET,
+ ...extra,
}),
);
+}
+
+function seed(): void {
+ legacy = false;
+ rmSync(CORPUS, { recursive: true, force: true });
+ rmSync(DRIVE, { recursive: true, force: true });
+ mkdirSync(path.join(DATA, "vid1"), { recursive: true });
+ writeFileSync(
+ path.join(DATA, "vid1", "metadata.info.json"),
+ JSON.stringify({ id: "vid1", upload_date: "20260601" }),
+ );
+ writeFileSync(path.join(DATA, "vid1", "transcript.en.vtt"), "WEBVTT\n");
+ mkdirSync(path.join(TARGET, "vid1"), { recursive: true });
+ writeFileSync(path.join(TARGET, "vid1", "audio.mp3"), "AUDIO");
+ symlinkSync(path.join("..", "..", "media", "vid1", "audio.mp3"), TIERED);
+ writeConfig({ mediaDir: TARGET });
symlinkSync(TARGET, LINK);
writeFileSync(paths.findmntBin, `#!/bin/sh\ntouch ${FINDMNT_MARK}\nexit 1\n`, {
mode: 0o755,
@@ -159,12 +185,29 @@ function seed(): void {
rmSync(FINDMNT_MARK, { force: true });
}
-// Anything that reaches the drive: a path under its root, or through the
-// channel's link (`data/` itself, followed, or anything below it).
+// The channel on the retired layout: its whole data/ on the drive, `data` an
+// absolute link to it, `dataDir` recorded (and no media tier).
+function seedLegacy(): void {
+ rmSync(LINK);
+ rmSync(TIERED);
+ writeFileSync(path.join(DATA, "vid1", "audio.mp3"), "AUDIO");
+ renameSync(DATA, LEGACY_TARGET);
+ symlinkSync(LEGACY_TARGET, DATA);
+ writeConfig({ dataDir: LEGACY_TARGET });
+ legacy = true;
+}
+
+// Anything that reaches the drive: a path under its root; through the media
+// link; a tiered file opened or statted through its link (an lstat or a
+// readlink of the link itself is on the corpus disk); and, on the legacy
+// layout, anything through `data/`.
function onDrive(c: Call): boolean {
const under = (p: string, base: string) =>
p === base || p.startsWith(base + path.sep);
- return under(c.path, DRIVE) || under(c.path, LINK);
+ if (under(c.path, DRIVE) || under(c.path, LINK)) return true;
+ const ofTheLink = ["lstat", "lstatSync", "readlink", "readlinkSync"].includes(c.fn);
+ if (c.path === TIERED && !ofTheLink) return true;
+ return legacy && under(c.path, DATA) && !(c.path === DATA && ofTheLink);
}
function stall(): void {
@@ -190,13 +233,18 @@ test("inspect with the config in hand: on a stalled drive only the marker is rea
stall();
calls = [];
const media = await inspectChannelMedia(paths, SLUG, CONFIG);
- assert.deepEqual(calls, [{ fn: "readFile", path: MARKER }]);
+ assert.deepEqual(calls, [
+ { fn: "lstat", path: DATA },
+ { fn: "readFile", path: MARKER },
+ ]);
assert.deepEqual(calls.filter(onDrive), []);
assert.equal(media.status, "stalled");
assert.equal(media.target, TARGET);
assert.match(String(media.detail), /^drive not answering \(location "USB drive", since /);
- // Held by both pool-wide builds, with a reason that names no path.
+ // Held for the media, with a reason that names no path — and its text is
+ // readable, so the builds do not hold it (release 17).
assert.equal(isMediaHeld(media.status), true);
+ assert.equal(media.text.readable, true);
assert.doesNotMatch(HELD_REASON.stalled, /\//);
});
@@ -209,6 +257,7 @@ test("inspect without the config reads config.json and nothing on the drive", as
calls.map((c) => [c.fn, path.relative(ROOT, c.path)]),
[
["readFile", path.join("corpus", "channels", SLUG, "config.json")],
+ ["lstat", path.join("corpus", "channels", SLUG, "data")],
["readFile", path.join("corpus", "channels", SLUG, ".relocating.json")],
],
);
@@ -268,7 +317,7 @@ test("the memo is keyed by the configured target, and a mover's forget clears it
calls.filter((c) => c.fn === "stat" && c.path === TARGET).length;
await inspectChannelMedia(paths, SLUG, CONFIG, { now });
// Another configured target is another key.
- const other = await inspectChannelMedia(paths, SLUG, { dataDir: path.join(DRIVE, "x", "data") }, { now });
+ const other = await inspectChannelMedia(paths, SLUG, { mediaDir: path.join(DRIVE, "x", "media") }, { now });
assert.equal(other.status, "inconsistent");
forgetChannelMedia(SLUG);
await inspectChannelMedia(paths, SLUG, CONFIG, { now });
@@ -316,7 +365,20 @@ test("volumeFreeBytes: a stalled location reads unknown, with no call on it", as
assert.deepEqual(calls.filter(onDrive), []);
});
-test("readChannelStat: no walk of data/ on a stalled drive", async () => {
+test("readChannelStat: a stalled MEDIA drive does not stop the count, and is not asked", async () => {
+ const before = await readChannelStat(paths, SLUG);
+ assert.equal(before?.videoCount, 1);
+ assert.equal(before?.downloadCount, 1);
+ stall();
+ calls = [];
+ const during = await readChannelStat(paths, SLUG);
+ assert.equal(during?.videoCount, 1);
+ assert.equal(during?.downloadCount, 1, "the tiered audio is a name in the listing");
+ assert.deepEqual(calls.filter(onDrive), []);
+});
+
+test("readChannelStat: no walk of a LEGACY channel's data/ on a stalled drive", async () => {
+ seedLegacy();
const before = await readChannelStat(paths, SLUG);
assert.equal(before?.videoCount, 1);
stall();
@@ -325,9 +387,26 @@ test("readChannelStat: no walk of data/ on a stalled drive", async () => {
assert.deepEqual(calls.filter(onDrive), []);
});
-test("recency: no tail read on a stalled drive, and no miss remembered for it", async () => {
+test("recency: a stalled MEDIA drive does not stop the tail read (metadata is text)", async () => {
+ const args = {
+ paths,
+ meta: [{ slug: SLUG, config: CONFIG }],
+ candidateIds: new Set(["vid1"]),
+ owner: new Map([["vid1", SLUG]]),
+ interpolate: false,
+ fresh: true,
+ };
+ stall();
+ calls = [];
+ const keys = await buildRecencyKeys(args);
+ assert.deepEqual(keys.get("vid1"), { key: "20260601", estimated: false });
+ assert.deepEqual(calls.filter(onDrive), []);
+});
+
+test("recency: no tail read on a LEGACY channel's stalled drive, and no miss remembered for it", async () => {
+ seedLegacy();
const owner = new Map([["vid1", SLUG]]);
- const meta = [{ slug: SLUG, config: CONFIG }];
+ const meta = [{ slug: SLUG, config: LEGACY_CONFIG }];
const args = {
paths,
meta,
@@ -363,13 +442,32 @@ test("a move onto a stalled location is refused without a stat of its root", asy
assert.deepEqual(calls.filter(onDrive), []);
});
-test("a snapshot refresh of a stalled channel throws before its walk", async () => {
+test("a snapshot refresh of a stalled-MEDIA channel is written from its text; its media bytes are unknown", async () => {
+ const ok = await generateChannelSnapshot(paths, SLUG);
+ assert.equal(ok.totalMediaBytes, 5, "the tiered audio, statted through its link");
+ assert.equal(typeof ok.totalTextBytes, "number");
+ stall();
+ calls = [];
+ const snap = await generateChannelSnapshot(paths, SLUG);
+ assert.deepEqual(calls.filter(onDrive), []);
+ assert.equal(snap.totals.downloaded, 1);
+ assert.equal("totalMediaBytes" in snap, false, "unknown, never 0");
+ assert.equal("totalAudioBytes" in snap, false);
+ assert.equal(typeof snap.totalTextBytes, "number");
+ const onDisk = JSON.parse(
+ readFileSync(path.join(paths.channelsDir, SLUG, "snapshot.json"), "utf8"),
+ ) as { totalMediaBytes?: number };
+ assert.equal(onDisk.totalMediaBytes, undefined);
+});
+
+test("a snapshot refresh of a LEGACY channel throws before its walk", async () => {
+ seedLegacy();
stall();
calls = [];
await assert.rejects(
() => generateChannelSnapshot(paths, SLUG),
(err: unknown) =>
- err instanceof ChannelMediaUnreachableError && err.status === "stalled",
+ err instanceof ChannelTextUnreadableError && err.status === "legacy",
);
assert.deepEqual(calls.filter(onDrive), []);
});
@@ -395,13 +493,15 @@ test("the saved-video store on a stalled drive reads unreachable, without a stat
assert.deepEqual(followed, []);
});
-test("the saved-video inventory, for a page: a stalled channel is named, not read", async () => {
+test("the saved-video inventory, for a page: a stalled LEGACY channel is named, not read", async () => {
// A saved video on the drive, so a channel that WAS read has an entry.
// (savedVideoInventory reads through fs-extra, whose functions graceful-fs
// captured before this file's spy was installed, so this case proves the skip
- // by what comes back, not by the spy.)
+ // by what comes back, not by the spy.) The pointer is text: only on the
+ // retired layout is it on the drive at all.
+ seedLegacy();
writeFileSync(
- path.join(TARGET, "vid1", "saved-video.json"),
+ path.join(LEGACY_TARGET, "vid1", "saved-video.json"),
JSON.stringify({ storedAt: "", dir: "/store", file: "v.mp4", bytes: 7 }),
);
assert.equal((await listSavedVideos({ paths, notAnswering: [] })).length, 1);
@@ -444,11 +544,12 @@ test("watchdog: inspect's target stat never answers → stalled, marked, and not
)) as { status: string; detail?: string };
assert.equal(media.status, "stalled");
assert.match(String(media.detail), /^drive not answering \(location "USB drive"/);
- // The next inspect, the guard and the walk make no call on the drive.
+ // The next inspect, the guard and the walk make no call on the drive — and
+ // the walk, which reads text, still counts (release 17).
calls = [];
assert.equal((await inspectChannelMedia(paths, SLUG, CONFIG)).status, "stalled");
await assert.rejects(() => assertChannelMediaReachable(paths, SLUG, CONFIG));
- assert.equal(await readChannelStat(paths, SLUG), null);
+ assert.equal((await readChannelStat(paths, SLUG))?.videoCount, 1);
assert.deepEqual(calls.filter(onDrive), []);
// Cleared (two clean answers), the drive is asked again.
recordLocationHealth(LOC, "ok");
@@ -460,20 +561,21 @@ test("watchdog: inspect's target stat never answers → stalled, marked, and not
);
});
-test("watchdog: a video directory's read never answers mid-walk → readChannelStat answers null", async () => {
+test("watchdog: a LEGACY video directory's read never answers mid-walk → readChannelStat answers null", async () => {
+ seedLegacy();
for (const id of ["vid2", "vid3", "vid4", "vid5", "vid6"]) {
- mkdirSync(path.join(TARGET, id), { recursive: true });
+ mkdirSync(path.join(LEGACY_TARGET, id), { recursive: true });
}
const out = await watchdogCase(["readdir"], async () => {
// The walk's first readdir (of data/ itself, through the link) answers;
// the video directories' do not.
hang = (c) =>
- c.fn === "readdir" && c.path.startsWith(path.join(LINK, "vid"));
+ c.fn === "readdir" && c.path.startsWith(path.join(DATA, "vid"));
return readChannelStat(paths, SLUG);
});
assert.equal(out, null);
// At most four video directories were asked before the stall refused the rest.
- const asked = calls.filter((c) => c.fn === "readdir" && c.path.startsWith(path.join(LINK, "vid")));
+ const asked = calls.filter((c) => c.fn === "readdir" && c.path.startsWith(path.join(DATA, "vid")));
assert.ok(asked.length <= 4, `${asked.length} video dirs asked`);
});
@@ -491,10 +593,11 @@ test("watchdog: volumeFreeBytes' stat never answers → unknown", async () => {
assert.equal(out.usb, undefined);
});
-test("watchdog: a recency tail read never answers → not dated, not remembered as a miss", async () => {
+test("watchdog: a LEGACY recency tail read never answers → not dated, not remembered as a miss", async () => {
+ seedLegacy();
const args = {
paths,
- meta: [{ slug: SLUG, config: CONFIG }],
+ meta: [{ slug: SLUG, config: LEGACY_CONFIG }],
candidateIds: new Set(["vid1"]),
owner: new Map([["vid1", SLUG]]),
interpolate: false,
@@ -521,24 +624,26 @@ test("watchdog: a move onto a root whose stat never answers is refused", async (
assert.match(String(problem), /drive not answering/);
});
-test("watchdog (M3): the snapshot walk's video unit never answers → the refresh throws and writes no snapshot", async () => {
+test("watchdog (M3, release 17): a tiered file's stat never answers → the snapshot is written, its media bytes unknown", async () => {
+ // Six more videos, each with a tiered audio link, so the walk has units
+ // past the first to refuse.
for (const id of ["vid2", "vid3", "vid4", "vid5", "vid6", "vid7"]) {
+ mkdirSync(path.join(DATA, id), { recursive: true });
+ writeFileSync(path.join(DATA, id, "metadata.info.json"), JSON.stringify({ id }));
mkdirSync(path.join(TARGET, id), { recursive: true });
+ writeFileSync(path.join(TARGET, id, "audio.mp3"), "AUDIO");
+ symlinkSync(path.join("..", "..", "media", id, "audio.mp3"), path.join(DATA, id, "audio.mp3"));
}
- const snapshotFile = path.join(paths.channelsDir, SLUG, "snapshot.json");
- await watchdogCase(["readdir"], async () => {
- // data/ itself answers (the listing, the reconcile pass); the video
- // directories do not.
- hang = (c) => c.fn === "readdir" && c.path.startsWith(path.join(LINK, "vid"));
- await assert.rejects(
- () => generateChannelSnapshot(paths, SLUG),
- (err: unknown) => err instanceof Error && err.name === "DriveNotAnsweringError",
- );
- });
- assert.equal(existsSync(snapshotFile), false, "the last snapshot.json stands");
- // At most four video directories reached the drive.
- const asked = calls.filter(
- (c) => c.fn === "readdir" && c.path.startsWith(path.join(LINK, "vid")),
- );
- assert.ok(asked.length <= 4, `${asked.length} video dirs asked`);
+ const tiered = (c: Call) =>
+ c.fn === "stat" && c.path.startsWith(DATA + path.sep) && c.path.endsWith(`${path.sep}audio.mp3`);
+ const snap = (await watchdogCase(["stat"], async () => {
+ // The text answers; a stat through a tiered link does not.
+ hang = tiered;
+ return generateChannelSnapshot(paths, SLUG);
+ })) as Awaited<ReturnType<typeof generateChannelSnapshot>>;
+ assert.equal(snap.totals.downloaded, 7);
+ assert.equal("totalMediaBytes" in snap, false, "unknown, never a partial sum");
+ // At most four video units reached the drive.
+ const asked = new Set(calls.filter(tiered).map((c) => path.dirname(c.path)));
+ assert.ok(asked.size <= 4, `${asked.size} video units asked`);
});
diff --git a/common/controller/storageWatch.test.ts b/common/controller/storageWatch.test.ts
@@ -30,6 +30,12 @@ import {
type LocationHealthState,
} from "../lib/storageHealth";
+// RELEASE 17 SLICE T1 made a channel whose `data/` is a link (or whose config
+// carries `dataDir`) `legacy`; these cases still build that retired layout and
+// expect it to read `ok`. Slice T2 rebases them on `media/` and un-skips them.
+const T1_SKIP = "release 17 T2 rebases the storage watch on mediaDir";
+
+
// THE CONFIRMATION COUNT IS MODULE STATE (see storageWatch.ts rule 3), so each
// case starts from a clean one — otherwise the second test inherits the first
// test's suspicions and pauses on what should be its first pass. The health
@@ -149,7 +155,7 @@ async function twoPasses(h: H) {
return { first, second };
}
-test("a channel whose target is gone is auto-paused, once, in one write", async () => {
+test("a channel whose target is gone is auto-paused, once, in one write", { skip: T1_SKIP }, async () => {
await withTmp(async (h) => {
await seedRelocated(h, "gone-a", { targetExists: false });
await seedRelocated(h, "gone-b", { targetExists: false });
@@ -189,7 +195,7 @@ test("a channel whose target is gone is auto-paused, once, in one write", async
});
});
-test("the drive coming back restores the tier it overwrote", async () => {
+test("the drive coming back restores the tier it overwrote", { skip: T1_SKIP }, async () => {
await withTmp(async (h) => {
await seedRelocated(h, "away", { targetExists: false });
const settings = h.io.read();
@@ -295,7 +301,7 @@ test("a record on an in-place channel is restored", async () => {
// `write: false` IS IDLE BOOT. It observes and reports; the write is the work,
// and idle boot refuses work.
-test("write: false reports the transition and changes nothing", async () => {
+test("write: false reports the transition and changes nothing", { skip: T1_SKIP }, async () => {
await withTmp(async (h) => {
await seedRelocated(h, "gone", { targetExists: false });
// The first pass only suspects, whatever `write` says.
@@ -334,7 +340,7 @@ test("no locations and nothing auto-paused is a free pass", async () => {
// has spun down and needs a beat to answer is indistinguishable from "not
// mounted" — and pausing on it rewrites the corpus's priority document for a
// drive that is fine.
-test("a drive that blips for one pass is never paused", async () => {
+test("a drive that blips for one pass is never paused", { skip: T1_SKIP }, async () => {
await withTmp(async (h) => {
await seedRelocated(h, "blip", { targetExists: false });
const first = await runStorageWatchPass({
@@ -379,7 +385,7 @@ test("a drive that blips for one pass is never paused", async () => {
// RESTORE STAYS SINGLE-PASS, and the asymmetry is the point: being slow to
// pause costs a few refused units (the start-of-work guards catch those), while
// being slow to restore leaves a lane off after the operator fixed the cable.
-test("the restore needs only one good pass", async () => {
+test("the restore needs only one good pass", { skip: T1_SKIP }, async () => {
await withTmp(async (h) => {
await seedRelocated(h, "back", { targetExists: false });
await twoPasses(h);
@@ -409,7 +415,7 @@ function scripted(answers: LocationHealthState[]) {
return async () => answers[Math.min(i++, answers.length - 1)];
}
-test("one missed probe stalls the location; pages then answer 'stalled' without asking", async () => {
+test("one missed probe stalls the location; pages then answer 'stalled' without asking", { skip: T1_SKIP }, async () => {
await withTmp(async (h) => {
await seedRelocated(h, "slow", { targetExists: true });
const lines: string[] = [];
@@ -434,7 +440,7 @@ test("one missed probe stalls the location; pages then answer 'stalled' without
});
});
-test("the stall clears only after two clean probes in a row", async () => {
+test("the stall clears only after two clean probes in a row", { skip: T1_SKIP }, async () => {
await withTmp(async (h) => {
await seedRelocated(h, "slow", { targetExists: true });
const probe = scripted(["stalled", "ok", "stalled", "ok", "ok"]);
diff --git a/common/jobs/jobKinds.ts b/common/jobs/jobKinds.ts
@@ -37,12 +37,14 @@ export type JobKindMeta = {
// Fallback scheduler tier when a record is not explicitly background. Left
// undefined today so Phase 4 maps it to "foreground" (unchanged behavior).
defaultTier?: SchedulerTier;
- // Whether this kind reads or writes files under `channels/<slug>/data/`.
- // Declarative, and consumed by exactly one thing: runManagedFunction refuses
- // to enqueue a media kind for a channel whose media is not reachable (a
- // relocated channel whose drive is unmounted, or one mid-relocation) rather
- // than letting it read an empty dir as the truth. See
- // common/lib/channelMedia.ts.
+ // Whether this kind OPENS OR WRITES A BIG FILE — the audio, a persisted
+ // source container, the raw live-chat replay (lib/mediaTier.ts says which):
+ // the files that may live on another drive (release 17, the media tier).
+ // Declarative, and consumed by runManagedFunction, which refuses a media kind
+ // for a channel whose media is not reachable (a relocated channel whose drive
+ // is unmounted or stalled, one mid-move, a `legacy` one) — at enqueue and
+ // again when its queue starts it — rather than letting it read an empty dir
+ // as the truth or write into a tree being copied. See common/lib/channelMedia.ts.
//
// ABSENT MEANS FALSE, and that is deliberate rather than lazy. The guard is
// opt-in so a kind that is not listed keeps exactly its current behavior, and
@@ -51,13 +53,24 @@ export type JobKindMeta = {
// that genuinely never opens a video dir (store playlist, clear markers) must
// not be refused for a drive it does not read.
//
- // THE TEST IS "DOES IT OPEN data/", NOT "IS IT BOOKKEEPING". Two kinds that
- // read as bookkeeping declare it anyway, and both were misses:
- // `normalize-transcripts` walks every video dir and writes a sidecar into
- // each, and `sync` reads the data dir to decide what is already downloaded
- // before downloading into it. Against an unmounted drive the first reports a
- // clean run over zero videos and the second concludes nothing is downloaded.
+ // THE TEST IS "DOES IT OPEN OR WRITE THE BIG FILE", NOT "IS IT BOOKKEEPING"
+ // (the doctrine since release 17; before it, "does it open data/"). `sync`
+ // and the metadata scan's cousins that DOWNLOAD write media; a transcription
+ // reads it. A kind that reads only the text — a digest, normalize, the
+ // availability checks, the metadata scan, a clip-window fetch — declares
+ // `needsText` instead, and runs while the channel's media is moving, stalled
+ // or unmounted.
needsMedia?: boolean;
+ // Whether this kind walks or writes the TEXT under `channels/<slug>/data/`
+ // (transcripts, cues, metadata, sidecars, `clips/`) and nothing else. The
+ // text guard (`assertChannelTextReadable`) refuses it only where the text
+ // itself cannot be read: a `legacy` channel (the retired whole-directory
+ // layout, its text on the far drive), a `data/` that is not a directory, a
+ // tier migration in flight. Against such a channel the walk would find
+ // nothing and report a clean run over zero videos — or, for the
+ // availability checks and the metadata scan, read every downloaded video as
+ // missing and re-request the whole channel. Absent means false.
+ needsText?: boolean;
};
// One entry per kind known to the system. `label` is included only where the
@@ -101,7 +114,8 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
drainable: true,
replayable: false,
queueKeyStrategy: "parallel",
- needsMedia: true,
+ needsMedia: false,
+ needsText: true,
},
"auto-backfill": {
kind: "auto-backfill",
@@ -149,7 +163,8 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
drainable: false,
replayable: true,
queueKeyStrategy: "custom",
- needsMedia: true,
+ needsMedia: false,
+ needsText: true,
},
// AI digest sweep, local (ollama) lane — the one that carries the corpus. Both
// digest kinds are drainable (the batch honors the drain signal: it stops
@@ -161,7 +176,8 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
drainable: true,
replayable: true,
queueKeyStrategy: "custom",
- needsMedia: true,
+ needsMedia: false,
+ needsText: true,
},
// Same batch, metered lane. Off unless settings.digest.remoteEnabled is true,
// and it lands on its own queue key so it runs CONCURRENTLY with the local lane
@@ -172,7 +188,8 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
drainable: true,
replayable: true,
queueKeyStrategy: "custom",
- needsMedia: true,
+ needsMedia: false,
+ needsText: true,
},
// Copy a duplicate cluster's canonical digest onto its aligned mirrors. A fast
// file operation gated by the timestamp-alignment check, so it is not drainable
@@ -183,7 +200,8 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
drainable: false,
replayable: false,
queueKeyStrategy: "parallel",
- needsMedia: true,
+ needsMedia: false,
+ needsText: true,
},
// Write the compact transcript.cues.json sidecar next to every raw transcript
// that lacks a current one — corpus-wide from the Pool on /sites, or one
@@ -196,16 +214,19 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
// write, so "let the in-flight one finish" is already how it behaves. Not
// replayable either — that would need a JobSpec and a jobReplayRegistry
// handler, and the button is one click from the card that reports the count.
- // needsMedia: it walks `data/<id>/` for every video and writes a sidecar into
- // each. Against an unmounted drive it finds nothing, reports a clean
- // 0/0/0/0 run and moves on — a sweep that silently skips a channel.
+ // needsText (release 17; needsMedia before it): it walks `data/<id>/` for
+ // every video and writes a sidecar into each — text, on the corpus disk.
+ // Against an unreadable text tier (a `legacy` channel whose drive is
+ // unmounted) it finds nothing, reports a clean 0/0/0/0 run and moves on — a
+ // sweep that silently skips a channel. A stalled MEDIA drive does not stop it.
"normalize-transcripts": {
kind: "normalize-transcripts",
label: "Normalize transcripts",
drainable: false,
replayable: false,
queueKeyStrategy: "custom",
- needsMedia: true,
+ needsMedia: false,
+ needsText: true,
},
"redownload-incomplete-bucket": {
kind: "redownload-incomplete-bucket",
@@ -237,7 +258,8 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
drainable: true,
replayable: true,
queueKeyStrategy: "platform",
- needsMedia: true,
+ needsMedia: false,
+ needsText: true,
},
"import-one": {
kind: "import-one",
@@ -258,7 +280,8 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
drainable: false,
replayable: true,
queueKeyStrategy: "platform",
- needsMedia: true,
+ needsMedia: false,
+ needsText: true,
},
"redownload-archive": {
kind: "redownload-archive",
@@ -342,7 +365,8 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
drainable: false,
replayable: true,
queueKeyStrategy: "custom",
- needsMedia: true,
+ needsMedia: false,
+ needsText: true,
},
"persist-kept": {
kind: "persist-kept",
@@ -404,17 +428,19 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
// the cleanup lanes are about `audio.*`. This is the only thing that removes
// one.
//
- // `needsMedia: true`, and it is the sharpest case for the flag in the table:
- // it DELETES. Against an unmounted drive every `readdir` of `data/` throws
- // and the walk would report a clean eviction of zero bytes — the operator
- // would read "nothing to reclaim" about a platter full of windows.
+ // `needsText` (release 17): `clips/` is on the corpus disk and is never
+ // tiered, so the media drive does not concern it. The guard still matters:
+ // it DELETES, and against an unreadable text tier (a `legacy` channel whose
+ // drive is unmounted) every `readdir` of `data/` throws and the walk would
+ // report a clean eviction of zero bytes.
"evict-clips": {
kind: "evict-clips",
label: "Evict fetched windows",
drainable: false,
replayable: false,
queueKeyStrategy: "parallel",
- needsMedia: true,
+ needsMedia: false,
+ needsText: true,
},
// THE SAVED-VIDEO STORE, ONTO A LOCATION AND BACK. Same mechanism as the
// channel move (relocateDir.ts is literally the same code) over one directory
@@ -499,19 +525,20 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
// because it contends for the same thing a download does — the source's
// patience.
//
- // `needsMedia` IS TRUE, despite the scan never creating a video directory,
- // and the test in JobKindMeta is exactly why: does it open `data/`? It does —
- // its whole target set is "listed, minus what is already on disk". Against an
- // unmounted drive it would read every downloaded video as unfetched and
- // re-request the entire channel. It writes no media; that is a different
- // question from whether it READS the media dir.
+ // `needsText` (release 17; `needsMedia` before it): its whole target set is
+ // "listed, minus what is already on disk", and "on disk" is the video dirs in
+ // `data/` — text. Against an unreadable text tier (a `legacy` channel whose
+ // drive is unmounted) it would read every downloaded video as unfetched and
+ // re-request the entire channel. It writes no media, so a stalled media
+ // drive does not hold it.
"metadata-scan": {
kind: "metadata-scan",
label: "Metadata scan",
drainable: true,
replayable: true,
queueKeyStrategy: "platform",
- needsMedia: true,
+ needsMedia: false,
+ needsText: true,
},
// THE HUB'S AND THE HOMEPAGE'S BUILD AND DEPLOY (release 13 slice W1). They
// ran from /sites — the hub since release 7, the homepage since release 11 —
@@ -607,21 +634,24 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
drainable: false,
replayable: false,
queueKeyStrategy: "platform",
- needsMedia: true,
+ needsMedia: false,
+ needsText: true,
},
"quick-availability-check": {
kind: "quick-availability-check",
drainable: false,
replayable: false,
queueKeyStrategy: "platform",
- needsMedia: true,
+ needsMedia: false,
+ needsText: true,
},
"check-maybe-missing": {
kind: "check-maybe-missing",
drainable: false,
replayable: false,
queueKeyStrategy: "platform",
- needsMedia: true,
+ needsMedia: false,
+ needsText: true,
},
// Replayable kinds that never had a JOB_KIND_LABELS entry: label omitted so
// jobKindLabel() keeps falling back to the raw kind (unchanged behavior).
@@ -667,9 +697,15 @@ export function isDrainableKind(kind: string): boolean {
return JOB_KINDS[kind]?.drainable ?? false;
}
-// Whether a kind's work reaches `channels/<slug>/data/`. Absent = false: the
-// media guard is opt-in, so an unlisted (or unknown) kind behaves exactly as it
-// did before the guard existed.
+// Whether a kind's work opens or writes a big file. Absent = false: the media
+// guard is opt-in, so an unlisted (or unknown) kind behaves exactly as it did
+// before the guard existed.
export function kindNeedsMedia(kind: string): boolean {
return JOB_KINDS[kind]?.needsMedia ?? false;
}
+
+// Whether a kind reads the text tier only (and so asks the text guard instead
+// of the media one). Absent = false.
+export function kindNeedsText(kind: string): boolean {
+ return JOB_KINDS[kind]?.needsText ?? false;
+}
diff --git a/common/jobs/streamCommand.ts b/common/jobs/streamCommand.ts
@@ -20,9 +20,10 @@ import {
import { writeJobMeta } from "./jobMeta";
import { maybePruneJobLogs } from "./listJobs";
import type { JobSpec } from "./jobSpec";
-import { kindNeedsMedia } from "./jobKinds";
+import { kindNeedsMedia, kindNeedsText } from "./jobKinds";
import {
assertChannelMediaReachable,
+ assertChannelTextReadable,
ChannelMediaUnreachableError,
} from "../lib/channelMedia";
import { mediaHoldText } from "../lib/channelMediaHold";
@@ -324,10 +325,25 @@ export async function runManagedCommand(
//
// A MOVE IS A HOLD, and the refusal says so in the hold's words ("held: its
// media is moving …") rather than calling the media unreachable.
+//
+// A TEXT KIND ASKS THE TEXT GUARD (release 17): a kind that declares
+// `needsText` reads only `data/`'s text, which stays on the corpus disk, so it
+// runs while the channel's media is moving, stalled or unmounted, and is
+// refused only where the text itself cannot be read (a `legacy` channel, a
+// `data/` that is not a directory, a tier migration in flight).
async function refuseForUnreachableMedia(
opts: CommonOpts,
): Promise<string | null> {
- if (!opts.channelSlug || !kindNeedsMedia(opts.kind)) return null;
+ if (!opts.channelSlug) return null;
+ if (!kindNeedsMedia(opts.kind)) {
+ if (!kindNeedsText(opts.kind)) return null;
+ try {
+ await assertChannelTextReadable(opts.paths, opts.channelSlug);
+ return null;
+ } catch (err) {
+ return (err as Error).message;
+ }
+ }
try {
await assertChannelMediaReachable(opts.paths, opts.channelSlug);
return null;
diff --git a/common/lib/channelConfig.ts b/common/lib/channelConfig.ts
@@ -126,6 +126,7 @@ export type ChannelConfig = {
extractionMode?: ExtractionMode;
savedVideosDir?: string;
dataDir?: string;
+ mediaDir?: string;
ytdlpExtraArgs?: string[];
subLangs?: string;
lastSyncedAt?: string;
@@ -167,7 +168,9 @@ export const CHANNEL_CONFIG_FIELD_DOCS: FieldDocs<ChannelConfig> = {
savedVideosDir:
"Per-channel override for the saved-video store root: this channel's persisted source videos live under `<savedVideosDir>/<slug>/<videoId>/`. Trimmed; blank = the global store.",
dataDir:
- "Where this channel's media ACTUALLY lives when relocated to another drive: the absolute path `channels/<slug>/data` is a symlink to. Absent = in place. Written ONLY by the relocate / re-point jobs on success — a record of what is on disk, never free text, because a value that disagrees with the link is an \"inconsistent\" channel every guard refuses.",
+ "RETIRED (release 17). The whole-directory layout's record: the absolute path `channels/<slug>/data` was a symlink to. Still parsed for one release so a write never erases it: a channel that carries it — or whose `data/` is a link — is `legacy`, and every media job, lane and build holds it until `archilyzer storage migrate-tier <slug>` moves its text back and its media into `mediaDir`. Never written by anything but that migration, which removes it.",
+ mediaDir:
+ "Where this channel's big files live when relocated: `channels/<slug>/media` is a symlink to it, `<root>/<slug>/media`. Absent = in place. Written only by relocate / re-point / the tier migration — a record of what is on disk, never free text, because a value that disagrees with the link is an \"inconsistent\" channel every media guard refuses. The text (`data/`) never moves.",
ytdlpExtraArgs: "Extra yt-dlp arguments, appended verbatim. Must be an array of strings or it is dropped.",
subLangs: "yt-dlp `--sub-langs` value for caption downloads.",
lastSyncedAt:
@@ -368,6 +371,7 @@ export const CHANNEL_CONFIG_COERCIONS: {
extractionMode: (v) => (v === "ytdlp" || v === "app" ? v : undefined),
savedVideosDir: trimmedNonBlank,
dataDir: trimmedNonBlank,
+ mediaDir: trimmedNonBlank,
ytdlpExtraArgs: (v) =>
Array.isArray(v) && v.every((x) => typeof x === "string")
? (v as string[])
diff --git a/common/lib/channelConfigSchema.test.ts b/common/lib/channelConfigSchema.test.ts
@@ -39,7 +39,7 @@ test("one key list: docs = coercions = schema shape, sync-state keys inside it",
assert.deepEqual(Object.keys(CHANNEL_CONFIG_COERCIONS), [...CHANNEL_CONFIG_KEYS]);
assert.deepEqual(Object.keys(channelConfigObjectSchema.shape), [...CHANNEL_CONFIG_KEYS]);
assert.deepEqual(Object.keys(CHANNEL_CONFIG_FIELD_DOCS), [...CHANNEL_CONFIG_KEYS]);
- assert.equal(CHANNEL_CONFIG_KEYS.length, 29);
+ assert.equal(CHANNEL_CONFIG_KEYS.length, 30);
assert.equal(sameKeys, true);
assert.equal(fits, true);
for (const k of CHANNEL_SYNC_STATE_KEYS) assert.ok(CHANNEL_CONFIG_KEYS.includes(k), k);
@@ -108,12 +108,14 @@ test("trims and normalises", () => {
socialHandle: " @me ",
savedVideosDir: " /x ",
dataDir: " /d ",
+ mediaDir: " /m/chan/media ",
cookiesFromBrowser: " firefox ",
downloadFilter: { include: " a ", exclude: "", includeLivestreams: true, rejectedLivestreams: "skip" },
})!;
assert.equal(cfg.socialHandle, "me");
assert.equal(cfg.savedVideosDir, "/x");
assert.equal(cfg.dataDir, "/d");
+ assert.equal(cfg.mediaDir, "/m/chan/media");
assert.equal(cfg.cookiesFromBrowser, "firefox");
assert.deepEqual(cfg.downloadFilter, { include: "a", includeLivestreams: true });
});
diff --git a/common/lib/channelConfigSchema.ts b/common/lib/channelConfigSchema.ts
@@ -57,6 +57,7 @@ export const channelConfigObjectSchema = z.object({
extractionMode: field("extractionMode"),
savedVideosDir: field("savedVideosDir"),
dataDir: field("dataDir"),
+ mediaDir: field("mediaDir"),
ytdlpExtraArgs: field("ytdlpExtraArgs"),
subLangs: field("subLangs"),
lastSyncedAt: field("lastSyncedAt"),
diff --git a/common/lib/channelMedia.test.ts b/common/lib/channelMedia.test.ts
@@ -5,7 +5,11 @@ import { tmpdir } from "node:os";
import path from "node:path";
import {
ChannelMediaUnreachableError,
+ ChannelTextUnreadableError,
assertChannelMediaReachable,
+ assertChannelTextReadable,
+ channelMediaStall,
+ channelTextStall,
clearRelocationMarker,
inspectChannelMedia,
readRelocationMarker,
@@ -13,6 +17,9 @@ import {
RELOCATION_MARKER_FILENAME,
type ChannelMediaPaths,
} from "./channelMedia";
+import { relocatedMediaDir } from "./mediaTier-server";
+import { recordLocationHealth, resetStorageHealth } from "./storageHealth";
+import { isMediaHeld, isTextHeld, HELD_REASON } from "./channelMediaHold";
// Run with:
// pnpm --filter yt-dlp-transcript-common exec tsx --test lib/channelMedia.test.ts
@@ -47,12 +54,12 @@ async function seedChannel(
return channelDir;
}
-test("relocatedDataDir fixes the <root>/<slug>/data suffix", () => {
+test("relocatedDataDir (retired) and relocatedMediaDir fix their suffixes", () => {
assert.equal(relocatedDataDir("/mnt/p", "alpha"), "/mnt/p/alpha/data");
- assert.equal(relocatedDataDir(" /mnt/p ", "alpha"), "/mnt/p/alpha/data");
+ assert.equal(relocatedMediaDir(" /mnt/p ", "alpha"), "/mnt/p/alpha/media");
});
-test("a real data dir with no config.dataDir is in-place", async () => {
+test("a real data dir and no media link, no mediaDir: in-place (classic)", async () => {
await withTmp(async (paths) => {
const channelDir = await seedChannel(paths, "alpha");
await mkdir(path.join(channelDir, "data", "v1"), { recursive: true });
@@ -61,7 +68,21 @@ test("a real data dir with no config.dataDir is in-place", async () => {
assert.equal(loc.relocated, false);
assert.equal(loc.target, undefined);
assert.equal(loc.dataDir, path.join(channelDir, "data"));
+ assert.equal(loc.mediaLink, path.join(channelDir, "media"));
+ assert.deepEqual(loc.text, { dir: path.join(channelDir, "data"), readable: true });
await assertChannelMediaReachable(paths, "alpha");
+ await assertChannelTextReadable(paths, "alpha");
+ });
+});
+
+test("a real media/ directory on the corpus disk is in-place (tiered in place)", async () => {
+ await withTmp(async (paths) => {
+ const channelDir = await seedChannel(paths, "alpha");
+ await mkdir(path.join(channelDir, "data", "v1"), { recursive: true });
+ await mkdir(path.join(channelDir, "media", "v1"), { recursive: true });
+ const loc = await inspectChannelMedia(paths, "alpha");
+ assert.equal(loc.status, "in-place");
+ assert.equal(loc.relocated, false);
});
});
@@ -70,33 +91,42 @@ test("a channel that has downloaded nothing is in-place, not an error", async ()
await seedChannel(paths, "alpha");
const loc = await inspectChannelMedia(paths, "alpha");
assert.equal(loc.status, "in-place");
+ assert.equal(loc.text.readable, true);
await assertChannelMediaReachable(paths, "alpha");
+ await assertChannelTextReadable(paths, "alpha");
});
});
-test("a link agreeing with config and pointing at a live dir is ok", async () => {
+async function seedRelocated(paths: ChannelMediaPaths, root: string, opts: { mount?: boolean } = {}) {
+ const target = relocatedMediaDir(root, "alpha");
+ if (opts.mount !== false) await mkdir(path.join(target, "v1"), { recursive: true });
+ const channelDir = await seedChannel(paths, "alpha", { mediaDir: target });
+ await mkdir(path.join(channelDir, "data", "v1"), { recursive: true });
+ await symlink(target, path.join(channelDir, "media"));
+ return { target, channelDir };
+}
+
+test("a media link agreeing with mediaDir and pointing at a live dir is ok", async () => {
await withTmp(async (paths, root) => {
- const target = relocatedDataDir(root, "alpha");
- await mkdir(path.join(target, "v1"), { recursive: true });
- const channelDir = await seedChannel(paths, "alpha", { dataDir: target });
- await symlink(target, path.join(channelDir, "data"));
+ const { target } = await seedRelocated(paths, root);
const loc = await inspectChannelMedia(paths, "alpha");
assert.equal(loc.status, "ok");
assert.equal(loc.relocated, true);
assert.equal(loc.target, target);
+ assert.equal(loc.text.readable, true);
await assertChannelMediaReachable(paths, "alpha");
+ await assertChannelTextReadable(paths, "alpha");
});
});
-test("a dangling link (drive not mounted) is unreachable and throws", async () => {
+test("a dangling media link (drive not mounted): media unreachable, text readable", async () => {
await withTmp(async (paths, root) => {
- const target = relocatedDataDir(root, "alpha");
- const channelDir = await seedChannel(paths, "alpha", { dataDir: target });
// The link is made WITHOUT creating the target: exactly an unmounted drive.
- await symlink(target, path.join(channelDir, "data"));
+ await seedRelocated(paths, root, { mount: false });
const loc = await inspectChannelMedia(paths, "alpha");
assert.equal(loc.status, "unreachable");
assert.equal(loc.relocated, true);
+ assert.equal(loc.text.readable, true);
assert.match(loc.detail ?? "", /does not exist/);
await assert.rejects(
() => assertChannelMediaReachable(paths, "alpha"),
@@ -108,24 +138,44 @@ test("a dangling link (drive not mounted) is unreachable and throws", async () =
return true;
},
);
+ // THE NEW RULE: the text is on the corpus disk, so it is not held.
+ await assertChannelTextReadable(paths, "alpha");
+ assert.equal(isMediaHeld(loc.status), true);
+ assert.equal(isTextHeld(loc.status), false);
});
});
test("an EMPTY mountpoint is still unreachable — the link points deep", async () => {
await withTmp(async (paths, root) => {
- const target = relocatedDataDir(root, "alpha");
- // The root exists (mountpoint present) but holds nothing.
await mkdir(root, { recursive: true });
- const channelDir = await seedChannel(paths, "alpha", { dataDir: target });
- await symlink(target, path.join(channelDir, "data"));
+ await seedRelocated(paths, root, { mount: false });
const loc = await inspectChannelMedia(paths, "alpha");
assert.equal(loc.status, "unreachable");
});
});
-test("a marker makes the channel in-transition whatever the disk says", async () => {
+test("a stalled media location: answered from memory, text still readable", async () => {
await withTmp(async (paths, root) => {
- const target = relocatedDataDir(root, "alpha");
+ resetStorageHealth();
+ try {
+ const { target } = await seedRelocated(paths, root);
+ recordLocationHealth({ id: "platter", label: "Platter", root }, "stalled");
+ const loc = await inspectChannelMedia(paths, "alpha", undefined, { fresh: true });
+ assert.equal(loc.status, "stalled");
+ assert.equal(loc.text.readable, true);
+ assert.ok(channelMediaStall({ mediaDir: target }));
+ assert.equal(channelTextStall({ mediaDir: target }), null);
+ await assert.rejects(() => assertChannelMediaReachable(paths, "alpha"));
+ await assertChannelTextReadable(paths, "alpha");
+ } finally {
+ resetStorageHealth();
+ }
+ });
+});
+
+test("a marker makes the channel in-transition; a media move leaves the text readable", async () => {
+ await withTmp(async (paths, root) => {
+ const target = relocatedMediaDir(root, "alpha");
const channelDir = await seedChannel(paths, "alpha");
await mkdir(path.join(channelDir, "data"), { recursive: true });
await writeFile(
@@ -135,81 +185,186 @@ test("a marker makes the channel in-transition whatever the disk says", async ()
direction: "out",
startedAt: new Date().toISOString(),
phase: "copy",
+ scope: "media",
}),
);
const loc = await inspectChannelMedia(paths, "alpha");
assert.equal(loc.status, "in-transition");
assert.equal(loc.marker?.phase, "copy");
assert.equal(loc.marker?.direction, "out");
+ assert.equal(loc.marker?.scope, "media");
assert.equal(loc.target, target);
+ assert.equal(loc.text.readable, true);
await assert.rejects(
() => assertChannelMediaReachable(paths, "alpha"),
ChannelMediaUnreachableError,
);
+ await assertChannelTextReadable(paths, "alpha");
const marker = await readRelocationMarker(paths, "alpha");
assert.equal(marker?.target, target);
});
});
-test("a link that disagrees with config is inconsistent, never guessed past", async () => {
+test("a tier-migration marker holds the text too", async () => {
await withTmp(async (paths, root) => {
- const real = relocatedDataDir(root, "alpha");
- const recorded = relocatedDataDir(path.join(root, "other"), "alpha");
+ const channelDir = await seedChannel(paths, "alpha");
+ await mkdir(path.join(channelDir, "data"), { recursive: true });
+ await writeFile(
+ path.join(channelDir, RELOCATION_MARKER_FILENAME),
+ JSON.stringify({
+ target: relocatedMediaDir(root, "alpha"),
+ direction: "out",
+ phase: "copy",
+ scope: "tier-migration",
+ }),
+ );
+ const loc = await inspectChannelMedia(paths, "alpha");
+ assert.equal(loc.status, "in-transition");
+ assert.equal(loc.text.readable, false);
+ await assert.rejects(
+ () => assertChannelTextReadable(paths, "alpha"),
+ (err: unknown) => {
+ assert.ok(err instanceof ChannelTextUnreadableError);
+ assert.match(err.message, /being migrated/);
+ return true;
+ },
+ );
+ });
+});
+
+test("LEGACY: a data link with dataDir — held by both guards, no call to the drive", async () => {
+ await withTmp(async (paths, root) => {
+ const target = relocatedDataDir(root, "alpha");
+ await mkdir(path.join(target, "v1"), { recursive: true });
+ const channelDir = await seedChannel(paths, "alpha", { dataDir: target });
+ await symlink(target, path.join(channelDir, "data"));
+ const loc = await inspectChannelMedia(paths, "alpha");
+ assert.equal(loc.status, "legacy");
+ assert.equal(loc.relocated, true);
+ assert.equal(loc.target, target);
+ assert.equal(loc.text.readable, false);
+ assert.match(loc.detail ?? "", /archilyzer storage migrate-tier alpha/);
+ assert.equal(isMediaHeld(loc.status), true);
+ assert.equal(isTextHeld(loc.status), true);
+ assert.match(HELD_REASON.legacy, /migrate-tier/);
+ await assert.rejects(
+ () => assertChannelMediaReachable(paths, "alpha"),
+ (err: unknown) => {
+ assert.ok(err instanceof ChannelMediaUnreachableError);
+ assert.match(err.message, /migrate-tier/);
+ return true;
+ },
+ );
+ await assert.rejects(
+ () => assertChannelTextReadable(paths, "alpha"),
+ (err: unknown) => {
+ assert.ok(err instanceof ChannelTextUnreadableError);
+ assert.equal(err.status, "legacy");
+ assert.match(err.message, /migrate-tier alpha/);
+ return true;
+ },
+ );
+ });
+});
+
+test("LEGACY: a dangling data link (unmounted) is legacy too, never 'no videos'", async () => {
+ await withTmp(async (paths, root) => {
+ const target = relocatedDataDir(root, "alpha");
+ const channelDir = await seedChannel(paths, "alpha", { dataDir: target });
+ await symlink(target, path.join(channelDir, "data"));
+ assert.equal((await inspectChannelMedia(paths, "alpha")).status, "legacy");
+ });
+});
+
+test("LEGACY: a data link with no dataDir, and dataDir with a real data dir", async () => {
+ await withTmp(async (paths, root) => {
+ const target = relocatedDataDir(root, "alpha");
+ await mkdir(target, { recursive: true });
+ const a = await seedChannel(paths, "alpha");
+ await symlink(target, path.join(a, "data"));
+ assert.equal((await inspectChannelMedia(paths, "alpha")).status, "legacy");
+
+ const b = await seedChannel(paths, "beta", { dataDir: relocatedDataDir(root, "beta") });
+ await mkdir(path.join(b, "data"), { recursive: true });
+ const loc = await inspectChannelMedia(paths, "beta");
+ assert.equal(loc.status, "legacy");
+ assert.equal(loc.text.readable, false);
+ assert.ok(channelTextStall({ dataDir: relocatedDataDir(root, "beta") }) === null);
+ });
+});
+
+test("a media link that disagrees with mediaDir is inconsistent, never guessed past", async () => {
+ await withTmp(async (paths, root) => {
+ const real = relocatedMediaDir(root, "alpha");
+ const recorded = relocatedMediaDir(path.join(root, "other"), "alpha");
await mkdir(real, { recursive: true });
- const channelDir = await seedChannel(paths, "alpha", { dataDir: recorded });
- await symlink(real, path.join(channelDir, "data"));
+ const channelDir = await seedChannel(paths, "alpha", { mediaDir: recorded });
+ await symlink(real, path.join(channelDir, "media"));
const loc = await inspectChannelMedia(paths, "alpha");
assert.equal(loc.status, "inconsistent");
assert.match(loc.detail ?? "", /points at/);
+ assert.equal(loc.text.readable, true);
await assert.rejects(
() => assertChannelMediaReachable(paths, "alpha"),
ChannelMediaUnreachableError,
);
+ await assertChannelTextReadable(paths, "alpha");
});
});
-test("a link with no config.dataDir is inconsistent", async () => {
+test("a media link with no mediaDir is inconsistent", async () => {
await withTmp(async (paths, root) => {
- const target = relocatedDataDir(root, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
await mkdir(target, { recursive: true });
const channelDir = await seedChannel(paths, "alpha");
- await symlink(target, path.join(channelDir, "data"));
+ await symlink(target, path.join(channelDir, "media"));
const loc = await inspectChannelMedia(paths, "alpha");
assert.equal(loc.status, "inconsistent");
assert.equal(loc.relocated, false);
- assert.match(loc.detail ?? "", /records no dataDir/);
+ assert.match(loc.detail ?? "", /records no mediaDir/);
});
});
-test("config.dataDir with a real directory on disk is inconsistent", async () => {
+test("mediaDir with a real media/ directory on disk is inconsistent", async () => {
await withTmp(async (paths, root) => {
- const target = relocatedDataDir(root, "alpha");
- const channelDir = await seedChannel(paths, "alpha", { dataDir: target });
- await mkdir(path.join(channelDir, "data"), { recursive: true });
+ const target = relocatedMediaDir(root, "alpha");
+ const channelDir = await seedChannel(paths, "alpha", { mediaDir: target });
+ await mkdir(path.join(channelDir, "media"), { recursive: true });
const loc = await inspectChannelMedia(paths, "alpha");
assert.equal(loc.status, "inconsistent");
assert.match(loc.detail ?? "", /never moved/);
});
});
-test("config.dataDir with no data/ at all is inconsistent (link gone)", async () => {
+test("mediaDir with no media link at all is inconsistent (link gone)", async () => {
await withTmp(async (paths, root) => {
- const target = relocatedDataDir(root, "alpha");
- await seedChannel(paths, "alpha", { dataDir: target });
+ const target = relocatedMediaDir(root, "alpha");
+ await seedChannel(paths, "alpha", { mediaDir: target });
const loc = await inspectChannelMedia(paths, "alpha");
assert.equal(loc.status, "inconsistent");
assert.match(loc.detail ?? "", /symlink is missing/);
});
});
+test("a data/ that is a file is inconsistent and its text unreadable", async () => {
+ await withTmp(async (paths) => {
+ const channelDir = await seedChannel(paths, "alpha");
+ await writeFile(path.join(channelDir, "data"), "not a dir");
+ const loc = await inspectChannelMedia(paths, "alpha");
+ assert.equal(loc.status, "inconsistent");
+ assert.equal(loc.text.readable, false);
+ await assert.rejects(() => assertChannelTextReadable(paths, "alpha"), ChannelTextUnreadableError);
+ });
+});
+
test("a passed config is used verbatim; config.json is only read when it is absent", async () => {
await withTmp(async (paths, root) => {
- const target = relocatedDataDir(root, "alpha");
+ const target = relocatedMediaDir(root, "alpha");
await mkdir(target, { recursive: true });
// config.json on disk says NOTHING about a relocation...
const channelDir = await seedChannel(paths, "alpha");
- await symlink(target, path.join(channelDir, "data"));
+ await symlink(target, path.join(channelDir, "media"));
// ...so reading it itself gives "inconsistent"...
assert.equal(
@@ -219,7 +374,7 @@ test("a passed config is used verbatim; config.json is only read when it is abse
// ...while a caller that hands over the config it already holds gets the
// answer for THAT config, with no second read.
const passed = await inspectChannelMedia(paths, "alpha", {
- dataDir: target,
+ mediaDir: target,
});
assert.equal(passed.status, "ok");
assert.equal(passed.target, target);
@@ -231,13 +386,13 @@ test("a passed config is used verbatim; config.json is only read when it is abse
});
});
-test("a blank config.dataDir means in place", async () => {
+test("a blank mediaDir or dataDir means in place", async () => {
await withTmp(async (paths) => {
- const channelDir = await seedChannel(paths, "alpha", { dataDir: " " });
+ const channelDir = await seedChannel(paths, "alpha", { dataDir: " ", mediaDir: " " });
await mkdir(path.join(channelDir, "data"), { recursive: true });
assert.equal((await inspectChannelMedia(paths, "alpha")).status, "in-place");
assert.equal(
- (await inspectChannelMedia(paths, "alpha", { dataDir: " " })).status,
+ (await inspectChannelMedia(paths, "alpha", { dataDir: " ", mediaDir: "" })).status,
"in-place",
);
});
@@ -253,11 +408,8 @@ test("an unreadable or missing config.json is not a relocation", async () => {
});
test("clearRelocationMarker removes the marker and touches nothing else", async () => {
- await withTmp(async (paths) => {
- const target = path.join(paths.channelsDir, "..", "platter", "alpha", "data");
- await mkdir(target, { recursive: true });
- const channelDir = await seedChannel(paths, "alpha", { dataDir: target });
- await symlink(target, path.join(channelDir, "data"));
+ await withTmp(async (paths, root) => {
+ const { target, channelDir } = await seedRelocated(paths, root);
await writeFile(
path.join(channelDir, RELOCATION_MARKER_FILENAME),
JSON.stringify({ target, direction: "out", phase: "swap", startedAt: "" }),
diff --git a/common/lib/channelMedia.ts b/common/lib/channelMedia.ts
@@ -12,25 +12,33 @@ import {
type LocationHealth,
} from "./storageHealth";
import { secondsText } from "./storageHealthTimings";
+import { MEDIA_LINK_NAME } from "./mediaTier-server";
// WHERE A CHANNEL'S MEDIA ACTUALLY IS, and whether it can be reached.
//
-// A channel's downloaded media lives at `channels/<slug>/data/`. That path is
-// joined inline at ~74 call sites and is the on-disk contract every reader,
-// yt-dlp's cwd-relative output template and the LMDB index depend on, so
-// relocating a channel to another drive does NOT change it: `data/` becomes an
-// absolute SYMLINK to `<root>/<slug>/data` and `config.json` records the target
-// in `dataDir`. Every existing reader follows the link transparently — there is
-// no symlink-aware code anywhere in common/, editor/ or export/, and there does
-// not need to be.
+// A channel's files live at `channels/<slug>/data/<id>/`. That path is joined
+// inline at ~74 call sites and is the on-disk contract every reader, yt-dlp's
+// cwd-relative output template and the LMDB index depend on, so it never moves.
+// Since release 17 only the BIG files leave it (lib/mediaTier.ts says which):
+// each becomes a RELATIVE link `data/<id>/<name> -> ../../media/<id>/<name>`,
+// and `channels/<slug>/media` is either a real directory on the corpus disk
+// (tiered in place) or ONE absolute symlink to `<root>/<slug>/media` on another
+// drive, recorded in `config.json` as `mediaDir` (relocated). The text —
+// transcripts, cues, metadata, every sidecar — stays on the corpus disk.
//
-// What that buys in call-site churn it owes in one new failure mode: an
-// unmounted drive. A dangling link reads as ENOENT, and the three places that
-// enumerate `data/` swallow ENOENT as "this channel has no videos" — which to an
-// unattended runner means *everything is undownloaded* and is an instruction to
-// re-download hundreds of gigabytes onto the volume that was too full to hold
-// them. This module is the one place that can tell those two apart, and the
-// guards that call it are what make the symlink safe.
+// What the link buys in call-site churn it owes in one failure mode: an
+// unmounted drive. A dangling link reads as ENOENT. For the media alone that is
+// survivable — a reader of the text never touches it — and this module is the
+// one place that can tell the media's states apart, so the guards that call it
+// are what make the link safe: `assertChannelMediaReachable` for a job that
+// opens a big file, `assertChannelTextReadable` for one that reads only text.
+//
+// THE RETIRED LAYOUT. Before release 17 a relocation moved the whole `data/`
+// (an absolute symlink `data -> <root>/<slug>/data`, `config.dataDir`). Such a
+// channel is `legacy`: its text is on the far drive too, so it is held by BOTH
+// guards — every lane, every media job, the index and stats builds — until
+// `archilyzer storage migrate-tier <slug>` brings its text home. `dataDir` is
+// still parsed one release for exactly that (channelConfig.ts).
//
// IT LIVES IN lib/ AND MAY NOT IMPORT controller/ (architecture.test.ts), which
// is where readChannelConfig is. Hence the optional `config` argument: a caller
@@ -52,18 +60,28 @@ export const RELOCATION_MARKER_FILENAME = ".relocating.json";
export type RelocationDirection = "out" | "back";
export type RelocationPhase = "copy" | "swap" | "reclaim";
+// What a marker is moving. `media` (or absent): the channel's media tier — its
+// text stays readable, so only its media writers are held. `tier-migration`:
+// the one-off migration off the retired layout (common/bin/migrate-media-tier.ts),
+// which rebuilds `data/` itself, so the text is held too.
+export type RelocationScope = "media" | "tier-migration";
+
export type RelocationMarker = {
- // Absolute path of the relocated data dir: <root>/<slug>/data.
+ // Absolute path of the relocated media dir: <root>/<slug>/media (the retired
+ // mover wrote <root>/<slug>/data).
target: string;
direction: RelocationDirection;
startedAt: string;
phase: RelocationPhase;
+ scope?: RelocationScope;
};
export type ChannelMediaStatus =
- // No relocation: `data/` is a real directory (or does not exist yet).
+ // No relocation: no `media` link (a classic channel, or one tiered into a
+ // real `media/` directory on the corpus disk) and no `mediaDir`.
| "in-place"
- // Relocated, link and config agree, and the target is a reachable directory.
+ // Relocated: the `media` link and `mediaDir` agree, and the target is a
+ // reachable directory.
| "ok"
// Relocated, but the target is not there — almost always an unmounted drive.
| "unreachable"
@@ -75,15 +93,30 @@ export type ChannelMediaStatus =
// (`lib/storageHealth.ts`). Answered from memory, WITHOUT a filesystem call:
// a call there would block one of the process's few I/O threads for as long
// as the drive takes to come back. Held and refused like `unreachable`.
- | "stalled";
+ | "stalled"
+ // The RETIRED whole-directory layout: `data/` is a link, or config.json still
+ // records `dataDir`. Its text is not on the corpus disk, so it is held by the
+ // text guard as well as the media one until `archilyzer storage migrate-tier`
+ // runs. Answered without a call to the far drive.
+ | "legacy";
export type ChannelMediaLocation = {
- // Always channelDir/data — the path every reader uses, relocated or not.
+ // Always channelDir/data — the real text dir every reader uses (a link only
+ // on a `legacy` channel).
dataDir: string;
- // Whether config.json records a relocation target.
+ // Always channelDir/media — the media tier's one name.
+ mediaLink: string;
+ // Whether config.json records a relocation target (`mediaDir`, or the
+ // retired `dataDir` on a legacy channel).
relocated: boolean;
- // config.dataDir (or, mid-transition with no config yet, the marker's target).
+ // config.mediaDir (or, mid-transition with no config yet, the marker's
+ // target; on a legacy channel the retired `dataDir`).
target?: string;
+ // The text tier: `data/`, and whether a reader may walk it. Not readable on a
+ // `legacy` channel (its text is on the far drive), when `data/` is something
+ // other than a directory, or mid tier-migration. A channel with no `data/`
+ // yet is readable (it has downloaded nothing).
+ text: { dir: string; readable: boolean };
status: ChannelMediaStatus;
// Operator-readable reason, set for every status except "in-place" and "ok".
detail?: string;
@@ -111,13 +144,37 @@ export class ChannelMediaUnreachableError extends Error {
}
}
-// The relocated layout, fixed so one root can hold many channels and the shape
-// mirrors the saved-video store (<root>/<slug>/<...>). The `<slug>/data` suffix
-// is not configurable: deleteChannel and renameChannel recognise a target by it.
+// Thrown by assertChannelTextReadable: the channel's TEXT cannot be walked — a
+// `legacy` channel, a `data/` that is not a directory, a tier migration in
+// flight. Same shape as the media error so a lane can skip on either.
+export class ChannelTextUnreadableError extends Error {
+ readonly slug: string;
+ readonly status: ChannelMediaStatus;
+ readonly location: ChannelMediaLocation;
+ constructor(slug: string, location: ChannelMediaLocation, detail: string) {
+ super(`Channel "${slug}": its text is not readable — ${detail}`);
+ this.name = "ChannelTextUnreadableError";
+ this.slug = slug;
+ this.status = location.status;
+ this.location = location;
+ }
+}
+
+// THE RETIRED layout's target shape, `<root>/<slug>/data`. Still exported for
+// the mover, the re-point and the rename until release 17 slice T2 rebases them
+// on `relocatedMediaDir` (lib/mediaTier-server.ts).
export function relocatedDataDir(root: string, slug: string): string {
return path.join(root.trim(), slug, "data");
}
+// The sentence a legacy channel is refused with, naming the way out.
+export function legacyDetail(slug: string): string {
+ return (
+ `its media layout is the retired whole-directory one — run ` +
+ `archilyzer storage migrate-tier ${slug}`
+ );
+}
+
export function channelMediaDir(
paths: ChannelMediaPaths,
slug: string,
@@ -139,12 +196,14 @@ function parseMarker(raw: unknown): RelocationMarker | null {
const direction = r.direction === "back" ? "back" : "out";
const phase =
r.phase === "swap" || r.phase === "reclaim" ? r.phase : "copy";
- return {
+ const marker: RelocationMarker = {
target: r.target,
direction,
startedAt: typeof r.startedAt === "string" ? r.startedAt : "",
phase,
};
+ if (r.scope === "media" || r.scope === "tier-migration") marker.scope = r.scope;
+ return marker;
}
export async function readRelocationMarker(
@@ -180,24 +239,37 @@ export async function clearRelocationMarker(
forgetChannelMedia(slug);
}
-// The `dataDir` field alone, read straight off config.json. Deliberately NOT
-// parseChannelConfig: this runs in guards on hot paths and must not depend on
-// the controller that owns the rest of the schema.
-async function readConfiguredDataDir(
+// The two fields this module needs, the way a ChannelConfig carries them.
+export type ChannelMediaConfig = Pick<ChannelConfig, "mediaDir" | "dataDir">;
+
+type Configured = { mediaDir?: string; dataDir?: string };
+
+function trimmed(v: unknown): string | undefined {
+ if (typeof v !== "string") return undefined;
+ const t = v.trim();
+ return t === "" ? undefined : t;
+}
+
+function configuredOf(config: ChannelMediaConfig | null | undefined): Configured {
+ return { mediaDir: trimmed(config?.mediaDir), dataDir: trimmed(config?.dataDir) };
+}
+
+// `mediaDir` and the retired `dataDir`, read straight off config.json.
+// Deliberately NOT parseChannelConfig: this runs in guards on hot paths and
+// must not depend on the controller that owns the rest of the schema.
+async function readConfigured(
paths: ChannelMediaPaths,
slug: string,
-): Promise<string | undefined> {
+): Promise<Configured> {
try {
const raw = await readFile(
path.join(paths.channelsDir, slug, "config.json"),
"utf8",
);
- const parsed = JSON.parse(raw) as { dataDir?: unknown };
- if (typeof parsed.dataDir !== "string") return undefined;
- const trimmed = parsed.dataDir.trim();
- return trimmed === "" ? undefined : trimmed;
+ const parsed = JSON.parse(raw) as { mediaDir?: unknown; dataDir?: unknown };
+ return { mediaDir: trimmed(parsed.mediaDir), dataDir: trimmed(parsed.dataDir) };
} catch {
- return undefined;
+ return {};
}
}
@@ -216,8 +288,11 @@ export function stalledMediaLocation(
): ChannelMediaLocation {
return {
dataDir,
+ mediaLink: path.join(path.dirname(dataDir), MEDIA_LINK_NAME),
relocated: true,
target: configured,
+ // The text is on the corpus disk: a stalled MEDIA drive does not hold it.
+ text: { dir: dataDir, readable: true },
status: "stalled",
detail: health
? `${NOT_ANSWERING} (location "${health.label}", ${sinceText(health.since)})`
@@ -227,12 +302,25 @@ export function stalledMediaLocation(
}
// THE STALL, for a caller holding a parsed config: the stalled location the
-// channel's media is on, or null. No I/O — the question every page and poll
-// that reads a channel's `data/` asks before it does.
+// channel's MEDIA is on, or null. No I/O — the question a page or poll that
+// opens a big file asks before it does. Keys on `mediaDir`; on a legacy
+// channel (no `mediaDir` yet) on its retired `dataDir`, where everything is.
export function channelMediaStall(
- config: Pick<ChannelConfig, "dataDir"> | null | undefined,
+ config: ChannelMediaConfig | null | undefined,
+): LocationHealth | null {
+ const c = configuredOf(config);
+ const dir = c.mediaDir ?? c.dataDir;
+ return dir ? stalledLocationForPath(dir) : null;
+}
+
+// THE STALL OF THE TEXT: non-null only for a legacy channel whose retired
+// `dataDir` is on a stalled location — the one layout whose text is on another
+// drive. A page that reads only text (a listing, a transcript) asks this, not
+// `channelMediaStall`: a stalled media drive never holds the text.
+export function channelTextStall(
+ config: ChannelMediaConfig | null | undefined,
): LocationHealth | null {
- const dir = config?.dataDir?.trim();
+ const dir = configuredOf(config).dataDir;
return dir ? stalledLocationForPath(dir) : null;
}
@@ -240,12 +328,12 @@ export function channelMediaStall(
// The memo
// ---------------------------------------------------------------------------
//
-// FIVE SECONDS, PER CHANNEL, KEYED BY SLUG AND THE CONFIGURED TARGET. The home
+// FIVE SECONDS, PER CHANNEL, KEYED BY SLUG AND THE CONFIGURED TARGETS. The home
// page, /channels and the auto-queue status poll (every three seconds, four
-// lanes) each inspect every channel; without this each of them costs three
+// lanes) each inspect every channel; without this each of them costs a few
// syscalls a relocated channel, every time, on a drive that may be the slow
-// one. The key carries the configured `dataDir`, so a move that rewrites it is
-// a new key at once, and the movers clear the memo outright
+// one. The key carries the configured `mediaDir` and `dataDir`, so a move that
+// rewrites either is a new key at once, and the movers clear the memo outright
// (`forgetChannelMedia`) whenever a marker is written or removed.
//
// THE STALL GATE IS ASKED BEFORE A REMEMBERED ANSWER IS GIVEN, so a drive that
@@ -300,33 +388,28 @@ export function forgetChannelMedia(slug?: string): void {
}
}
-function memoKey(paths: ChannelMediaPaths, slug: string, configured?: string): string {
- return `${paths.channelsDir}\u0000${slug}\u0000${configured ?? ""}`;
+function memoKey(paths: ChannelMediaPaths, slug: string, c: Configured): string {
+ return `${paths.channelsDir}\u0000${slug}\u0000${c.mediaDir ?? ""}\u0000${c.dataDir ?? ""}`;
}
// Past this many entries, expired ones are swept on insert. A corpus has tens
// of channels; this only matters to a process that inspects many corpora.
const MEMO_SWEEP_AT = 512;
-// Two stats and (at most) one small JSON read. Render-safe: nothing here walks a
-// directory, so calling it per channel on a listing page costs three syscalls a
-// row — or none, for five seconds after the last answer (see the memo above).
-// On a stalled location the one call that reaches the drive (the target's
-// `stat`) is not made; the marker, the link and config.json are on the corpus
-// disk and are read as usual.
+// A few lstats and (at most) one small JSON read. Render-safe: nothing here
+// walks a directory, so calling it per channel on a listing page costs a few
+// syscalls a row — or none, for five seconds after the last answer (see the
+// memo above). On a stalled location the one call that reaches the drive (the
+// media target's `stat`) is not made; the marker, the links and config.json are
+// on the corpus disk and are read as usual.
export async function inspectChannelMedia(
paths: ChannelMediaPaths,
slug: string,
- config?: Pick<ChannelConfig, "dataDir"> | null,
+ config?: ChannelMediaConfig | null,
opts: InspectOptions = {},
): Promise<ChannelMediaLocation> {
- const dataDir = channelMediaDir(paths, slug);
const configured =
- config === undefined
- ? await readConfiguredDataDir(paths, slug)
- : config?.dataDir && config.dataDir.trim() !== ""
- ? config.dataDir.trim()
- : undefined;
+ config === undefined ? await readConfigured(paths, slug) : configuredOf(config);
const now = opts.now ?? Date.now();
const key = memoKey(paths, slug, configured);
@@ -334,14 +417,17 @@ export async function inspectChannelMedia(
if (!opts.fresh) {
const hit = memo.get(key);
if (hit && now - hit.at < CHANNEL_MEDIA_MEMO_MS) {
- if (hit.location.status !== "in-transition" && configured) {
- const stall = stalledLocationForPath(configured);
- if (stall) return stalledMediaLocation(dataDir, configured, stall);
+ const st = hit.location.status;
+ if (st !== "in-transition" && st !== "legacy" && configured.mediaDir) {
+ const stall = stalledLocationForPath(configured.mediaDir);
+ if (stall) {
+ return stalledMediaLocation(hit.location.dataDir, configured.mediaDir, stall);
+ }
}
- return { ...hit.location };
+ return cloneLocation(hit.location);
}
}
- const location = await inspectOnDisk(paths, slug, dataDir, configured);
+ const location = await inspectOnDisk(paths, slug, configured);
// A stall is not remembered: the health state is already its memory.
if (!opts.fresh && location.status !== "stalled") {
if (memo.size >= MEMO_SWEEP_AT) {
@@ -351,25 +437,53 @@ export async function inspectChannelMedia(
}
memo.set(key, { at: now, location });
}
- return { ...location };
+ return cloneLocation(location);
+}
+
+function cloneLocation(l: ChannelMediaLocation): ChannelMediaLocation {
+ return { ...l, text: { ...l.text } };
+}
+
+// `data/` on the corpus disk: absent (nothing downloaded yet — readable), a
+// directory (readable), a link (the retired layout), or something else.
+async function textState(
+ dataDir: string,
+): Promise<"absent" | "dir" | "link" | "other"> {
+ try {
+ const l = await lstat(dataDir);
+ if (l.isSymbolicLink()) return "link";
+ return l.isDirectory() ? "dir" : "other";
+ } catch {
+ return "absent";
+ }
}
async function inspectOnDisk(
paths: ChannelMediaPaths,
slug: string,
- dataDir: string,
- configured: string | undefined,
+ configured: Configured,
): Promise<ChannelMediaLocation> {
+ const dataDir = channelMediaDir(paths, slug);
+ const mediaLink = path.join(paths.channelsDir, slug, MEDIA_LINK_NAME);
+ const { mediaDir } = configured;
+ const text = await textState(dataDir);
+ const textReadable = text === "absent" || text === "dir";
+ const base = { dataDir, mediaLink };
+
// THE MARKER FIRST. It is in the channel dir, on the corpus disk, so reading
// it costs the drive nothing — and a channel mid-move reads `in-transition`
// whatever its drive is doing, which is what a resumed move and every guard
- // key off.
+ // key off. A media move leaves the text readable; a tier migration does not.
const marker = await readRelocationMarker(paths, slug);
if (marker) {
return {
- dataDir,
- relocated: Boolean(configured),
- target: configured ?? marker.target,
+ ...base,
+ relocated: Boolean(mediaDir ?? configured.dataDir),
+ target: mediaDir ?? marker.target,
+ text: {
+ dir: dataDir,
+ readable: textReadable && marker.scope !== "tier-migration",
+ },
status: "in-transition",
detail:
`a media relocation (${marker.direction}) is in progress or was ` +
@@ -378,30 +492,67 @@ async function inspectOnDisk(
};
}
- // THE GATE: a channel whose configured target is on a location whose drive
- // is not answering is answered from memory, before the link is looked at and
- // before the target's stat, which would hold an I/O thread for as long as the
- // drive takes.
- if (configured) {
- const stall = stalledLocationForPath(configured);
- if (stall) return stalledMediaLocation(dataDir, configured, stall);
+ // THE RETIRED LAYOUT, from the corpus disk alone: a `data` link or a
+ // recorded `dataDir`. Never a call to the far drive — the migration is what
+ // reads it, with the editor stopped.
+ if (text === "link" || configured.dataDir) {
+ let linkTarget: string | undefined;
+ if (text === "link") {
+ try {
+ linkTarget = await readlink(dataDir);
+ } catch {
+ /* unreadable: the config's value, if any, stands */
+ }
+ }
+ return {
+ ...base,
+ relocated: true,
+ target: configured.dataDir ?? linkTarget,
+ text: { dir: dataDir, readable: false },
+ status: "legacy",
+ detail: legacyDetail(slug),
+ };
+ }
+
+ const textRecord = { dir: dataDir, readable: textReadable };
+ if (!textReadable) {
+ return {
+ ...base,
+ relocated: Boolean(mediaDir),
+ target: mediaDir,
+ text: textRecord,
+ status: "inconsistent",
+ detail: `${dataDir} is neither a directory nor a symlink`,
+ };
+ }
+
+ // THE GATE: a channel whose media is on a location whose drive is not
+ // answering is answered from memory, before the link is looked at and before
+ // the target's stat, which would hold an I/O thread for as long as the drive
+ // takes. Its text stays readable.
+ if (mediaDir) {
+ const stall = stalledLocationForPath(mediaDir);
+ if (stall) return stalledMediaLocation(dataDir, mediaDir, stall);
}
let link: Awaited<ReturnType<typeof lstat>> | null = null;
try {
- link = await lstat(dataDir);
+ link = await lstat(mediaLink);
} catch {
- // No data/ at all. With no configured target that is just a channel that
- // has downloaded nothing yet — the overwhelmingly common case, and not an
- // error. With one, the link this channel is supposed to have is gone.
- if (!configured) return { dataDir, relocated: false, status: "in-place" };
+ // No media link at all. With no configured target that is a classic
+ // channel — its media are real files in `data/<id>/` — the overwhelmingly
+ // common case, and not an error. With one, the link is gone.
+ if (!mediaDir) {
+ return { ...base, relocated: false, text: textRecord, status: "in-place" };
+ }
return {
- dataDir,
+ ...base,
relocated: true,
- target: configured,
+ target: mediaDir,
+ text: textRecord,
status: "inconsistent",
detail:
- `config.json records dataDir ${configured} but ${dataDir} does not ` +
+ `config.json records mediaDir ${mediaDir} but ${mediaLink} does not ` +
`exist — the symlink is missing`,
};
}
@@ -409,34 +560,36 @@ async function inspectOnDisk(
if (link.isSymbolicLink()) {
let linkTarget = "";
try {
- linkTarget = await readlink(dataDir);
+ linkTarget = await readlink(mediaLink);
} catch {
/* readlink of a link we just lstat'd: treat as unreadable below */
}
- if (!configured) {
+ if (!mediaDir) {
return {
- dataDir,
+ ...base,
relocated: false,
target: linkTarget || undefined,
+ text: textRecord,
status: "inconsistent",
detail:
- `${dataDir} is a symlink to ${linkTarget || "(unreadable)"} but ` +
- `config.json records no dataDir`,
+ `${mediaLink} is a symlink to ${linkTarget || "(unreadable)"} but ` +
+ `config.json records no mediaDir`,
};
}
- if (path.resolve(linkTarget) !== path.resolve(configured)) {
+ if (path.resolve(linkTarget) !== path.resolve(mediaDir)) {
return {
- dataDir,
+ ...base,
relocated: true,
- target: configured,
+ target: mediaDir,
+ text: textRecord,
status: "inconsistent",
detail:
- `${dataDir} points at ${linkTarget || "(unreadable)"} but ` +
- `config.json records ${configured}`,
+ `${mediaLink} points at ${linkTarget || "(unreadable)"} but ` +
+ `config.json records ${mediaDir}`,
};
}
- // The link points at a DEEP path (<root>/<slug>/data), so an unmounted root
- // gives ENOENT here. An empty mountpoint can never be mistaken for the
+ // The link points at a DEEP path (<root>/<slug>/media), so an unmounted
+ // root gives ENOENT here. An empty mountpoint can never be mistaken for the
// media, which is the whole reason the suffix is fixed.
//
// THE ONE CALL HERE THAT REACHES THE DRIVE, so it goes through the
@@ -444,55 +597,61 @@ async function inspectOnDisk(
// not answered within the budget (`storage.health.budgetMs`, 3 s by
// default) marks it stalled and answers `stalled` now.
try {
- const st = await onDrive(configured, () => stat(configured));
+ const st = await onDrive(mediaDir, () => stat(mediaDir));
if (!st.isDirectory()) {
return {
- dataDir,
+ ...base,
relocated: true,
- target: configured,
+ target: mediaDir,
+ text: textRecord,
status: "unreachable",
- detail: `${configured} exists but is not a directory`,
+ detail: `${mediaDir} exists but is not a directory`,
};
}
} catch (err) {
if (isDriveNotAnswering(err)) {
- return stalledMediaLocation(dataDir, configured, err.health, err.message);
+ return stalledMediaLocation(dataDir, mediaDir, err.health, err.message);
}
return {
- dataDir,
+ ...base,
relocated: true,
- target: configured,
+ target: mediaDir,
+ text: textRecord,
status: "unreachable",
- detail: `${configured} does not exist (drive not mounted?)`,
+ detail: `${mediaDir} does not exist (drive not mounted?)`,
};
}
- return { dataDir, relocated: true, target: configured, status: "ok" };
+ return { ...base, relocated: true, target: mediaDir, text: textRecord, status: "ok" };
}
if (!link.isDirectory()) {
return {
- dataDir,
- relocated: Boolean(configured),
- target: configured,
+ ...base,
+ relocated: Boolean(mediaDir),
+ target: mediaDir,
+ text: textRecord,
status: "inconsistent",
- detail: `${dataDir} is neither a directory nor a symlink`,
+ detail: `${mediaLink} is neither a directory nor a symlink`,
};
}
- if (configured) {
+ if (mediaDir) {
return {
- dataDir,
+ ...base,
relocated: true,
- target: configured,
+ target: mediaDir,
+ text: textRecord,
status: "inconsistent",
detail:
- `config.json records dataDir ${configured} but ${dataDir} is a real ` +
+ `config.json records mediaDir ${mediaDir} but ${mediaLink} is a real ` +
`directory — the media was never moved, or was moved back by hand`,
};
}
- return { dataDir, relocated: false, status: "in-place" };
+ // Tiered in place: `media/` is a real directory on the corpus disk.
+ return { ...base, relocated: false, text: textRecord, status: "in-place" };
}
+// THE MEDIA GUARD, for a job that opens or writes a BIG file.
// "ok" and "in-place" pass; everything else throws. An in-transition or
// inconsistent channel is refused for the same reason an unreachable one is:
// the caller would otherwise read a half-populated or empty dir as the truth.
@@ -500,18 +659,10 @@ async function inspectOnDisk(
//
// ALWAYS FRESH: this is the start-of-work guard, and a remembered "ok" from a
// few seconds ago is not what a job about to read `data/` should be told.
-//
-// THIS CHECK HAS A TWIN. `checkChannelReachable` in
-// `umtool/report-to-video/cues.mjs` repeats the same statuses in plain `.mjs`,
-// because umtool's bins run under bare node with no `tsx` and cannot import
-// this module. It guards the same failure: a relocated channel whose drive is
-// not mounted reads as ENOENT, and the cue resolver would otherwise answer from
-// the published archive — cutting clips from a snapshot's cues instead of the
-// corpus's. Change the checks here and change them there.
export async function assertChannelMediaReachable(
paths: ChannelMediaPaths,
slug: string,
- config?: Pick<ChannelConfig, "dataDir"> | null,
+ config?: ChannelMediaConfig | null,
): Promise<ChannelMediaLocation> {
const location = await inspectChannelMedia(paths, slug, config, {
fresh: true,
@@ -521,3 +672,39 @@ export async function assertChannelMediaReachable(
}
throw new ChannelMediaUnreachableError(slug, location);
}
+
+// THE TEXT GUARD (release 17): passes for every channel whose `data/` is a
+// readable directory on the corpus disk — whatever its MEDIA is doing. A
+// stalled, unreachable, inconsistent or moving media tier does not hold a
+// reader of the text: the index and stats builds, the snapshot, the digests,
+// normalize. Refused: a `legacy` channel (its text is on the far drive), a
+// `data/` that is not a directory, a tier migration in flight.
+//
+// ALWAYS FRESH, for the reason the media guard is.
+//
+// THIS CHECK HAS A TWIN. `checkChannelReachable` in
+// `umtool/report-to-video/cues.mjs` repeats it in plain `.mjs` (refuse a
+// legacy channel — a `data` link or a recorded `dataDir`; refuse a marker only
+// when its `scope` is `tier-migration`), because umtool's bins run under bare
+// node with no `tsx` and cannot import this module. The cue resolver reads
+// TEXT; on a legacy channel whose drive is not mounted the cues read as
+// ENOENT, and it would otherwise answer from the published archive — cutting
+// clips from a snapshot's cues instead of the corpus's. Change the checks here
+// and change them there.
+export async function assertChannelTextReadable(
+ paths: ChannelMediaPaths,
+ slug: string,
+ config?: ChannelMediaConfig | null,
+): Promise<ChannelMediaLocation> {
+ const location = await inspectChannelMedia(paths, slug, config, {
+ fresh: true,
+ });
+ if (location.text.readable) return location;
+ const detail =
+ location.status === "legacy"
+ ? (location.detail ?? legacyDetail(slug))
+ : location.status === "in-transition"
+ ? `its media layout is being migrated (${location.detail ?? "a marker is present"})`
+ : `${location.dataDir} is not a readable directory`;
+ throw new ChannelTextUnreadableError(slug, location, detail);
+}
diff --git a/common/lib/channelMediaHold.ts b/common/lib/channelMediaHold.ts
@@ -7,26 +7,39 @@ import {
type StorageLocation,
} from "./storageLocations";
-// THE HOLD, in the words both pool-wide builds use.
+// THE HOLD, in the words the builds, the lanes and the surfaces use.
//
// The index build (controller/buildIndex.ts) and the stats build
-// (controller/buildStats.ts) each walk every channel's `data/`. A channel whose
-// media cannot be read — a relocated `data/` on an unmounted drive, a move in
-// progress, a link and a config that disagree — is HELD by both: not rescanned,
-// and what the last build knew of it kept, rather than read as a channel with
-// no videos and removed (lib/channelMedia.ts says why that reading is the
-// dangerous one). This module is only the shared vocabulary: which statuses
+// (controller/buildStats.ts) each walk every channel's `data/`. Since release 17
+// they read the TEXT tier only and are never held by the media tier: a channel
+// whose text cannot be read — the retired whole-directory layout (`legacy`),
+// a `data/` that is not a directory — is HELD by both: not rescanned, and what
+// the last build knew of it kept, rather than read as a channel with no videos
+// and removed (lib/channelMedia.ts says why that reading is the dangerous one).
+// The media hold (an unmounted media drive, a move in progress, a link and a
+// config that disagree) holds the lanes and jobs that open a big file. This module is only the shared vocabulary: which statuses
// hold, why, in words with no path in them (/storage shows the paths), and the
// ways out a refusal names. Each build decides for itself what "kept" means.
//
// Pure: no I/O. The caller asks inspectChannelMedia and passes the answer in.
// "ok" and "in-place" are read; every other status holds, including any a later
-// inspectChannelMedia adds.
+// inspectChannelMedia adds (`legacy` among them). The MEDIA hold: what the
+// transcription, download and backfill lanes and every media job key on.
export function isMediaHeld(status: ChannelMediaStatus): boolean {
return status !== "ok" && status !== "in-place";
}
+// THE TEXT HOLD (release 17): only the retired whole-directory layout holds the
+// text, because only there is it on another drive. The digest lane keys on
+// this, so a digest runs on a channel whose media is moving, stalled or
+// unmounted. (The text guard, `assertChannelTextReadable`, also refuses a
+// `data/` that is not a directory and a tier migration in flight — conditions
+// a status alone does not carry; a lane that holds a location asks it.)
+export function isTextHeld(status: ChannelMediaStatus): boolean {
+ return status === "legacy";
+}
+
// Why a channel is held, without the paths inspectChannelMedia's `detail`
// carries.
//
@@ -40,6 +53,8 @@ export const HELD_REASON: Record<ChannelMediaStatus, string> = {
"in-transition": "its media is moving (a move is in progress or was interrupted)",
inconsistent: "its data link and its config disagree",
stalled: "its drive is not answering (a stalled disk)",
+ legacy:
+ "its media layout is the retired whole-directory one (run archilyzer storage migrate-tier)",
ok: "reachable",
"in-place": "reachable",
};
@@ -54,14 +69,15 @@ export function mediaHoldText(status: ChannelMediaStatus): string | null {
}
// The reason, and the storage location's label when the channel's media is on
-// one. `dataDir` is the channel config's; the inspector's target stands in when
-// the config names none (a move in flight).
+// one. `mediaDir` is the channel config's (`mediaDir`, or the retired `dataDir`
+// on a legacy channel); the inspector's target stands in when the config names
+// none (a move in flight).
export function heldReason(
media: Pick<ChannelMediaLocation, "status" | "target">,
- dataDir: string | undefined,
+ mediaDir: string | undefined,
locations: StorageLocation[],
): string {
- const label = locationLabelOfDataDir(dataDir ?? media.target, locations);
+ const label = locationLabelOfDataDir(mediaDir ?? media.target, locations);
return `${HELD_REASON[media.status]}${label ? `, on location "${label}"` : ""}`;
}
diff --git a/common/lib/fileSchemaDocs.ts b/common/lib/fileSchemaDocs.ts
@@ -162,7 +162,7 @@ export function renderChannelMarkdown(): string {
"Every other key is optional and has NO default of its own: an absent " +
"key means whatever its description says — for the per-channel " +
"overrides, inherit the global setting of the same name; for `name`, " +
- "`url`, `dataDir`, `subLangs` and the sync-state stamps, simply unset. " +
+ "`url`, `mediaDir`, `subLangs` and the sync-state stamps, simply unset. " +
"So an ill-typed or out-of-range value is not coerced — it is DROPPED, " +
"as if the file did not spell it. Unknown keys (including the retired " +
"`excludeFromSync`, now a paused `sync` tier in the channel-priority " +
@@ -177,7 +177,7 @@ export function renderChannelMarkdown(): string {
"and changes only its own keys, so a stamp and a form save made at once " +
"in the editor both land. The two exceptions write a whole config, and " +
"only when there is no readable file to patch: a media move and a " +
- "channel rename record `dataDir` from their own copy of the config.",
+ "channel rename record `mediaDir` from their own copy of the config.",
);
out.push("");
out.push(REGENERATE);
diff --git a/common/lib/storageHealth.test.ts b/common/lib/storageHealth.test.ts
@@ -681,3 +681,11 @@ test("DT review L4: a timeout on no known location names the budget the call ran
setDriveCallBudget(5_000);
assert.equal(await refused, "drive not answering (a read did not answer within 0.08 s)");
});
+
+test("rootOfUnknownPath strips <root>/<slug>/media (release 17) and the retired <root>/<slug>/data", async () => {
+ const { rootOfUnknownPath } = await import("./storageHealth");
+ assert.equal(rootOfUnknownPath("/mnt/p/chan/media"), "/mnt/p");
+ assert.equal(rootOfUnknownPath("/mnt/p/chan/media/"), "/mnt/p");
+ assert.equal(rootOfUnknownPath("/mnt/p/chan/data"), "/mnt/p");
+ assert.equal(rootOfUnknownPath("/mnt/p/chan/other"), "/mnt/p/chan/other");
+});
diff --git a/common/lib/storageHealth.ts b/common/lib/storageHealth.ts
@@ -524,12 +524,14 @@ function slotKeyOfLocation(id: string): string {
return `loc:${id}`;
}
-// The root a path on no configured location is under: `<root>/<slug>/data`
-// (relocatedDataDir's shape) gives `<root>`; anything else is its own key.
-function rootOfUnknownPath(p: string): string {
+// The root a path on no configured location is under: `<root>/<slug>/media`
+// (relocatedMediaDir's shape, release 17) or the retired `<root>/<slug>/data`
+// gives `<root>`; anything else is its own key.
+export function rootOfUnknownPath(p: string): string {
const clean = p.replace(/\/+$/, "");
const parts = clean.split("/");
- return parts.length > 2 && parts[parts.length - 1] === "data"
+ const last = parts[parts.length - 1];
+ return parts.length > 2 && (last === "media" || last === "data")
? parts.slice(0, -2).join("/") || "/"
: clean;
}
diff --git a/editor/app/channels/[slug]/shardActions.ts b/editor/app/channels/[slug]/shardActions.ts
@@ -9,7 +9,7 @@ import {
type ShardOp,
} from "yt-dlp-transcript-common/controller/shard";
import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels";
-import { assertChannelMediaReachable } from "yt-dlp-transcript-common/lib/channelMedia";
+import { assertChannelTextReadable } from "yt-dlp-transcript-common/lib/channelMedia";
import { runYtdlp } from "yt-dlp-transcript-common/ytdlp/runYtdlp";
import { runWhisperBatch } from "yt-dlp-transcript-common/controller/whisperBatch";
import { runAvailabilityCheck } from "yt-dlp-transcript-common/controller/checkAvailability";
@@ -84,8 +84,12 @@ export async function saveShardConfigAction(
//
// It is at the top rather than in the download branch alone because all three
// read the same dirs for the same reason.
+ //
+ // THE TEXT GUARD (release 17): the slices are computed over the video dirs
+ // in `data/` — the text tier, on the corpus disk — so only an unreadable
+ // text tier (a `legacy` channel) can make them wrong.
try {
- await assertChannelMediaReachable(paths, slug);
+ await assertChannelTextReadable(paths, slug);
} catch (e) {
return { ok: false, error: (e as Error).message };
}
diff --git a/editor/app/components/MediaLocationBadge.tsx b/editor/app/components/MediaLocationBadge.tsx
@@ -54,6 +54,9 @@ const LABELS: Record<ChannelMediaStatus, string | null> = {
"in-transition": "Media moving",
inconsistent: "Media inconsistent",
stalled: "Media not answering",
+ // Release 17: the retired whole-directory layout (`archilyzer storage
+ // migrate-tier`). Added by slice T1 so the status union stays exhaustive.
+ legacy: "Media layout retired",
};
// The one-word state, for the compact rendering of a NAMED location: "on
@@ -66,6 +69,7 @@ const SHORT_STATUS: Record<ChannelMediaStatus, string | null> = {
"in-transition": "moving",
inconsistent: "inconsistent",
stalled: "not answering",
+ legacy: "layout retired",
};
// Null means "draw nothing" — an in-place channel, or no location at all (a
