commit 9cf4ec234dc395a3c5014957da9c4d51f4b9692c
parent b7b17bc1036f21f09eafc51d727a2f92d7339a16
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 1 Oct 2026 17:26:58 -0400
common, umtool: the lanes by tier, normalize-live-chat's media guard, the cues twin as a text guard
isChannelHeldForLane is the lane hold (digest: the text hold; the rest: the
media hold) and is tested over real inspect answers; normalizeAllLiveChat
skips a channel whose media is not reachable; the needsMedia -> needsText
flips are pinned; a text job runs over an unmounted media drive and is
refused on a legacy channel; the finalisers tier and a deleter derefs, at
their call sites. umtool's cues.mjs twin refuses a legacy channel (a data
link or a recorded dataDir, mounted or not) and a marker only when its scope
is tier-migration.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
8 files changed, 516 insertions(+), 134 deletions(-)
diff --git a/common/controller/autoRunner.test.ts b/common/controller/autoRunner.test.ts
@@ -1,6 +1,6 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import {
@@ -12,6 +12,7 @@ import {
priorityContextFor,
resetPriorityContextForTest,
sharedAutoQueueState,
+ isChannelHeldForLane,
} from "./autoRunner";
import {
defaultChannelPriority,
@@ -33,7 +34,7 @@ import type {
} from "../jobs/autoQueuePolicy";
import type { Paths } from "../lib/paths";
import type { SiteSettings } from "../lib/settings";
-import { forgetChannelMedia } from "../lib/channelMedia";
+import { forgetChannelMedia, inspectChannelMedia } from "../lib/channelMedia";
import { bucketLaneOperationId } from "../lib/operations";
// THE RUNNER'S HALF OF S1 (plans/channel-priority.md), and it is here rather
@@ -711,6 +712,9 @@ test("a channel with a move marker projects no work on any lane, and the hold li
);
forgetChannelMedia(slug);
for (const lane of LANES) {
+ // (Release 17: the digest lane is held only by the text hold, and a media
+ // move leaves the text readable — `isChannelHeldForLane` below. This
+ // fixture has no digest work, so its total is 0 either way.)
assert.equal(
await laneTotal(lane, paths, configs),
0,
@@ -723,3 +727,66 @@ test("a channel with a move marker projects no work on any lane, and the hold li
forgetChannelMedia(slug);
assert.ok((await laneTotal("transcription", paths, configs)) > 0);
});
+
+
+// --- Release 17: the lanes by tier ------------------------------------------
+//
+// A stalled, unmounted or moving MEDIA tier holds the lanes that open the big
+// files (transcription, download, backfill) and lets the digest lane run: a
+// digest reads and writes text, which stays on the corpus disk. Only the text
+// hold — a `legacy` channel, a tier migration in flight — keeps the digest lane
+// off a channel. Decided from the inspector's real answers over fixtures.
+
+test("release 17: an unmounted media drive holds transcription/download/backfill, lets digest run; legacy holds all four", async () => {
+ const paths = pendingFixture();
+ const slug = "tiered";
+ const channelDir = path.join(paths.channelsDir, slug);
+ mkdirSync(path.join(channelDir, "data", "v1"), { recursive: true });
+ const target = path.join(path.dirname(paths.channelsDir), "platter", slug, "media");
+ const writeCfg = (extra: Record<string, unknown>) =>
+ writeFileSync(
+ path.join(channelDir, "config.json"),
+ JSON.stringify({ url: "https://www.youtube.com/@t", handling: "transcribe", ...extra }),
+ );
+ writeCfg({ mediaDir: target });
+ symlinkSync(target, path.join(channelDir, "media")); // dangling: not mounted
+ const unmounted = await inspectChannelMedia(paths, slug, undefined, { fresh: true });
+ assert.equal(unmounted.status, "unreachable");
+ const heldBy = (loc: Parameters<typeof isChannelHeldForLane>[1]) =>
+ Object.fromEntries(LANES.map((lane) => [lane, isChannelHeldForLane(lane, loc)]));
+ assert.deepEqual(heldBy(unmounted), {
+ transcription: true,
+ download: true,
+ digest: false,
+ backfill: true,
+ });
+
+ // A media move in flight: the same.
+ writeFileSync(
+ path.join(channelDir, ".relocating.json"),
+ JSON.stringify({ target, direction: "out", startedAt: "", phase: "copy", scope: "media" }),
+ );
+ const moving = await inspectChannelMedia(paths, slug, undefined, { fresh: true });
+ assert.equal(moving.status, "in-transition");
+ assert.equal(heldBy(moving).digest, false);
+ assert.equal(heldBy(moving).transcription, true);
+ // A tier migration rebuilds data/ itself: the digest lane is held too.
+ writeFileSync(
+ path.join(channelDir, ".relocating.json"),
+ JSON.stringify({ target, direction: "out", startedAt: "", phase: "copy", scope: "tier-migration" }),
+ );
+ const migrating = await inspectChannelMedia(paths, slug, undefined, { fresh: true });
+ assert.equal(heldBy(migrating).digest, true);
+ rmSync(path.join(channelDir, ".relocating.json"));
+
+ // The retired layout holds every lane.
+ writeCfg({ dataDir: path.join(path.dirname(paths.channelsDir), "platter", slug, "data") });
+ const legacy = await inspectChannelMedia(paths, slug, undefined, { fresh: true });
+ assert.equal(legacy.status, "legacy");
+ assert.deepEqual(heldBy(legacy), {
+ transcription: true,
+ download: true,
+ digest: true,
+ backfill: true,
+ });
+});
diff --git a/common/controller/autoRunner.ts b/common/controller/autoRunner.ts
@@ -87,6 +87,7 @@ import { isGateHeld } from "../lib/pauseGates";
import {
inspectChannelMedia,
readRelocationMarker,
+ type ChannelMediaLocation,
type ChannelMediaStatus,
} from "../lib/channelMedia";
import { isMediaHeld, isTextHeld, mediaHoldText } from "../lib/channelMediaHold";
@@ -557,6 +558,21 @@ export function makeFocusHoldReporter(
// too — an operator watching the log sees the channel leave and come back.
const mediaSkipLogged = new Map<string, ChannelMediaStatus>();
+// WHETHER A LANE SKIPS A CHANNEL, from its inspect answer (release 17). The
+// digest lane writes only text, so only the text hold — a `legacy` channel, a
+// `data/` it cannot read, a tier migration in flight — keeps it off a channel;
+// it runs while the media is moving, stalled or unmounted. The transcription,
+// download and backfill lanes open the big files and keep the media hold.
+export function isChannelHeldForLane(
+ kind: AutoQueueKind,
+ location: Pick<ChannelMediaLocation, "status" | "text">,
+): boolean {
+ if (kind === "digest") {
+ return isTextHeld(location.status) || !location.text.readable;
+ }
+ return isMediaHeld(location.status);
+}
+
function noteSkippedForMedia(
slug: string,
status: ChannelMediaStatus,
@@ -644,12 +660,7 @@ async function buildChannelWork(
// transcription, download and backfill lanes open the big files and keep
// the media hold.
const location = media[i];
- const held = location
- ? kind === "digest"
- ? isTextHeld(location.status) || !location.text.readable
- : isMediaHeld(location.status)
- : false;
- if (location && held) {
+ if (location && isChannelHeldForLane(kind, location)) {
noteSkippedForMedia(slug, location.status, location.detail);
continue;
}
diff --git a/common/controller/mediaTierHooks.test.ts b/common/controller/mediaTierHooks.test.ts
@@ -0,0 +1,145 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ chmod,
+ lstat,
+ mkdir,
+ mkdtemp,
+ readFile,
+ readlink,
+ rename,
+ rm,
+ symlink,
+ writeFile,
+} from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import { tierVideoDir } from "../lib/mediaTier-server";
+import { transcodeAudio } from "./transcode";
+import { normalizeLiveChat, isLiveChatCuesFresh } from "./normalizeLiveChat";
+import { removeWrongFormatAudio } from "./removeWrongFormatAudio";
+import { normalizeAllLiveChat } from "./normalizeAllLiveChat";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/mediaTierHooks.test.ts
+//
+// RELEASE 17, THE MEDIA TIER, AT ITS CALL SITES: the finalisers call the hook
+// and the deleters go through the link. A channel tiered IN PLACE — a real
+// `channels/<slug>/media/` on the same disk — is the fixture: the hook's
+// cross-device path is lib/mediaTier-server.test.ts's.
+
+async function fixture() {
+ const root = await mkdtemp(path.join(tmpdir(), "media-hooks-"));
+ const paths = {
+ transcriptsDir: root,
+ channelsDir: path.join(root, "channels"),
+ ffmpegBin: path.join(root, "ffmpeg"),
+ } as Paths;
+ const channelDir = path.join(paths.channelsDir, "chan");
+ const videoDir = path.join(channelDir, "data", "v1");
+ const mediaDir = path.join(channelDir, "media");
+ await mkdir(videoDir, { recursive: true });
+ await mkdir(mediaDir);
+ await writeFile(
+ path.join(channelDir, "config.json"),
+ JSON.stringify({ handling: "transcribe", url: "https://example.com/c", audioFormat: "mp3" }),
+ );
+ await writeFile(
+ path.join(videoDir, "metadata.info.json"),
+ JSON.stringify({ id: "v1", title: "A video", duration: 60 }),
+ );
+ // A fake ffmpeg: copies its input (-i) to its last argument.
+ await writeFile(
+ paths.ffmpegBin,
+ '#!/bin/sh\nin=""\nprev=""\nfor a in "$@"; do if [ "$prev" = "-i" ]; then in="$a"; fi; prev="$a"; last="$a"; done\ncp "$in" "$last"\n',
+ );
+ await chmod(paths.ffmpegBin, 0o755);
+ return { root, paths, channelDir, videoDir, mediaDir };
+}
+
+test("transcodeAudio tiers its output, and re-tiers over a link the rename replaced", async () => {
+ const f = await fixture();
+ try {
+ await writeFile(path.join(f.videoDir, "audio.m4a"), "SOURCE-1");
+ const run = () =>
+ transcodeAudio({
+ paths: f.paths,
+ videoDir: f.videoDir,
+ sourceFilename: "audio.m4a",
+ targetFormat: "mp3",
+ onLog: () => {},
+ signal: new AbortController().signal,
+ });
+ await run();
+ const out = path.join(f.videoDir, "audio.mp3");
+ assert.ok((await lstat(out)).isSymbolicLink());
+ assert.equal(await readlink(out), "../../media/v1/audio.mp3");
+ assert.equal(await readFile(out, "utf8"), "SOURCE-1");
+ // Again, over the link: the stale bytes in media/ are replaced, the link
+ // made again — nothing orphaned, nothing left real.
+ await writeFile(path.join(f.videoDir, "audio.m4a"), "SOURCE-2");
+ await run();
+ assert.ok((await lstat(out)).isSymbolicLink());
+ assert.equal(await readFile(path.join(f.mediaDir, "v1", "audio.mp3"), "utf8"), "SOURCE-2");
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+});
+
+test("normalizeLiveChat tiers the raw replay after writing the cues; freshness never reaches the media drive", async () => {
+ const f = await fixture();
+ try {
+ await writeFile(path.join(f.videoDir, "transcript.live_chat.json"), "");
+ const first = await normalizeLiveChat({ videoDir: f.videoDir, channelSlug: "chan" });
+ assert.equal(first.status, "wrote");
+ const raw = path.join(f.videoDir, "transcript.live_chat.json");
+ assert.ok((await lstat(raw)).isSymbolicLink());
+ assert.ok((await lstat(path.join(f.videoDir, "live_chat.cues.json"))).isFile(), "the cues are text");
+ // The media drive goes away: the cues are still fresh, from the link's mtime.
+ await rename(f.mediaDir, `${f.mediaDir}-away`);
+ assert.equal((await isLiveChatCuesFresh(f.videoDir)).fresh, true);
+ assert.equal((await normalizeLiveChat({ videoDir: f.videoDir, channelSlug: "chan" })).status, "fresh");
+ await rename(`${f.mediaDir}-away`, f.mediaDir);
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+});
+
+test("a deleter derefs: removeWrongFormatAudio removes the bytes in media/, not just the link", async () => {
+ const f = await fixture();
+ try {
+ await writeFile(path.join(f.videoDir, "audio.mp3"), "KEEP");
+ await writeFile(path.join(f.videoDir, "audio.m4a"), "WRONG");
+ await tierVideoDir(f.videoDir);
+ assert.ok((await lstat(path.join(f.videoDir, "audio.m4a"))).isSymbolicLink());
+ const r = await removeWrongFormatAudio({ channelSlug: "chan", paths: f.paths, onLog: () => {} });
+ assert.equal(r.removedFiles, 1);
+ assert.equal(existsSync(path.join(f.mediaDir, "v1", "audio.m4a")), false, "no orphan on the media tier");
+ await assert.rejects(lstat(path.join(f.videoDir, "audio.m4a")));
+ assert.equal(await readFile(path.join(f.videoDir, "audio.mp3"), "utf8"), "KEEP");
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+});
+
+test("normalize-live-chat skips a channel whose media is not reachable", async () => {
+ const f = await fixture();
+ try {
+ // Relocated media on an unmounted drive.
+ await rm(f.mediaDir, { recursive: true });
+ const target = path.join(f.root, "platter", "chan", "media");
+ await writeFile(
+ path.join(f.channelDir, "config.json"),
+ JSON.stringify({ handling: "transcribe", url: "https://example.com/c", mediaDir: target }),
+ );
+ await symlink(target, f.mediaDir);
+ const lines: string[] = [];
+ const r = await normalizeAllLiveChat({ paths: f.paths, onLog: (l) => lines.push(l) });
+ assert.equal(r.failed, 1);
+ assert.ok(lines.some((l) => /Normalize live chat chan: SKIPPED — .*not reachable/.test(l)), lines.join("\n"));
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+});
diff --git a/common/controller/normalizeAllLiveChat.ts b/common/controller/normalizeAllLiveChat.ts
@@ -9,6 +9,7 @@ import pLimit from "p-limit";
import { listChannelStatsFromDisk } from "./channels";
import { normalizeLiveChat } from "./normalizeLiveChat";
import type { Paths } from "../lib/paths";
+import { assertChannelMediaReachable } from "../lib/channelMedia";
export type NormalizeAllLiveChatOptions = {
paths: Paths;
@@ -38,6 +39,18 @@ export async function normalizeAllLiveChat(
};
for (const ch of channels) {
if (opts.signal?.aborted) break;
+ // THE MEDIA GUARD (release 17): the raw replay is media — on a tiered
+ // channel a link into channels/<slug>/media, possibly on another drive —
+ // and a stale cues file is re-derived by READING it. A channel whose media
+ // is not reachable (unmounted, stalled, moving, legacy) is skipped and
+ // counted, never read as a clean pass over zero videos.
+ try {
+ await assertChannelMediaReachable(opts.paths, ch.slug, ch.config);
+ } catch (err) {
+ log(`Normalize live chat ${ch.slug}: SKIPPED — ${(err as Error).message}`);
+ result.failed++;
+ continue;
+ }
const dataDir = path.join(opts.paths.channelsDir, ch.slug, "data");
const videoIds = await readdir(dataDir).catch(() => [] as string[]);
log(`Normalize live chat ${ch.slug}: ${videoIds.length} videos`);
diff --git a/common/jobs/jobKinds.test.ts b/common/jobs/jobKinds.test.ts
@@ -1,6 +1,12 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { isDrainableKind, jobKindLabel, getJobKind } from "./jobKinds";
+import {
+ isDrainableKind,
+ jobKindLabel,
+ getJobKind,
+ kindNeedsMedia,
+ kindNeedsText,
+} from "./jobKinds";
// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test jobs/jobKinds.test.ts
//
@@ -120,3 +126,72 @@ test("label-less and unknown kinds fall back to the raw kind", () => {
assert.equal(jobKindLabel("totally-unknown"), "totally-unknown");
assert.equal(getJobKind("totally-unknown"), undefined);
});
+
+
+// RELEASE 17: `needsMedia` means "opens or writes the BIG file". These kinds
+// read only the text tier and flipped to `needsText`; the rest stay media.
+const TEXT_KINDS = [
+ "auto-digest",
+ "digest-channel-local",
+ "digest-channel-remote",
+ "digest-share-cluster",
+ "normalize-transcripts",
+ "purge-superseded-auto-subs",
+ "fetch-window",
+ "evict-clips",
+ "metadata-scan",
+ "download-missing-subs",
+ "check-availability",
+ "quick-availability-check",
+ "check-maybe-missing",
+ "check-kept-deleted",
+];
+const STILL_MEDIA = [
+ "auto-transcribe",
+ "auto-download",
+ "auto-download-unit",
+ "auto-backfill",
+ "whisper-all",
+ "whisper-bucket-downloaded-no-transcript",
+ "whisper-bucket-auto-subs",
+ "sync",
+ "import-one",
+ "download-from-playlist",
+ "download-missing",
+ "redownload-archive",
+ "redownload-incomplete-bucket",
+ "retry-bucket",
+ "persist-kept",
+ "whisper-video",
+ "transcribe-one",
+ "download-one-pipeline",
+ "transcode-audio",
+ "diarize-channel",
+ "backfill-channel",
+ "scan-media",
+ "scan-media-channel",
+ "clean-audio-transcribed",
+ "clean-extra-audio-formats",
+ "remove-wrong-format-audio",
+];
+
+test("release 17: the text kinds flipped from needsMedia to needsText", () => {
+ for (const k of TEXT_KINDS) {
+ assert.ok(getJobKind(k), `${k} is registered`);
+ assert.equal(kindNeedsMedia(k), false, `${k} does not open a big file`);
+ assert.equal(kindNeedsText(k), true, `${k} reads the text tier`);
+ }
+});
+
+test("release 17: every media kind keeps needsMedia, and none is also a text kind", () => {
+ for (const k of STILL_MEDIA) {
+ assert.ok(getJobKind(k), `${k} is registered`);
+ assert.equal(kindNeedsMedia(k), true, k);
+ assert.equal(kindNeedsText(k), false, k);
+ }
+ // The movers fix an unreachable channel and declare neither.
+ for (const k of ["relocate-channel-media", "relocate-saved-videos", "repoint-storage-location"]) {
+ assert.equal(kindNeedsMedia(k), false, k);
+ assert.equal(kindNeedsText(k), false, k);
+ }
+});
diff --git a/common/jobs/streamCommand.test.ts b/common/jobs/streamCommand.test.ts
@@ -1,6 +1,6 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import type { Paths } from "../lib/paths";
@@ -217,6 +217,68 @@ test("a media job queued before a move's marker and started after it refuses at
}
});
+// RELEASE 17: a TEXT kind (`needsText`) asks the text guard. It runs while the
+// channel's media drive is unmounted, and is refused — before any record —
+// only where the text itself cannot be read: the retired layout.
+test("a text job runs over an unmounted media drive and is refused on a legacy channel", async () => {
+ const { paths: base, root } = await jobsDir();
+ const paths = { ...base, channelsDir: path.join(root, "channels") } as Paths;
+ const channelDir = path.join(paths.channelsDir, "alpha");
+ await mkdir(path.join(channelDir, "data"), { recursive: true });
+ const target = path.join(root, "platter", "alpha", "media");
+ await writeFile(
+ path.join(channelDir, "config.json"),
+ JSON.stringify({ handling: "youtube", url: "https://example.com/a", mediaDir: target }),
+ );
+ await symlink(target, path.join(channelDir, "media")); // not mounted
+ try {
+ let ran = 0;
+ const job = ok(
+ await runManagedFunction({
+ kind: "normalize-transcripts",
+ queueKey: `test:text-guard:${newJobId()}`,
+ paths,
+ channelSlug: "alpha",
+ fn: async () => {
+ ran += 1;
+ },
+ }),
+ );
+ assert.equal((await job.done).status, "done");
+ assert.equal(ran, 1);
+ // A media kind on the same channel is refused, before any record.
+ const media = await runManagedFunction({
+ kind: "whisper-all",
+ queueKey: `test:text-guard:${newJobId()}`,
+ paths,
+ channelSlug: "alpha",
+ fn: async () => {},
+ });
+ assert.equal(media.ok, false);
+ assert.match((media as { error: string }).error, /does not exist \(drive not mounted\?\)/);
+
+ // The retired layout: the text kind is refused too, naming the way out.
+ await writeFile(
+ path.join(channelDir, "config.json"),
+ JSON.stringify({ handling: "youtube", url: "https://example.com/a", dataDir: "/mnt/platter/alpha/data" }),
+ );
+ const refused = await runManagedFunction({
+ kind: "normalize-transcripts",
+ queueKey: `test:text-guard:${newJobId()}`,
+ paths,
+ channelSlug: "alpha",
+ fn: async () => {
+ ran += 1;
+ },
+ });
+ assert.equal(refused.ok, false);
+ assert.match((refused as { error: string }).error, /archilyzer storage migrate-tier alpha/);
+ assert.equal(ran, 1);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
// THE ONE CANCEL THAT MUST NOT: the graceful-shutdown reaper cancels every
// queued job only so the exit cannot promote one into a child. Nobody cancelled
// it, and its `queued` sidecar is what the boot pass (bootQueuedJobs.ts)
diff --git a/umtool/report-to-video/cues.mjs b/umtool/report-to-video/cues.mjs
@@ -48,7 +48,7 @@
// option that is reproducible on a machine with no corpus.
// Whatever answers, the returned record carries `from` so a caller can record it.
-import { readFile, writeFile, mkdir, lstat, readlink, stat } from "node:fs/promises";
+import { readFile, writeFile, mkdir, lstat, readlink } from "node:fs/promises";
import path from "node:path";
import os from "node:os";
import { createHash } from "node:crypto";
@@ -239,31 +239,37 @@ export function createCueSource({
return { ...record, from: "http" };
}
- // A TWIN OF THE EDITOR'S GUARD. `assertChannelMediaReachable`
+ // A TWIN OF THE EDITOR'S TEXT GUARD. `assertChannelTextReadable`
// (`common/lib/channelMedia.ts`) is the same check in TypeScript, and that
// file carries a pointer back here — change one, change the other. It is
// copied rather than imported because umtool's bins run under plain node
// with no `tsx` and no build step, and that stays true for now.
//
- // WHY IT EXISTS. A channel's media can be relocated to another drive:
- // `channels/<slug>/data/` becomes an absolute SYMLINK to `<root>/<slug>/data`
- // and `config.json` records the target in `dataDir`. An unmounted drive then
- // reads as a plain ENOENT, which `load` used to swallow as "no local copy"
- // and answer from the published archive instead — silently cutting from a
- // snapshot whose cues can differ from the corpus by seconds (see the note at
- // the top of this file). Unreachable has to be loud.
+ // WHY IT EXISTS. This resolver reads TEXT — `transcript.cues.json`. Since
+ // release 17 a channel's text stays on the corpus disk whatever its media is
+ // doing (only the big files move, into `channels/<slug>/media`), so an
+ // unmounted MEDIA drive does not concern it. The one layout whose text is on
+ // another drive is the RETIRED whole-directory one — `data/` an absolute
+ // symlink to `<root>/<slug>/data`, `config.json` recording `dataDir` — and
+ // there an unmounted drive reads as a plain ENOENT, which `load` used to
+ // swallow as "no local copy" and answer from the published archive instead:
+ // silently cutting from a snapshot whose cues can differ from the corpus by
+ // seconds (see the note at the top of this file). So a `legacy` channel is
+ // refused, loudly, with its way out — mounted or not. And a marker is
+ // refused only when its `scope` is `tier-migration` (the migration rebuilds
+ // `data/` itself); a media move leaves the text where it is.
//
// SCOPE, DELIBERATELY NARROW. This fires only for a channel the local corpus
// actually holds. No `channels/` dir at all (a clone with no corpus), or a
- // channel this corpus does not mirror, leaves `data/` absent with no
- // configured target — which the editor calls "in-place" and passes, and which
- // here still falls through to HTTP. That is the supported archive-only case,
- // not a failure.
+ // channel this corpus does not mirror, leaves `data/` absent with nothing
+ // recorded — which the editor calls readable and passes, and which here still
+ // falls through to HTTP. That is the supported archive-only case, not a
+ // failure.
const checkedChannels = new Map();
function unreachable(channelSlug, dataDir, detail) {
return new CueLookupError(
- `channel "${channelSlug}": local media is not reachable — ${detail}`,
+ `channel "${channelSlug}": its local text is not readable — ${detail}`,
{ channelSlug, videoId: null, tried: [dataDir] },
);
}
@@ -275,84 +281,65 @@ export function createCueSource({
throw unreachable(channelSlug, dataDir, detail);
};
- let configured;
+ let retired;
try {
const parsed = JSON.parse(await readFile(path.join(channelDir, "config.json"), "utf8"));
if (typeof parsed.dataDir === "string" && parsed.dataDir.trim()) {
- configured = parsed.dataDir.trim();
+ retired = parsed.dataDir.trim();
}
} catch {
- // No config.json, or one that is not JSON: nothing records a relocation.
+ // No config.json, or one that is not JSON: nothing records a layout.
}
- // A relocation in flight (or interrupted) means `data/` is half of two
- // places at once. The editor refuses such a channel; so does this.
+ // A tier migration in flight (or interrupted) is rebuilding `data/`: half
+ // of two places at once. The editor refuses such a channel; so does this.
+ // A media move (`scope` "media", or none) leaves the text alone.
let marker = null;
try {
marker = JSON.parse(await readFile(path.join(channelDir, ".relocating.json"), "utf8"));
} catch {
/* no marker: the normal case */
}
- if (marker && typeof marker.target === "string" && marker.target.trim()) {
+ if (
+ marker &&
+ typeof marker.target === "string" &&
+ marker.target.trim() &&
+ marker.scope === "tier-migration"
+ ) {
fail(
- `a media relocation (${marker.direction ?? "out"}) is in progress or was ` +
- `interrupted at phase "${marker.phase ?? "copy"}" — target ${marker.target}`,
+ `its media layout is being migrated (phase "${marker.phase ?? "copy"}") — ` +
+ `wait for archilyzer storage migrate-tier to finish`,
);
}
- let link;
+ let link = null;
try {
link = await lstat(dataDir);
} catch {
- // No `data/` at all. With no configured target this is a channel that has
- // downloaded nothing — or is not mirrored here — and is not an error.
- if (!configured) return;
- fail(
- `config.json records dataDir ${configured} but ${dataDir} does not exist ` +
- `— the symlink is missing`,
- );
+ /* no data/ at all: below */
}
- if (link.isSymbolicLink()) {
- let target = "";
- try {
- target = await readlink(dataDir);
- } catch {
- /* unreadable link: reported as such below */
- }
- if (!configured) {
- fail(
- `${dataDir} is a symlink to ${target || "(unreadable)"} but config.json ` +
- `records no dataDir`,
- );
- }
- if (path.resolve(target) !== path.resolve(configured)) {
- fail(
- `${dataDir} points at ${target || "(unreadable)"} but config.json ` +
- `records ${configured}`,
- );
- }
- // The link points at a DEEP path (<root>/<slug>/data), so an unmounted
- // root gives ENOENT here and an empty mountpoint can never be mistaken
- // for the media.
- let st = null;
- try {
- st = await stat(configured);
- } catch {
- /* reported below */
+ // THE RETIRED LAYOUT: a `data` link, or a recorded `dataDir`. Refused
+ // whether or not its drive is mounted, never followed.
+ if ((link && link.isSymbolicLink()) || retired) {
+ let target = retired ?? "";
+ if (link && link.isSymbolicLink() && !target) {
+ try {
+ target = await readlink(dataDir);
+ } catch {
+ /* unreadable link: named as such */
+ }
}
- if (!st) fail(`${configured} does not exist (drive not mounted?)`);
- if (!st.isDirectory()) fail(`${configured} exists but is not a directory`);
- return;
- }
-
- if (!link.isDirectory()) fail(`${dataDir} is neither a directory nor a symlink`);
- if (configured) {
fail(
- `config.json records dataDir ${configured} but ${dataDir} is a real ` +
- `directory — the media was never moved, or was moved back by hand`,
+ `its media layout is the retired whole-directory one${target ? ` (${target})` : ""} — ` +
+ `run archilyzer storage migrate-tier ${channelSlug}`,
);
}
+
+ // No `data/`: a channel that has downloaded nothing — or is not mirrored
+ // here — and not an error.
+ if (!link) return;
+ if (!link.isDirectory()) fail(`${dataDir} is neither a directory nor a symlink`);
}
function assertChannelReachable(channelSlug) {
diff --git a/umtool/report-to-video/cues.test.mjs b/umtool/report-to-video/cues.test.mjs
@@ -148,28 +148,36 @@ test("a local corpus is preferred over the network", async () => {
}
});
-// --- a channel the corpus holds but cannot reach ----------------------------
+// --- a channel the corpus holds but whose text it cannot read ---------------
//
-// The bug these cover: `data/` may be a symlink to another drive, with the
-// target recorded in the channel's `config.json` as `dataDir`. An unmounted
-// drive reads as a plain ENOENT, which used to fall through to the archive —
-// so a relocated channel silently cut from a snapshot's cues, which can differ
-// from the corpus's by seconds.
+// The bug these cover: on the RETIRED layout `data/` is a symlink to another
+// drive, with the target recorded in the channel's `config.json` as `dataDir`.
+// An unmounted drive reads as a plain ENOENT, which used to fall through to the
+// archive — so a relocated channel silently cut from a snapshot's cues, which
+// can differ from the corpus's by seconds. Since release 17 such a channel is
+// `legacy` and refused mounted or not (its way out is `migrate-tier`); a
+// channel whose MEDIA alone is relocated (`media/` a link, `mediaDir`) keeps
+// its text on the corpus disk and is read as usual, drive or no drive.
const SCRATCH = process.env.CUES_TEST_DIR ?? tmpdir();
// A mirrored channel: <root>/channels/<slug>/ with a config.json, and `data/`
-// however the caller wants it.
-async function corpusWith(slug, { dataDir, data } = {}) {
+// (and `media`) however the caller wants it.
+async function corpusWith(slug, { dataDir, data, mediaDir } = {}) {
const root = await mkdtemp(path.join(SCRATCH, "cues-reach-"));
const channelDir = path.join(root, slug);
await mkdir(channelDir, { recursive: true });
await writeFile(
path.join(channelDir, "config.json"),
- JSON.stringify(dataDir ? { url: "https://x/", dataDir } : { url: "https://x/" }),
+ JSON.stringify({
+ url: "https://x/",
+ ...(dataDir ? { dataDir } : {}),
+ ...(mediaDir ? { mediaDir } : {}),
+ }),
);
if (data === "symlink") await symlink(dataDir, path.join(channelDir, "data"));
if (data === "dir") await mkdir(path.join(channelDir, "data"), { recursive: true });
+ if (mediaDir) await symlink(mediaDir, path.join(channelDir, "media"));
return root;
}
@@ -179,66 +187,71 @@ async function writeCues(dir, videoId, record) {
await writeFile(path.join(vdir, "transcript.cues.json"), JSON.stringify(record));
}
-test("a relocated channel whose drive is not mounted throws, and never fetches", async () => {
+function sourceOver(root, seen) {
+ return createCueSource({
+ channelsDir: root,
+ siteOrigin: ORIGIN,
+ cacheDir: null,
+ fetchImpl: stubFetch(ROUTES, seen),
+ });
+}
+
+test("a legacy channel whose drive is not mounted throws, names the way out, and never fetches", async () => {
const missing = path.join(SCRATCH, "cues-not-mounted-" + process.pid, "chan", "data");
const root = await corpusWith("chan", { dataDir: missing, data: "symlink" });
const seen = [];
try {
- const src = createCueSource({
- channelsDir: root,
- siteOrigin: ORIGIN,
- cacheDir: null,
- fetchImpl: stubFetch(ROUTES, seen),
- });
- await assert.rejects(() => src.load("chan", "vid1"), (err) => {
+ await assert.rejects(() => sourceOver(root, seen).load("chan", "vid1"), (err) => {
assert.equal(err.name, "CueLookupError");
assert.match(err.message, /chan/);
- assert.match(err.message, /drive not mounted/);
- assert.ok(err.message.includes(missing), "the error names the unreachable path");
+ assert.match(err.message, /retired whole-directory one/);
+ assert.match(err.message, /archilyzer storage migrate-tier chan/);
+ assert.ok(err.message.includes(missing), "the error names the retired target");
return true;
});
- assert.deepEqual(seen, [], "an unreachable channel must not fall through to the archive");
+ assert.deepEqual(seen, [], "a legacy channel must not fall through to the archive");
} finally {
await rm(root, { recursive: true, force: true });
}
});
-test("a dataDir with no symlink at all is refused too", async () => {
- const root = await corpusWith("chan", { dataDir: path.join(SCRATCH, "elsewhere") });
+test("a legacy channel is refused even with its drive mounted — never followed", async () => {
+ const elsewhere = await mkdtemp(path.join(SCRATCH, "cues-drive-"));
+ const target = path.join(elsewhere, "chan", "data");
+ await mkdir(target, { recursive: true });
+ await writeCues(target, "vid1", { ...RECORD, title: "RELOCATED COPY" });
+ const root = await corpusWith("chan", { dataDir: target, data: "symlink" });
const seen = [];
try {
- const src = createCueSource({
- channelsDir: root,
- siteOrigin: ORIGIN,
- cacheDir: null,
- fetchImpl: stubFetch(ROUTES, seen),
- });
- await assert.rejects(() => src.load("chan", "vid1"), (err) => {
- assert.match(err.message, /symlink is missing/);
- return true;
- });
+ await assert.rejects(() => sourceOver(root, seen).load("chan", "vid1"), /migrate-tier chan/);
+ assert.deepEqual(seen, []);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ await rm(elsewhere, { recursive: true, force: true });
+ }
+});
+
+test("a recorded dataDir with a real data/ is legacy too", async () => {
+ const root = await corpusWith("chan", { dataDir: path.join(SCRATCH, "elsewhere"), data: "dir" });
+ const seen = [];
+ try {
+ await assert.rejects(() => sourceOver(root, seen).load("chan", "vid1"), /retired whole-directory/);
assert.deepEqual(seen, []);
} finally {
await rm(root, { recursive: true, force: true });
}
});
-test("a relocation in flight is refused rather than half-read", async () => {
+test("a tier migration in flight is refused rather than half-read", async () => {
const root = await corpusWith("chan", { data: "dir" });
await writeFile(
path.join(root, "chan", ".relocating.json"),
- JSON.stringify({ target: "/mnt/big/chan/data", direction: "out", phase: "copy" }),
+ JSON.stringify({ target: "/mnt/big/chan/media", direction: "out", phase: "copy", scope: "tier-migration" }),
);
const seen = [];
try {
- const src = createCueSource({
- channelsDir: root,
- siteOrigin: ORIGIN,
- cacheDir: null,
- fetchImpl: stubFetch(ROUTES, seen),
- });
- await assert.rejects(() => src.load("chan", "vid1"), (err) => {
- assert.match(err.message, /relocation \(out\) is in progress/);
+ await assert.rejects(() => sourceOver(root, seen).load("chan", "vid1"), (err) => {
+ assert.match(err.message, /being migrated \(phase "copy"\)/);
return true;
});
assert.deepEqual(seen, []);
@@ -247,27 +260,36 @@ test("a relocation in flight is refused rather than half-read", async () => {
}
});
-test("a reachable dataDir is read from, wherever it points", async () => {
- const elsewhere = await mkdtemp(path.join(SCRATCH, "cues-drive-"));
- const target = path.join(elsewhere, "chan", "data");
- await mkdir(target, { recursive: true });
- await writeCues(target, "vid1", { ...RECORD, title: "RELOCATED COPY" });
- const root = await corpusWith("chan", { dataDir: target, data: "symlink" });
+test("a media move in flight is not refused: the text stays where it is", async () => {
+ const root = await corpusWith("chan", { data: "dir" });
+ await writeCues(path.join(root, "chan", "data"), "vid1", { ...RECORD, title: "LOCAL DURING MOVE" });
+ await writeFile(
+ path.join(root, "chan", ".relocating.json"),
+ JSON.stringify({ target: "/mnt/big/chan/media", direction: "out", phase: "copy", scope: "media" }),
+ );
const seen = [];
try {
- const src = createCueSource({
- channelsDir: root,
- siteOrigin: ORIGIN,
- cacheDir: null,
- fetchImpl: stubFetch(ROUTES, seen),
- });
- const got = await src.load("chan", "vid1");
+ const got = await sourceOver(root, seen).load("chan", "vid1");
assert.equal(got.from, "local");
- assert.equal(got.title, "RELOCATED COPY");
- assert.deepEqual(seen, [], "a reachable relocation is still a local read");
+ assert.equal(got.title, "LOCAL DURING MOVE");
+ assert.deepEqual(seen, []);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("relocated MEDIA on an unmounted drive: the text is read locally as usual", async () => {
+ const missing = path.join(SCRATCH, "cues-media-not-mounted-" + process.pid, "chan", "media");
+ const root = await corpusWith("chan", { data: "dir", mediaDir: missing });
+ await writeCues(path.join(root, "chan", "data"), "vid1", { ...RECORD, title: "TEXT ON THE SSD" });
+ const seen = [];
+ try {
+ const got = await sourceOver(root, seen).load("chan", "vid1");
+ assert.equal(got.from, "local");
+ assert.equal(got.title, "TEXT ON THE SSD");
+ assert.deepEqual(seen, [], "a local read, whatever the media drive is doing");
} finally {
await rm(root, { recursive: true, force: true });
- await rm(elsewhere, { recursive: true, force: true });
}
});
