commit 798301c505b49a6e673161b2dc2674d15c3a503a
parent ea2d3d04d126af72cf52b77dfe0d48a78e791d88
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sat, 12 Sep 2026 02:34:54 -0400
umtool: an unreachable local channel stops the cue resolver instead of the archive answering
`fromLocal` read `channels/<slug>/data/<id>/transcript.cues.json` and let
`load` swallow ENOENT as "no local copy", falling through to the published
archive. A channel whose media was relocated to another drive — `data/` an
absolute symlink, the target in `config.json`'s `dataDir` — reads exactly like
that when the drive is not mounted, so an operator with a corpus silently got a
snapshot's cues: measured on this corpus, one video of four had 65 of its 84 cue
texts rewritten and timings shifted by up to 2.24s between publish and local.
Clips then get cut in the wrong place, quietly.
`checkChannelReachable` replicates `assertChannelMediaReachable`'s statuses in
plain `.mjs` — no import from common, because umtool's bins run under bare node
— and throws a CueLookupError naming the slug and the path. A CueLookupError
carries no `code`, so `load`'s ENOENT fallback rethrows it rather than reaching
for HTTP, and the test asserts zero fetches.
Its scope is deliberately narrow, and matches the editor's: a missing
`channels/` dir, or a channel this corpus does not mirror, leaves `data/` absent
with no configured target — "in-place" to the editor, and still an HTTP fallback
here. A repo with no corpus is a first-class way to use this project and stays
one. `--cue-source local` refuses to fall back exactly as before.
`channelMedia.ts` gains a comment pointing at the copy, so the next change to
the check finds its twin.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
3 files changed, 322 insertions(+), 3 deletions(-)
diff --git a/common/lib/channelMedia.ts b/common/lib/channelMedia.ts
@@ -316,6 +316,14 @@ export async function inspectChannelMedia(
// "ok" and "in-place" pass; everything else throws. An in-transition or
// inconsistent channel is refused for the same reason an unreachable one is:
// the caller would otherwise read a half-populated or empty dir as the truth.
+//
+// THIS CHECK HAS A TWIN. `checkChannelReachable` in
+// `umtool/report-to-video/cues.mjs` repeats the same statuses in plain `.mjs`,
+// because umtool's bins run under bare node with no `tsx` and cannot import
+// this module. It guards the same failure: a relocated channel whose drive is
+// not mounted reads as ENOENT, and the cue resolver would otherwise answer from
+// the published archive — cutting clips from a snapshot's cues instead of the
+// corpus's. Change the checks here and change them there.
export async function assertChannelMediaReachable(
paths: ChannelMediaPaths,
slug: string,
diff --git a/umtool/report-to-video/cues.mjs b/umtool/report-to-video/cues.mjs
@@ -25,7 +25,10 @@
// 4. take the record whose `id` matches
//
// Local wins when present: it is faster, works offline, and is the operator's own
-// data. HTTP is the fallback, not a preference.
+// data. HTTP is the fallback, not a preference — and only for a channel this
+// corpus does not hold. A channel it DOES hold whose media cannot be reached (a
+// relocated `data/` whose drive is not mounted) is an error, never a quiet
+// downgrade to the archive; see checkChannelReachable below.
//
// THE TWO SOURCES CAN DISAGREE, AND IT IS NOT ROUNDING. A published archive is a
// snapshot; a live corpus keeps moving. Re-synced platform captions, an
@@ -45,7 +48,7 @@
// option that is reproducible on a machine with no corpus.
// Whatever answers, the returned record carries `from` so a caller can record it.
-import { readFile, writeFile, mkdir } from "node:fs/promises";
+import { readFile, writeFile, mkdir, lstat, readlink, stat } from "node:fs/promises";
import path from "node:path";
import os from "node:os";
import { createHash } from "node:crypto";
@@ -236,7 +239,137 @@ export function createCueSource({
return { ...record, from: "http" };
}
+ // A TWIN OF THE EDITOR'S GUARD. `assertChannelMediaReachable`
+ // (`common/lib/channelMedia.ts`) is the same check in TypeScript, and that
+ // file carries a pointer back here — change one, change the other. It is
+ // copied rather than imported because umtool's bins run under plain node
+ // with no `tsx` and no build step, and that stays true for now.
+ //
+ // WHY IT EXISTS. A channel's media can be relocated to another drive:
+ // `channels/<slug>/data/` becomes an absolute SYMLINK to `<root>/<slug>/data`
+ // and `config.json` records the target in `dataDir`. An unmounted drive then
+ // reads as a plain ENOENT, which `load` used to swallow as "no local copy"
+ // and answer from the published archive instead — silently cutting from a
+ // snapshot whose cues can differ from the corpus by seconds (see the note at
+ // the top of this file). Unreachable has to be loud.
+ //
+ // SCOPE, DELIBERATELY NARROW. This fires only for a channel the local corpus
+ // actually holds. No `channels/` dir at all (a clone with no corpus), or a
+ // channel this corpus does not mirror, leaves `data/` absent with no
+ // configured target — which the editor calls "in-place" and passes, and which
+ // here still falls through to HTTP. That is the supported archive-only case,
+ // not a failure.
+ const checkedChannels = new Map();
+
+ function unreachable(channelSlug, dataDir, detail) {
+ return new CueLookupError(
+ `channel "${channelSlug}": local media is not reachable — ${detail}`,
+ { channelSlug, videoId: null, tried: [dataDir] },
+ );
+ }
+
+ async function checkChannelReachable(channelSlug) {
+ const channelDir = path.join(channelsDir, channelSlug);
+ const dataDir = path.join(channelDir, "data");
+ const fail = (detail) => {
+ throw unreachable(channelSlug, dataDir, detail);
+ };
+
+ let configured;
+ try {
+ const parsed = JSON.parse(await readFile(path.join(channelDir, "config.json"), "utf8"));
+ if (typeof parsed.dataDir === "string" && parsed.dataDir.trim()) {
+ configured = parsed.dataDir.trim();
+ }
+ } catch {
+ // No config.json, or one that is not JSON: nothing records a relocation.
+ }
+
+ // A relocation in flight (or interrupted) means `data/` is half of two
+ // places at once. The editor refuses such a channel; so does this.
+ let marker = null;
+ try {
+ marker = JSON.parse(await readFile(path.join(channelDir, ".relocating.json"), "utf8"));
+ } catch {
+ /* no marker: the normal case */
+ }
+ if (marker && typeof marker.target === "string" && marker.target.trim()) {
+ fail(
+ `a media relocation (${marker.direction ?? "out"}) is in progress or was ` +
+ `interrupted at phase "${marker.phase ?? "copy"}" — target ${marker.target}`,
+ );
+ }
+
+ let link;
+ try {
+ link = await lstat(dataDir);
+ } catch {
+ // No `data/` at all. With no configured target this is a channel that has
+ // downloaded nothing — or is not mirrored here — and is not an error.
+ if (!configured) return;
+ fail(
+ `config.json records dataDir ${configured} but ${dataDir} does not exist ` +
+ `— the symlink is missing`,
+ );
+ }
+
+ if (link.isSymbolicLink()) {
+ let target = "";
+ try {
+ target = await readlink(dataDir);
+ } catch {
+ /* unreadable link: reported as such below */
+ }
+ if (!configured) {
+ fail(
+ `${dataDir} is a symlink to ${target || "(unreadable)"} but config.json ` +
+ `records no dataDir`,
+ );
+ }
+ if (path.resolve(target) !== path.resolve(configured)) {
+ fail(
+ `${dataDir} points at ${target || "(unreadable)"} but config.json ` +
+ `records ${configured}`,
+ );
+ }
+ // The link points at a DEEP path (<root>/<slug>/data), so an unmounted
+ // root gives ENOENT here and an empty mountpoint can never be mistaken
+ // for the media.
+ let st = null;
+ try {
+ st = await stat(configured);
+ } catch {
+ /* reported below */
+ }
+ if (!st) fail(`${configured} does not exist (drive not mounted?)`);
+ if (!st.isDirectory()) fail(`${configured} exists but is not a directory`);
+ return;
+ }
+
+ if (!link.isDirectory()) fail(`${dataDir} is neither a directory nor a symlink`);
+ if (configured) {
+ fail(
+ `config.json records dataDir ${configured} but ${dataDir} is a real ` +
+ `directory — the media was never moved, or was moved back by hand`,
+ );
+ }
+ }
+
+ function assertChannelReachable(channelSlug) {
+ // One check per channel per run, cached as a promise so a rejection replays
+ // for every clip of that channel instead of re-statting for each.
+ let pending = checkedChannels.get(channelSlug);
+ if (!pending) {
+ pending = checkChannelReachable(channelSlug);
+ checkedChannels.set(channelSlug, pending);
+ }
+ return pending;
+ }
+
async function fromLocal(channelSlug, videoId) {
+ // Throws a CueLookupError — which carries no `code`, so `load`'s ENOENT
+ // fallback rethrows it rather than reaching for the archive.
+ await assertChannelReachable(channelSlug);
const p = path.join(channelsDir, channelSlug, "data", videoId, "transcript.cues.json");
const parsed = JSON.parse(await readFile(p, "utf8"));
return { ...parsed, from: "local" };
diff --git a/umtool/report-to-video/cues.test.mjs b/umtool/report-to-video/cues.test.mjs
@@ -7,7 +7,7 @@
// Run with: pnpm test:scripts
import assert from "node:assert/strict";
import test from "node:test";
-import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
+import { mkdtemp, mkdir, symlink, writeFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
@@ -148,6 +148,184 @@ test("a local corpus is preferred over the network", async () => {
}
});
+// --- a channel the corpus holds but cannot reach ----------------------------
+//
+// The bug these cover: `data/` may be a symlink to another drive, with the
+// target recorded in the channel's `config.json` as `dataDir`. An unmounted
+// drive reads as a plain ENOENT, which used to fall through to the archive —
+// so a relocated channel silently cut from a snapshot's cues, which can differ
+// from the corpus's by seconds.
+
+const SCRATCH = process.env.CUES_TEST_DIR ?? tmpdir();
+
+// A mirrored channel: <root>/channels/<slug>/ with a config.json, and `data/`
+// however the caller wants it.
+async function corpusWith(slug, { dataDir, data } = {}) {
+ const root = await mkdtemp(path.join(SCRATCH, "cues-reach-"));
+ const channelDir = path.join(root, slug);
+ await mkdir(channelDir, { recursive: true });
+ await writeFile(
+ path.join(channelDir, "config.json"),
+ JSON.stringify(dataDir ? { url: "https://x/", dataDir } : { url: "https://x/" }),
+ );
+ if (data === "symlink") await symlink(dataDir, path.join(channelDir, "data"));
+ if (data === "dir") await mkdir(path.join(channelDir, "data"), { recursive: true });
+ return root;
+}
+
+async function writeCues(dir, videoId, record) {
+ const vdir = path.join(dir, videoId);
+ await mkdir(vdir, { recursive: true });
+ await writeFile(path.join(vdir, "transcript.cues.json"), JSON.stringify(record));
+}
+
+test("a relocated channel whose drive is not mounted throws, and never fetches", async () => {
+ const missing = path.join(SCRATCH, "cues-not-mounted-" + process.pid, "chan", "data");
+ const root = await corpusWith("chan", { dataDir: missing, data: "symlink" });
+ const seen = [];
+ try {
+ const src = createCueSource({
+ channelsDir: root,
+ siteOrigin: ORIGIN,
+ cacheDir: null,
+ fetchImpl: stubFetch(ROUTES, seen),
+ });
+ await assert.rejects(() => src.load("chan", "vid1"), (err) => {
+ assert.equal(err.name, "CueLookupError");
+ assert.match(err.message, /chan/);
+ assert.match(err.message, /drive not mounted/);
+ assert.ok(err.message.includes(missing), "the error names the unreachable path");
+ return true;
+ });
+ assert.deepEqual(seen, [], "an unreachable channel must not fall through to the archive");
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("a dataDir with no symlink at all is refused too", async () => {
+ const root = await corpusWith("chan", { dataDir: path.join(SCRATCH, "elsewhere") });
+ const seen = [];
+ try {
+ const src = createCueSource({
+ channelsDir: root,
+ siteOrigin: ORIGIN,
+ cacheDir: null,
+ fetchImpl: stubFetch(ROUTES, seen),
+ });
+ await assert.rejects(() => src.load("chan", "vid1"), (err) => {
+ assert.match(err.message, /symlink is missing/);
+ return true;
+ });
+ assert.deepEqual(seen, []);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("a relocation in flight is refused rather than half-read", async () => {
+ const root = await corpusWith("chan", { data: "dir" });
+ await writeFile(
+ path.join(root, "chan", ".relocating.json"),
+ JSON.stringify({ target: "/mnt/big/chan/data", direction: "out", phase: "copy" }),
+ );
+ const seen = [];
+ try {
+ const src = createCueSource({
+ channelsDir: root,
+ siteOrigin: ORIGIN,
+ cacheDir: null,
+ fetchImpl: stubFetch(ROUTES, seen),
+ });
+ await assert.rejects(() => src.load("chan", "vid1"), (err) => {
+ assert.match(err.message, /relocation \(out\) is in progress/);
+ return true;
+ });
+ assert.deepEqual(seen, []);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("a reachable dataDir is read from, wherever it points", async () => {
+ const elsewhere = await mkdtemp(path.join(SCRATCH, "cues-drive-"));
+ const target = path.join(elsewhere, "chan", "data");
+ await mkdir(target, { recursive: true });
+ await writeCues(target, "vid1", { ...RECORD, title: "RELOCATED COPY" });
+ const root = await corpusWith("chan", { dataDir: target, data: "symlink" });
+ const seen = [];
+ try {
+ const src = createCueSource({
+ channelsDir: root,
+ siteOrigin: ORIGIN,
+ cacheDir: null,
+ fetchImpl: stubFetch(ROUTES, seen),
+ });
+ const got = await src.load("chan", "vid1");
+ assert.equal(got.from, "local");
+ assert.equal(got.title, "RELOCATED COPY");
+ assert.deepEqual(seen, [], "a reachable relocation is still a local read");
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ await rm(elsewhere, { recursive: true, force: true });
+ }
+});
+
+test("a reachable channel that simply lacks this video still falls back to HTTP", async () => {
+ // The narrow scope of the guard: present-but-empty is not unreachable.
+ const root = await corpusWith("chan", { data: "dir" });
+ try {
+ const src = createCueSource({
+ channelsDir: root,
+ siteOrigin: ORIGIN,
+ cacheDir: null,
+ fetchImpl: stubFetch(ROUTES),
+ });
+ const got = await src.load("chan", "vid1");
+ assert.equal(got.from, "http");
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("no local corpus at all is the archive-only case, not an unreachable channel", async () => {
+ // A clone with no `transcripts/channels` is a first-class way to use this
+ // repo: the cues come over HTTP and nothing about that is an error.
+ const seen = [];
+ const src = createCueSource({
+ channelsDir: path.join(SCRATCH, "definitely-no-corpus-here"),
+ siteOrigin: ORIGIN,
+ cacheDir: null,
+ fetchImpl: stubFetch(ROUTES, seen),
+ });
+ const got = await src.load("chan", "vid1");
+ assert.equal(got.from, "http");
+ assert.deepEqual(seen, [
+ `${ORIGIN}/corpus.json`,
+ `${ORIGIN}/transcripts/chan/manifest.json`,
+ `${ORIGIN}/transcripts/chan/page-0000.json`,
+ ]);
+});
+
+test("--cue-source local still refuses to fall back, unreachable or not", async () => {
+ const root = await corpusWith("chan", { data: "dir" });
+ try {
+ const src = createCueSource({
+ channelsDir: root,
+ siteOrigin: ORIGIN,
+ cacheDir: null,
+ fetchImpl: stubFetch(ROUTES),
+ prefer: "local",
+ });
+ await assert.rejects(() => src.load("chan", "vid1"), (err) => {
+ assert.match(err.message, /forbids falling back/);
+ return true;
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
// --- the Rumble two-id trap -------------------------------------------------
test("a published-id miss fails loudly, naming the two-id trap", async () => {
