commit 1fce53985d71a0e79aa59ed98afc5320a70e25c1
parent ea2d3d04d126af72cf52b77dfe0d48a78e791d88
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sat, 12 Sep 2026 02:52:01 -0400
merge: one-core/phase-2-s2b — cues.mjs refuses an unreachable channel instead of cutting from HTTP
S2b of Phase 2, reviewed clean: umtool's cue resolver replicates the editor's
reachability check in plain .mjs, verdict for verdict, and throws with zero
fetches when a relocated channel's drive is not mounted. A channel with no
local data and no dataDir stays in-place, as the editor reads it. Gates on
the slice tip: tsc clean, test:scripts 78/1, common 1077, umtool e2e moved
by nothing against the base.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
4 files changed, 447 insertions(+), 3 deletions(-)
diff --git a/common/lib/channelMedia.ts b/common/lib/channelMedia.ts
@@ -316,6 +316,14 @@ export async function inspectChannelMedia(
// "ok" and "in-place" pass; everything else throws. An in-transition or
// inconsistent channel is refused for the same reason an unreachable one is:
// the caller would otherwise read a half-populated or empty dir as the truth.
+//
+// THIS CHECK HAS A TWIN. `checkChannelReachable` in
+// `umtool/report-to-video/cues.mjs` repeats the same statuses in plain `.mjs`,
+// because umtool's bins run under bare node with no `tsx` and cannot import
+// this module. It guards the same failure: a relocated channel whose drive is
+// not mounted reads as ENOENT, and the cue resolver would otherwise answer from
+// the published archive — cutting clips from a snapshot's cues instead of the
+// corpus's. Change the checks here and change them there.
export async function assertChannelMediaReachable(
paths: ChannelMediaPaths,
slug: string,
diff --git a/plans/one-core-phase-2.md b/plans/one-core-phase-2.md
@@ -382,3 +382,128 @@ client chunk. The prerender is the only step that fails, and it fails at base.
`plans/tools/jeralyzer-corpus-2026-09-12.json` was not re-fetched; the live diff
is S3's gate, after the search pipeline lands.
+
+### S2b — shipped 2026-09-12
+
+Branch `one-core/phase-2-s2b`, off `7f86aef` (the `integrate/2026-09-storage-priority`
+tip with S1 merged). Two commits, `745677f` → `fa3873a` (this note), unmerged. Nothing
+outside `umtool/report-to-video/` moved except one comment in `common/lib/channelMedia.ts`
+and this note. **No URL shape, no `corpus.json` byte, no CONTRACT version, no
+architecture allow-list entry.**
+
+| commit | what |
+|---|---|
+| `745677f` | `checkChannelReachable` in `cues.mjs` + 7 test cases; the cross-reference comment |
+| `fa3873a` | this note |
+
+#### What the guard actually refuses, and what it deliberately does not
+
+`fromLocal` now calls `checkChannelReachable(slug)` before reading the cue file. It
+replicates `inspectChannelMedia` / `assertChannelMediaReachable`'s statuses in plain
+`.mjs` — `dataDir` read straight out of `config.json`, `.relocating.json` marker,
+`lstat` on `data/`, `readlink` compared against the configured target, `stat` on the
+deep `<root>/<slug>/data` path — and throws a `CueLookupError` naming the slug and the
+path. A `CueLookupError` carries no `code`, so `load`'s `ENOENT`/`ENOTDIR` fallback
+rethrows it instead of reaching for the archive; the test asserts **zero** fetches.
+Checked once per channel per run, memoised as a promise so a rejection replays for
+every clip of that channel.
+
+Throws: a symlinked `data/` whose target is missing (the unmounted drive — the bug),
+or is not a directory, or disagrees with `config.json`; a `dataDir` in `config.json`
+with no `data/` at all; a real `data/` directory with a `dataDir` recorded anyway; a
+`.relocating.json` marker.
+
+Does **not** throw, and this is the deliberate line: a channel whose `data/` is simply
+absent with no configured target. That covers both "no `channels/` dir at all" (a clone
+with no corpus — a first-class way to use this repo) and "a corpus that does not mirror
+this channel", and it is exactly what the editor's `inspectChannelMedia` calls
+`in-place` and `assertChannelMediaReachable` passes. Deciding otherwise would have hard-
+failed every clip of a non-mirrored channel for any operator who has a corpus at all,
+and it would have made the twin a near-copy rather than a copy. A reachable channel that
+merely lacks THIS video still falls through to HTTP, as before — two tests pin both.
+
+`--cue-source local` is untouched: it still refuses to fall back, with its own message,
+and a test pins that the guard does not swallow it. `pageFileName` / `pageUrlFrom`
+untouched.
+
+`common/lib/channelMedia.ts` gains a comment above `assertChannelMediaReachable`
+pointing at the copy and saying why it is a copy (umtool's bins run under bare node).
+The function is not changed.
+
+#### The `tsx` change list for Phase 5
+
+Every entry re-grepped at `745677f`; three paths in the S2b brief were wrong and are
+corrected here.
+
+| what | where | note |
+|---|---|---|
+| six shebangs | `umtool/report-to-video/{build-video,check-availability,compose-chrome,render-cards,resolve-windows,verify-build}.mjs:1` | all `#!/usr/bin/env node`. 40 MORE node shebangs live under `umtool/` (`bin/umtool.mjs`, `e2e/fixtures/make-fixture.mjs`, 38 under `song/`) — the "six" is report-to-video only |
+| six spawn argv | `umtool/lib/report/driver.mjs:70,93,99,105,140,167` | **not** `umtool/report-to-video/driver.mjs` — there is no such file |
+| one printed hint | `umtool/bin/umtool.mjs:516` | **not** `umtool/umtool.mjs`; prints `node umtool/report-to-video/resolve-windows.mjs …` |
+| one `execFileSync` | `umtool/e2e/clip-bench.spec.ts:145` (script path on `:146`) | |
+| the test runner | root `package.json:24` — `test:scripts` = `node --test scripts/*.test.mjs umtool/report-to-video/*.test.mjs` | |
+| the missing deps | `umtool/report-to-video/package.json` | it has NO `dependencies` and no `devDependencies` at all, so `tsx` has nothing to be declared in. `umtool/package.json` does have `dependencies` — the brief named the wrong file |
+
+Not made here, per the brief.
+
+#### One thing this slice did not close
+
+`fromLocal` is not the only place that joins `channels/<slug>/data/<id>/transcript.cues.json`
+by hand and treats a failure as "no cues": `umtool/report-to-video/check-availability.mjs:54`
+(`cueMeta`) and `umtool/lib/projects/report.mjs:124` (`cuePathFor`) do the same, and an
+unmounted relocated channel still reads to them as an absent file. Neither answers a cue
+WINDOW — they feed availability rows and the project index — so neither can cut a clip in
+the wrong place, which is why they are out of this slice. They want the same guard when
+umtool's local-corpus access is next touched.
+
+#### Gates
+
+- `pnpm -r exec tsc --noEmit` — **clean** in all eight packages.
+- `pnpm test:scripts` — **78 passed / 1 skipped** (baseline 71/1; +7, the new cases).
+ This is where the cues tests are collected; `umtool/package.json` has **no `test`
+ script** of its own (only `dev`/`build`/`start`/`typecheck`/`e2e`), which is the one
+ divergence from the brief's "umtool's own test command".
+- `pnpm --filter yt-dlp-transcript-common test` — **1077 passed / 0 failed**, unchanged.
+- `node --test umtool/report-to-video/cues.test.mjs` alone — **21 passed / 1 skipped**
+ (the LIVE network case is the skip, as before).
+- umtool e2e, behind the queue lock from worktree #11 (`UMTOOL_E2E_PORT=4151`,
+ via `pnpm wt run`), with `SONG_DIR=/home/user/reports/quartering-uh-song/data`:
+ **129 passed / 40 failed / 2 skipped** — and the SAME numbers with the SAME 40
+ test names at the base commit `7f86aef`, diffed line for line (`git switch
+ --detach 7f86aef` in this worktree, same command, same env). This slice moves
+ nothing in that suite. The 40 are environmental and were already recorded:
+ the heavy song fixtures are not on this machine
+ (`$SONG_DIR/{wav48,asr,media}` do not exist, only `cand2/` does, and
+ `make-fixture.mjs` symlinks those three only `if (existsSync)`), so every spec
+ that needs audio, ASR or the face detector fails — `find` 11, `triage` 9,
+ `faces` 6, `usage` 5, `browse` 3, `undo` 3, `deck` 2, `projects` 1, with
+ `no media for v1`, `detect v1@60.00 -> 404` and a missing `every note (N)`
+ table. What DID run, and passes on both: the whole of `build.spec.ts`,
+ `clip-bench.spec.ts` (which shells `resolve-windows.mjs` with `CHANNELS_DIR`
+ pointed at the fixture corpus — the one spec that exercises the changed path
+ end to end) and `report-longform.spec.ts`.
+
+
+Not run, and deliberately: the export `next build`, the mcp bench, the compose-site
+fixture diff and the export/hub/2origin e2e suites that §Verification asks of every
+slice. Nothing in S2b is reachable from any of them — the diff is one `.mjs` file
+under `umtool/`, its test file, and a comment. `pnpm -r exec tsc --noEmit` covers the
+one TypeScript file touched.
+
+#### Divergences from the S2b brief
+
+1. **`umtool` has no `test` script.** The brief said to run "umtool's own test
+ command"; `umtool/package.json` has `dev`, `build`, `start`, `typecheck` and `e2e`
+ only. The cues tests are collected by the ROOT `test:scripts`, which is the number
+ quoted above, and were also run directly with `node --test`.
+2. **Three paths in the Phase 5 `tsx` list were wrong** — `driver.mjs`,
+ `umtool.mjs` and the package missing `dependencies`. Corrected in the table above,
+ each re-grepped.
+3. **A missing channel dir does not throw.** The brief's "missing, or `dataDir` set but
+ not mounted" reads as if an absent channel dir should also fail. It does not, for the
+ reason given above: the editor's twin calls that `in-place`, and throwing would break
+ the archive-only workflow this repo advertises. The case that motivated the slice —
+ a relocated channel whose drive is gone — throws.
+4. **Test fixtures use `tmpdir()`, not the agent scratch dir.** A test that hard-coded a
+ job scratch path would not survive the job. `CUES_TEST_DIR` overrides the root, and
+ that is what the scratch-dir runs used.
diff --git a/umtool/report-to-video/cues.mjs b/umtool/report-to-video/cues.mjs
@@ -25,7 +25,10 @@
// 4. take the record whose `id` matches
//
// Local wins when present: it is faster, works offline, and is the operator's own
-// data. HTTP is the fallback, not a preference.
+// data. HTTP is the fallback, not a preference — and only for a channel this
+// corpus does not hold. A channel it DOES hold whose media cannot be reached (a
+// relocated `data/` whose drive is not mounted) is an error, never a quiet
+// downgrade to the archive; see checkChannelReachable below.
//
// THE TWO SOURCES CAN DISAGREE, AND IT IS NOT ROUNDING. A published archive is a
// snapshot; a live corpus keeps moving. Re-synced platform captions, an
@@ -45,7 +48,7 @@
// option that is reproducible on a machine with no corpus.
// Whatever answers, the returned record carries `from` so a caller can record it.
-import { readFile, writeFile, mkdir } from "node:fs/promises";
+import { readFile, writeFile, mkdir, lstat, readlink, stat } from "node:fs/promises";
import path from "node:path";
import os from "node:os";
import { createHash } from "node:crypto";
@@ -236,7 +239,137 @@ export function createCueSource({
return { ...record, from: "http" };
}
+ // A TWIN OF THE EDITOR'S GUARD. `assertChannelMediaReachable`
+ // (`common/lib/channelMedia.ts`) is the same check in TypeScript, and that
+ // file carries a pointer back here — change one, change the other. It is
+ // copied rather than imported because umtool's bins run under plain node
+ // with no `tsx` and no build step, and that stays true for now.
+ //
+ // WHY IT EXISTS. A channel's media can be relocated to another drive:
+ // `channels/<slug>/data/` becomes an absolute SYMLINK to `<root>/<slug>/data`
+ // and `config.json` records the target in `dataDir`. An unmounted drive then
+ // reads as a plain ENOENT, which `load` used to swallow as "no local copy"
+ // and answer from the published archive instead — silently cutting from a
+ // snapshot whose cues can differ from the corpus by seconds (see the note at
+ // the top of this file). Unreachable has to be loud.
+ //
+ // SCOPE, DELIBERATELY NARROW. This fires only for a channel the local corpus
+ // actually holds. No `channels/` dir at all (a clone with no corpus), or a
+ // channel this corpus does not mirror, leaves `data/` absent with no
+ // configured target — which the editor calls "in-place" and passes, and which
+ // here still falls through to HTTP. That is the supported archive-only case,
+ // not a failure.
+ const checkedChannels = new Map();
+
+ function unreachable(channelSlug, dataDir, detail) {
+ return new CueLookupError(
+ `channel "${channelSlug}": local media is not reachable — ${detail}`,
+ { channelSlug, videoId: null, tried: [dataDir] },
+ );
+ }
+
+ async function checkChannelReachable(channelSlug) {
+ const channelDir = path.join(channelsDir, channelSlug);
+ const dataDir = path.join(channelDir, "data");
+ const fail = (detail) => {
+ throw unreachable(channelSlug, dataDir, detail);
+ };
+
+ let configured;
+ try {
+ const parsed = JSON.parse(await readFile(path.join(channelDir, "config.json"), "utf8"));
+ if (typeof parsed.dataDir === "string" && parsed.dataDir.trim()) {
+ configured = parsed.dataDir.trim();
+ }
+ } catch {
+ // No config.json, or one that is not JSON: nothing records a relocation.
+ }
+
+ // A relocation in flight (or interrupted) means `data/` is half of two
+ // places at once. The editor refuses such a channel; so does this.
+ let marker = null;
+ try {
+ marker = JSON.parse(await readFile(path.join(channelDir, ".relocating.json"), "utf8"));
+ } catch {
+ /* no marker: the normal case */
+ }
+ if (marker && typeof marker.target === "string" && marker.target.trim()) {
+ fail(
+ `a media relocation (${marker.direction ?? "out"}) is in progress or was ` +
+ `interrupted at phase "${marker.phase ?? "copy"}" — target ${marker.target}`,
+ );
+ }
+
+ let link;
+ try {
+ link = await lstat(dataDir);
+ } catch {
+ // No `data/` at all. With no configured target this is a channel that has
+ // downloaded nothing — or is not mirrored here — and is not an error.
+ if (!configured) return;
+ fail(
+ `config.json records dataDir ${configured} but ${dataDir} does not exist ` +
+ `— the symlink is missing`,
+ );
+ }
+
+ if (link.isSymbolicLink()) {
+ let target = "";
+ try {
+ target = await readlink(dataDir);
+ } catch {
+ /* unreadable link: reported as such below */
+ }
+ if (!configured) {
+ fail(
+ `${dataDir} is a symlink to ${target || "(unreadable)"} but config.json ` +
+ `records no dataDir`,
+ );
+ }
+ if (path.resolve(target) !== path.resolve(configured)) {
+ fail(
+ `${dataDir} points at ${target || "(unreadable)"} but config.json ` +
+ `records ${configured}`,
+ );
+ }
+ // The link points at a DEEP path (<root>/<slug>/data), so an unmounted
+ // root gives ENOENT here and an empty mountpoint can never be mistaken
+ // for the media.
+ let st = null;
+ try {
+ st = await stat(configured);
+ } catch {
+ /* reported below */
+ }
+ if (!st) fail(`${configured} does not exist (drive not mounted?)`);
+ if (!st.isDirectory()) fail(`${configured} exists but is not a directory`);
+ return;
+ }
+
+ if (!link.isDirectory()) fail(`${dataDir} is neither a directory nor a symlink`);
+ if (configured) {
+ fail(
+ `config.json records dataDir ${configured} but ${dataDir} is a real ` +
+ `directory — the media was never moved, or was moved back by hand`,
+ );
+ }
+ }
+
+ function assertChannelReachable(channelSlug) {
+ // One check per channel per run, cached as a promise so a rejection replays
+ // for every clip of that channel instead of re-statting for each.
+ let pending = checkedChannels.get(channelSlug);
+ if (!pending) {
+ pending = checkChannelReachable(channelSlug);
+ checkedChannels.set(channelSlug, pending);
+ }
+ return pending;
+ }
+
async function fromLocal(channelSlug, videoId) {
+ // Throws a CueLookupError — which carries no `code`, so `load`'s ENOENT
+ // fallback rethrows it rather than reaching for the archive.
+ await assertChannelReachable(channelSlug);
const p = path.join(channelsDir, channelSlug, "data", videoId, "transcript.cues.json");
const parsed = JSON.parse(await readFile(p, "utf8"));
return { ...parsed, from: "local" };
diff --git a/umtool/report-to-video/cues.test.mjs b/umtool/report-to-video/cues.test.mjs
@@ -7,7 +7,7 @@
// Run with: pnpm test:scripts
import assert from "node:assert/strict";
import test from "node:test";
-import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
+import { mkdtemp, mkdir, symlink, writeFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
@@ -148,6 +148,184 @@ test("a local corpus is preferred over the network", async () => {
}
});
+// --- a channel the corpus holds but cannot reach ----------------------------
+//
+// The bug these cover: `data/` may be a symlink to another drive, with the
+// target recorded in the channel's `config.json` as `dataDir`. An unmounted
+// drive reads as a plain ENOENT, which used to fall through to the archive —
+// so a relocated channel silently cut from a snapshot's cues, which can differ
+// from the corpus's by seconds.
+
+const SCRATCH = process.env.CUES_TEST_DIR ?? tmpdir();
+
+// A mirrored channel: <root>/channels/<slug>/ with a config.json, and `data/`
+// however the caller wants it.
+async function corpusWith(slug, { dataDir, data } = {}) {
+ const root = await mkdtemp(path.join(SCRATCH, "cues-reach-"));
+ const channelDir = path.join(root, slug);
+ await mkdir(channelDir, { recursive: true });
+ await writeFile(
+ path.join(channelDir, "config.json"),
+ JSON.stringify(dataDir ? { url: "https://x/", dataDir } : { url: "https://x/" }),
+ );
+ if (data === "symlink") await symlink(dataDir, path.join(channelDir, "data"));
+ if (data === "dir") await mkdir(path.join(channelDir, "data"), { recursive: true });
+ return root;
+}
+
+async function writeCues(dir, videoId, record) {
+ const vdir = path.join(dir, videoId);
+ await mkdir(vdir, { recursive: true });
+ await writeFile(path.join(vdir, "transcript.cues.json"), JSON.stringify(record));
+}
+
+test("a relocated channel whose drive is not mounted throws, and never fetches", async () => {
+ const missing = path.join(SCRATCH, "cues-not-mounted-" + process.pid, "chan", "data");
+ const root = await corpusWith("chan", { dataDir: missing, data: "symlink" });
+ const seen = [];
+ try {
+ const src = createCueSource({
+ channelsDir: root,
+ siteOrigin: ORIGIN,
+ cacheDir: null,
+ fetchImpl: stubFetch(ROUTES, seen),
+ });
+ await assert.rejects(() => src.load("chan", "vid1"), (err) => {
+ assert.equal(err.name, "CueLookupError");
+ assert.match(err.message, /chan/);
+ assert.match(err.message, /drive not mounted/);
+ assert.ok(err.message.includes(missing), "the error names the unreachable path");
+ return true;
+ });
+ assert.deepEqual(seen, [], "an unreachable channel must not fall through to the archive");
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("a dataDir with no symlink at all is refused too", async () => {
+ const root = await corpusWith("chan", { dataDir: path.join(SCRATCH, "elsewhere") });
+ const seen = [];
+ try {
+ const src = createCueSource({
+ channelsDir: root,
+ siteOrigin: ORIGIN,
+ cacheDir: null,
+ fetchImpl: stubFetch(ROUTES, seen),
+ });
+ await assert.rejects(() => src.load("chan", "vid1"), (err) => {
+ assert.match(err.message, /symlink is missing/);
+ return true;
+ });
+ assert.deepEqual(seen, []);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("a relocation in flight is refused rather than half-read", async () => {
+ const root = await corpusWith("chan", { data: "dir" });
+ await writeFile(
+ path.join(root, "chan", ".relocating.json"),
+ JSON.stringify({ target: "/mnt/big/chan/data", direction: "out", phase: "copy" }),
+ );
+ const seen = [];
+ try {
+ const src = createCueSource({
+ channelsDir: root,
+ siteOrigin: ORIGIN,
+ cacheDir: null,
+ fetchImpl: stubFetch(ROUTES, seen),
+ });
+ await assert.rejects(() => src.load("chan", "vid1"), (err) => {
+ assert.match(err.message, /relocation \(out\) is in progress/);
+ return true;
+ });
+ assert.deepEqual(seen, []);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("a reachable dataDir is read from, wherever it points", async () => {
+ const elsewhere = await mkdtemp(path.join(SCRATCH, "cues-drive-"));
+ const target = path.join(elsewhere, "chan", "data");
+ await mkdir(target, { recursive: true });
+ await writeCues(target, "vid1", { ...RECORD, title: "RELOCATED COPY" });
+ const root = await corpusWith("chan", { dataDir: target, data: "symlink" });
+ const seen = [];
+ try {
+ const src = createCueSource({
+ channelsDir: root,
+ siteOrigin: ORIGIN,
+ cacheDir: null,
+ fetchImpl: stubFetch(ROUTES, seen),
+ });
+ const got = await src.load("chan", "vid1");
+ assert.equal(got.from, "local");
+ assert.equal(got.title, "RELOCATED COPY");
+ assert.deepEqual(seen, [], "a reachable relocation is still a local read");
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ await rm(elsewhere, { recursive: true, force: true });
+ }
+});
+
+test("a reachable channel that simply lacks this video still falls back to HTTP", async () => {
+ // The narrow scope of the guard: present-but-empty is not unreachable.
+ const root = await corpusWith("chan", { data: "dir" });
+ try {
+ const src = createCueSource({
+ channelsDir: root,
+ siteOrigin: ORIGIN,
+ cacheDir: null,
+ fetchImpl: stubFetch(ROUTES),
+ });
+ const got = await src.load("chan", "vid1");
+ assert.equal(got.from, "http");
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("no local corpus at all is the archive-only case, not an unreachable channel", async () => {
+ // A clone with no `transcripts/channels` is a first-class way to use this
+ // repo: the cues come over HTTP and nothing about that is an error.
+ const seen = [];
+ const src = createCueSource({
+ channelsDir: path.join(SCRATCH, "definitely-no-corpus-here"),
+ siteOrigin: ORIGIN,
+ cacheDir: null,
+ fetchImpl: stubFetch(ROUTES, seen),
+ });
+ const got = await src.load("chan", "vid1");
+ assert.equal(got.from, "http");
+ assert.deepEqual(seen, [
+ `${ORIGIN}/corpus.json`,
+ `${ORIGIN}/transcripts/chan/manifest.json`,
+ `${ORIGIN}/transcripts/chan/page-0000.json`,
+ ]);
+});
+
+test("--cue-source local still refuses to fall back, unreachable or not", async () => {
+ const root = await corpusWith("chan", { data: "dir" });
+ try {
+ const src = createCueSource({
+ channelsDir: root,
+ siteOrigin: ORIGIN,
+ cacheDir: null,
+ fetchImpl: stubFetch(ROUTES),
+ prefer: "local",
+ });
+ await assert.rejects(() => src.load("chan", "vid1"), (err) => {
+ assert.match(err.message, /forbids falling back/);
+ return true;
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
// --- the Rumble two-id trap -------------------------------------------------
test("a published-id miss fails loudly, naming the two-id trap", async () => {
