Archilyzer · Source

archilyzer

Archilyzer
git clone https://archilyzer.pages.dev/source/archilyzer.git
Log | Files | Refs | README | LICENSE

commit 1ddbd1c64b98f204f5186b72b2326a9e3d3b2323
parent 1939ef917c4888eb13c68427ca6fcc4834819256
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date:   Fri, 11 Sep 2026 11:06:39 -0400

common: one module knows where a channel's media actually is

`channels/<slug>/data` is joined inline at ~74 call sites and is the on-disk
contract yt-dlp's cwd-relative output template, every reader and the LMDB index
depend on. Relocating a channel to another drive therefore does not move that
path — it becomes an absolute symlink and `config.json` records the target in a
new `dataDir` field. Nothing in common/, editor/ or export/ is symlink-aware and
nothing needs to be.

What the symlink owes is one new failure mode: a dangling link reads ENOENT, and
the three places that enumerate `data/` swallow ENOENT as "no videos" — which to
an unattended runner means everything is undownloaded and is an instruction to
re-download hundreds of gigabytes onto the volume that was too full to hold
them. `inspectChannelMedia` is the one place that can tell an unmounted drive
from an empty channel: two stats and (at most) a four-line JSON read, returning
in-place / ok / unreachable / in-transition / inconsistent.

It lives in lib/ and so may not import controller/, which is where
readChannelConfig is. Hence the optional `config` argument — a caller holding a
parsed config passes it and pays nothing; absent, the module reads config.json
itself and pulls out the one field. No architecture.test.ts allow-list entry.

The link points at a DEEP path (<root>/<slug>/data), so an empty mountpoint is
still unreachable rather than being mistaken for the media. Disk and config
disagreeing in either direction is "inconsistent" and is never guessed past.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Diffstat:
Mcommon/lib/channelConfig.ts | 11+++++++++++
Acommon/lib/channelMedia.test.ts | 252+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Acommon/lib/channelMedia.ts | 309+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 572 insertions(+), 0 deletions(-)

diff --git a/common/lib/channelConfig.ts b/common/lib/channelConfig.ts @@ -99,6 +99,14 @@ export type ChannelConfig = { // channel's large videos land on a different disk than the rest. Resolved by // savedVideoDir() in common/lib/savedVideo.ts. Empty/whitespace = use global. savedVideosDir?: string; + // Where this channel's downloaded media ACTUALLY lives, when it has been + // relocated to another drive: the absolute path `channels/<slug>/data` is a + // symlink to. Blank/absent = in place. Written ONLY by the relocate job on + // success (common/controller/relocateChannelMedia.ts) — it is a record of + // what is on disk, never a free-text field, because a value that disagrees + // with the link is an "inconsistent" channel that every guard refuses. See + // common/lib/channelMedia.ts. + dataDir?: string; ytdlpExtraArgs?: string[]; subLangs?: string; lastSyncedAt?: string; @@ -258,6 +266,9 @@ export function parseChannelConfig(raw: unknown): ChannelConfig | null { if (typeof r.savedVideosDir === "string" && r.savedVideosDir.trim() !== "") { config.savedVideosDir = r.savedVideosDir.trim(); } + if (typeof r.dataDir === "string" && r.dataDir.trim() !== "") { + config.dataDir = r.dataDir.trim(); + } if ( Array.isArray(r.ytdlpExtraArgs) && r.ytdlpExtraArgs.every((x) => typeof x === "string") diff --git a/common/lib/channelMedia.test.ts b/common/lib/channelMedia.test.ts @@ -0,0 +1,252 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { + ChannelMediaUnreachableError, + assertChannelMediaReachable, + inspectChannelMedia, + readRelocationMarker, + relocatedDataDir, + RELOCATION_MARKER_FILENAME, + type ChannelMediaPaths, +} from "./channelMedia"; + +// Run with: +// pnpm --filter yt-dlp-transcript-common exec tsx --test lib/channelMedia.test.ts +// +// Everything here happens inside one mkdtemp; nothing reads a real corpus. + +async function withTmp( + fn: (paths: ChannelMediaPaths, root: string, dir: string) => Promise<void>, +): Promise<void> { + const dir = await mkdtemp(path.join(tmpdir(), "ttb-media-")); + const paths: ChannelMediaPaths = { channelsDir: path.join(dir, "channels") }; + const root = path.join(dir, "platter"); + await mkdir(paths.channelsDir, { recursive: true }); + try { + await fn(paths, root, dir); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +async function seedChannel( + paths: ChannelMediaPaths, + slug: string, + config: Record<string, unknown> = {}, +): Promise<string> { + const channelDir = path.join(paths.channelsDir, slug); + await mkdir(channelDir, { recursive: true }); + await writeFile( + path.join(channelDir, "config.json"), + JSON.stringify({ url: "https://example.com/c", ...config }, null, 2), + ); + return channelDir; +} + +test("relocatedDataDir fixes the <root>/<slug>/data suffix", () => { + assert.equal(relocatedDataDir("/mnt/p", "alpha"), "/mnt/p/alpha/data"); + assert.equal(relocatedDataDir(" /mnt/p ", "alpha"), "/mnt/p/alpha/data"); +}); + +test("a real data dir with no config.dataDir is in-place", async () => { + await withTmp(async (paths) => { + const channelDir = await seedChannel(paths, "alpha"); + await mkdir(path.join(channelDir, "data", "v1"), { recursive: true }); + const loc = await inspectChannelMedia(paths, "alpha"); + assert.equal(loc.status, "in-place"); + assert.equal(loc.relocated, false); + assert.equal(loc.target, undefined); + assert.equal(loc.dataDir, path.join(channelDir, "data")); + await assertChannelMediaReachable(paths, "alpha"); + }); +}); + +test("a channel that has downloaded nothing is in-place, not an error", async () => { + await withTmp(async (paths) => { + await seedChannel(paths, "alpha"); + const loc = await inspectChannelMedia(paths, "alpha"); + assert.equal(loc.status, "in-place"); + await assertChannelMediaReachable(paths, "alpha"); + }); +}); + +test("a link agreeing with config and pointing at a live dir is ok", async () => { + await withTmp(async (paths, root) => { + const target = relocatedDataDir(root, "alpha"); + await mkdir(path.join(target, "v1"), { recursive: true }); + const channelDir = await seedChannel(paths, "alpha", { dataDir: target }); + await symlink(target, path.join(channelDir, "data")); + const loc = await inspectChannelMedia(paths, "alpha"); + assert.equal(loc.status, "ok"); + assert.equal(loc.relocated, true); + assert.equal(loc.target, target); + await assertChannelMediaReachable(paths, "alpha"); + }); +}); + +test("a dangling link (drive not mounted) is unreachable and throws", async () => { + await withTmp(async (paths, root) => { + const target = relocatedDataDir(root, "alpha"); + const channelDir = await seedChannel(paths, "alpha", { dataDir: target }); + // The link is made WITHOUT creating the target: exactly an unmounted drive. + await symlink(target, path.join(channelDir, "data")); + const loc = await inspectChannelMedia(paths, "alpha"); + assert.equal(loc.status, "unreachable"); + assert.equal(loc.relocated, true); + assert.match(loc.detail ?? "", /does not exist/); + await assert.rejects( + () => assertChannelMediaReachable(paths, "alpha"), + (err: unknown) => { + assert.ok(err instanceof ChannelMediaUnreachableError); + assert.equal(err.slug, "alpha"); + assert.equal(err.status, "unreachable"); + assert.match(err.message, /not reachable/); + return true; + }, + ); + }); +}); + +test("an EMPTY mountpoint is still unreachable — the link points deep", async () => { + await withTmp(async (paths, root) => { + const target = relocatedDataDir(root, "alpha"); + // The root exists (mountpoint present) but holds nothing. + await mkdir(root, { recursive: true }); + const channelDir = await seedChannel(paths, "alpha", { dataDir: target }); + await symlink(target, path.join(channelDir, "data")); + const loc = await inspectChannelMedia(paths, "alpha"); + assert.equal(loc.status, "unreachable"); + }); +}); + +test("a marker makes the channel in-transition whatever the disk says", async () => { + await withTmp(async (paths, root) => { + const target = relocatedDataDir(root, "alpha"); + const channelDir = await seedChannel(paths, "alpha"); + await mkdir(path.join(channelDir, "data"), { recursive: true }); + await writeFile( + path.join(channelDir, RELOCATION_MARKER_FILENAME), + JSON.stringify({ + target, + direction: "out", + startedAt: new Date().toISOString(), + phase: "copy", + }), + ); + const loc = await inspectChannelMedia(paths, "alpha"); + assert.equal(loc.status, "in-transition"); + assert.equal(loc.marker?.phase, "copy"); + assert.equal(loc.marker?.direction, "out"); + assert.equal(loc.target, target); + await assert.rejects( + () => assertChannelMediaReachable(paths, "alpha"), + ChannelMediaUnreachableError, + ); + + const marker = await readRelocationMarker(paths, "alpha"); + assert.equal(marker?.target, target); + }); +}); + +test("a link that disagrees with config is inconsistent, never guessed past", async () => { + await withTmp(async (paths, root) => { + const real = relocatedDataDir(root, "alpha"); + const recorded = relocatedDataDir(path.join(root, "other"), "alpha"); + await mkdir(real, { recursive: true }); + const channelDir = await seedChannel(paths, "alpha", { dataDir: recorded }); + await symlink(real, path.join(channelDir, "data")); + const loc = await inspectChannelMedia(paths, "alpha"); + assert.equal(loc.status, "inconsistent"); + assert.match(loc.detail ?? "", /points at/); + await assert.rejects( + () => assertChannelMediaReachable(paths, "alpha"), + ChannelMediaUnreachableError, + ); + }); +}); + +test("a link with no config.dataDir is inconsistent", async () => { + await withTmp(async (paths, root) => { + const target = relocatedDataDir(root, "alpha"); + await mkdir(target, { recursive: true }); + const channelDir = await seedChannel(paths, "alpha"); + await symlink(target, path.join(channelDir, "data")); + const loc = await inspectChannelMedia(paths, "alpha"); + assert.equal(loc.status, "inconsistent"); + assert.equal(loc.relocated, false); + assert.match(loc.detail ?? "", /records no dataDir/); + }); +}); + +test("config.dataDir with a real directory on disk is inconsistent", async () => { + await withTmp(async (paths, root) => { + const target = relocatedDataDir(root, "alpha"); + const channelDir = await seedChannel(paths, "alpha", { dataDir: target }); + await mkdir(path.join(channelDir, "data"), { recursive: true }); + const loc = await inspectChannelMedia(paths, "alpha"); + assert.equal(loc.status, "inconsistent"); + assert.match(loc.detail ?? "", /never moved/); + }); +}); + +test("config.dataDir with no data/ at all is inconsistent (link gone)", async () => { + await withTmp(async (paths, root) => { + const target = relocatedDataDir(root, "alpha"); + await seedChannel(paths, "alpha", { dataDir: target }); + const loc = await inspectChannelMedia(paths, "alpha"); + assert.equal(loc.status, "inconsistent"); + assert.match(loc.detail ?? "", /symlink is missing/); + }); +}); + +test("a passed config is used verbatim; config.json is only read when it is absent", async () => { + await withTmp(async (paths, root) => { + const target = relocatedDataDir(root, "alpha"); + await mkdir(target, { recursive: true }); + // config.json on disk says NOTHING about a relocation... + const channelDir = await seedChannel(paths, "alpha"); + await symlink(target, path.join(channelDir, "data")); + + // ...so reading it itself gives "inconsistent"... + assert.equal( + (await inspectChannelMedia(paths, "alpha")).status, + "inconsistent", + ); + // ...while a caller that hands over the config it already holds gets the + // answer for THAT config, with no second read. + const passed = await inspectChannelMedia(paths, "alpha", { + dataDir: target, + }); + assert.equal(passed.status, "ok"); + assert.equal(passed.target, target); + + // An explicit null means "I have no config" and must not silently fall back + // to reading the file. + const nulled = await inspectChannelMedia(paths, "alpha", null); + assert.equal(nulled.status, "inconsistent"); + }); +}); + +test("a blank config.dataDir means in place", async () => { + await withTmp(async (paths) => { + const channelDir = await seedChannel(paths, "alpha", { dataDir: " " }); + await mkdir(path.join(channelDir, "data"), { recursive: true }); + assert.equal((await inspectChannelMedia(paths, "alpha")).status, "in-place"); + assert.equal( + (await inspectChannelMedia(paths, "alpha", { dataDir: " " })).status, + "in-place", + ); + }); +}); + +test("an unreadable or missing config.json is not a relocation", async () => { + await withTmp(async (paths) => { + const channelDir = path.join(paths.channelsDir, "alpha"); + await mkdir(path.join(channelDir, "data"), { recursive: true }); + await writeFile(path.join(channelDir, "config.json"), "{ not json"); + assert.equal((await inspectChannelMedia(paths, "alpha")).status, "in-place"); + }); +}); diff --git a/common/lib/channelMedia.ts b/common/lib/channelMedia.ts @@ -0,0 +1,309 @@ +import path from "node:path"; +import { lstat, readFile, readlink, stat } from "node:fs/promises"; +import type { Paths } from "./paths"; +import type { ChannelConfig } from "./channelConfig"; + +// WHERE A CHANNEL'S MEDIA ACTUALLY IS, and whether it can be reached. +// +// A channel's downloaded media lives at `channels/<slug>/data/`. That path is +// joined inline at ~74 call sites and is the on-disk contract every reader, +// yt-dlp's cwd-relative output template and the LMDB index depend on, so +// relocating a channel to another drive does NOT change it: `data/` becomes an +// absolute SYMLINK to `<root>/<slug>/data` and `config.json` records the target +// in `dataDir`. Every existing reader follows the link transparently — there is +// no symlink-aware code anywhere in common/, editor/ or export/, and there does +// not need to be. +// +// What that buys in call-site churn it owes in one new failure mode: an +// unmounted drive. A dangling link reads as ENOENT, and the three places that +// enumerate `data/` swallow ENOENT as "this channel has no videos" — which to an +// unattended runner means *everything is undownloaded* and is an instruction to +// re-download hundreds of gigabytes onto the volume that was too full to hold +// them. This module is the one place that can tell those two apart, and the +// guards that call it are what make the symlink safe. +// +// IT LIVES IN lib/ AND MAY NOT IMPORT controller/ (architecture.test.ts), which +// is where readChannelConfig is. Hence the optional `config` argument: a caller +// holding a parsed config passes it and pays nothing, and when it is absent this +// module reads `<channelDir>/config.json` itself and pulls out the one field it +// needs. That is a four-line JSON read, not a second config parser — nothing +// here validates or defaults anything else in the file. + +// Only the paths field this module needs, so a caller (and a test) does not have +// to build a whole Paths to ask where a channel's media is. +export type ChannelMediaPaths = Pick<Paths, "channelsDir">; + +// Marker written in the CHANNEL dir (never in data/, which is the thing being +// moved) for the duration of a relocation. Its presence means "media is in +// transition" to every guard, and its `phase` is what lets an interrupted job +// resume rather than restart. +export const RELOCATION_MARKER_FILENAME = ".relocating.json"; + +export type RelocationDirection = "out" | "back"; +export type RelocationPhase = "copy" | "swap" | "reclaim"; + +export type RelocationMarker = { + // Absolute path of the relocated data dir: <root>/<slug>/data. + target: string; + direction: RelocationDirection; + startedAt: string; + phase: RelocationPhase; +}; + +export type ChannelMediaStatus = + // No relocation: `data/` is a real directory (or does not exist yet). + | "in-place" + // Relocated, link and config agree, and the target is a reachable directory. + | "ok" + // Relocated, but the target is not there — almost always an unmounted drive. + | "unreachable" + // A relocation is in flight (or was interrupted): the marker is present. + | "in-transition" + // Disk and config disagree, in either direction. Never guessed past. + | "inconsistent"; + +export type ChannelMediaLocation = { + // Always channelDir/data — the path every reader uses, relocated or not. + dataDir: string; + // Whether config.json records a relocation target. + relocated: boolean; + // config.dataDir (or, mid-transition with no config yet, the marker's target). + target?: string; + status: ChannelMediaStatus; + // Operator-readable reason, set for every status except "in-place" and "ok". + detail?: string; + // The in-flight marker, when one is present. + marker?: RelocationMarker; +}; + +// Thrown by assertChannelMediaReachable. A distinct class so a caller can tell +// "this channel's drive is not mounted" from any other I/O failure and skip +// rather than fail the whole lane. +export class ChannelMediaUnreachableError extends Error { + readonly slug: string; + readonly status: ChannelMediaStatus; + readonly location: ChannelMediaLocation; + constructor(slug: string, location: ChannelMediaLocation) { + super( + `Channel "${slug}": media is not reachable — ${ + location.detail ?? location.status + }`, + ); + this.name = "ChannelMediaUnreachableError"; + this.slug = slug; + this.status = location.status; + this.location = location; + } +} + +// The relocated layout, fixed so one root can hold many channels and the shape +// mirrors the saved-video store (<root>/<slug>/<...>). The `<slug>/data` suffix +// is not configurable: deleteChannel and renameChannel recognise a target by it. +export function relocatedDataDir(root: string, slug: string): string { + return path.join(root.trim(), slug, "data"); +} + +export function channelMediaDir( + paths: ChannelMediaPaths, + slug: string, +): string { + return path.join(paths.channelsDir, slug, "data"); +} + +export function relocationMarkerPath( + paths: ChannelMediaPaths, + slug: string, +): string { + return path.join(paths.channelsDir, slug, RELOCATION_MARKER_FILENAME); +} + +function parseMarker(raw: unknown): RelocationMarker | null { + if (!raw || typeof raw !== "object") return null; + const r = raw as Record<string, unknown>; + if (typeof r.target !== "string" || r.target.trim() === "") return null; + const direction = r.direction === "back" ? "back" : "out"; + const phase = + r.phase === "swap" || r.phase === "reclaim" ? r.phase : "copy"; + return { + target: r.target, + direction, + startedAt: typeof r.startedAt === "string" ? r.startedAt : "", + phase, + }; +} + +export async function readRelocationMarker( + paths: ChannelMediaPaths, + slug: string, +): Promise<RelocationMarker | null> { + try { + const raw = await readFile(relocationMarkerPath(paths, slug), "utf8"); + return parseMarker(JSON.parse(raw)); + } catch { + return null; + } +} + +// The `dataDir` field alone, read straight off config.json. Deliberately NOT +// parseChannelConfig: this runs in guards on hot paths and must not depend on +// the controller that owns the rest of the schema. +async function readConfiguredDataDir( + paths: ChannelMediaPaths, + slug: string, +): Promise<string | undefined> { + try { + const raw = await readFile( + path.join(paths.channelsDir, slug, "config.json"), + "utf8", + ); + const parsed = JSON.parse(raw) as { dataDir?: unknown }; + if (typeof parsed.dataDir !== "string") return undefined; + const trimmed = parsed.dataDir.trim(); + return trimmed === "" ? undefined : trimmed; + } catch { + return undefined; + } +} + +// Two stats and (at most) one small JSON read. Render-safe: nothing here walks a +// directory, so calling it per channel on a listing page costs three syscalls a +// row. +export async function inspectChannelMedia( + paths: ChannelMediaPaths, + slug: string, + config?: Pick<ChannelConfig, "dataDir"> | null, +): Promise<ChannelMediaLocation> { + const dataDir = channelMediaDir(paths, slug); + const configured = + config === undefined + ? await readConfiguredDataDir(paths, slug) + : config?.dataDir && config.dataDir.trim() !== "" + ? config.dataDir.trim() + : undefined; + + const marker = await readRelocationMarker(paths, slug); + if (marker) { + return { + dataDir, + relocated: Boolean(configured), + target: configured ?? marker.target, + status: "in-transition", + detail: + `a media relocation (${marker.direction}) is in progress or was ` + + `interrupted at phase "${marker.phase}" — target ${marker.target}`, + marker, + }; + } + + let link: Awaited<ReturnType<typeof lstat>> | null = null; + try { + link = await lstat(dataDir); + } catch { + // No data/ at all. With no configured target that is just a channel that + // has downloaded nothing yet — the overwhelmingly common case, and not an + // error. With one, the link this channel is supposed to have is gone. + if (!configured) return { dataDir, relocated: false, status: "in-place" }; + return { + dataDir, + relocated: true, + target: configured, + status: "inconsistent", + detail: + `config.json records dataDir ${configured} but ${dataDir} does not ` + + `exist — the symlink is missing`, + }; + } + + if (link.isSymbolicLink()) { + let linkTarget = ""; + try { + linkTarget = await readlink(dataDir); + } catch { + /* readlink of a link we just lstat'd: treat as unreadable below */ + } + if (!configured) { + return { + dataDir, + relocated: false, + target: linkTarget || undefined, + status: "inconsistent", + detail: + `${dataDir} is a symlink to ${linkTarget || "(unreadable)"} but ` + + `config.json records no dataDir`, + }; + } + if (path.resolve(linkTarget) !== path.resolve(configured)) { + return { + dataDir, + relocated: true, + target: configured, + status: "inconsistent", + detail: + `${dataDir} points at ${linkTarget || "(unreadable)"} but ` + + `config.json records ${configured}`, + }; + } + // The link points at a DEEP path (<root>/<slug>/data), so an unmounted root + // gives ENOENT here. An empty mountpoint can never be mistaken for the + // media, which is the whole reason the suffix is fixed. + try { + const st = await stat(configured); + if (!st.isDirectory()) { + return { + dataDir, + relocated: true, + target: configured, + status: "unreachable", + detail: `${configured} exists but is not a directory`, + }; + } + } catch { + return { + dataDir, + relocated: true, + target: configured, + status: "unreachable", + detail: `${configured} does not exist (drive not mounted?)`, + }; + } + return { dataDir, relocated: true, target: configured, status: "ok" }; + } + + if (!link.isDirectory()) { + return { + dataDir, + relocated: Boolean(configured), + target: configured, + status: "inconsistent", + detail: `${dataDir} is neither a directory nor a symlink`, + }; + } + + if (configured) { + return { + dataDir, + relocated: true, + target: configured, + status: "inconsistent", + detail: + `config.json records dataDir ${configured} but ${dataDir} is a real ` + + `directory — the media was never moved, or was moved back by hand`, + }; + } + return { dataDir, relocated: false, status: "in-place" }; +} + +// "ok" and "in-place" pass; everything else throws. An in-transition or +// inconsistent channel is refused for the same reason an unreachable one is: +// the caller would otherwise read a half-populated or empty dir as the truth. +export async function assertChannelMediaReachable( + paths: ChannelMediaPaths, + slug: string, + config?: Pick<ChannelConfig, "dataDir"> | null, +): Promise<ChannelMediaLocation> { + const location = await inspectChannelMedia(paths, slug, config); + if (location.status === "ok" || location.status === "in-place") { + return location; + } + throw new ChannelMediaUnreachableError(slug, location); +}