import path from "node:path"; import { lstat, readFile, readlink, rm, stat } from "node:fs/promises"; import type { Paths } from "./paths"; import type { ChannelConfig } from "./channelConfig"; import { NOT_ANSWERING, healthTimings, isDriveNotAnswering, onDrive, sinceText, stalledLocationForPath, type LocationHealth, } from "./storageHealth"; import { secondsText } from "./storageHealthTimings"; import { MEDIA_LINK_NAME } from "./mediaTier-server"; // WHERE A CHANNEL'S MEDIA ACTUALLY IS, and whether it can be reached. // // A channel's files live at `channels//data//`. That path is joined // inline at ~74 call sites and is the on-disk contract every reader, yt-dlp's // cwd-relative output template and the LMDB index depend on, so it never moves. // Since release 17 only the BIG files leave it (lib/mediaTier.ts says which): // each becomes a RELATIVE link `data// -> ../../media//`, // and `channels//media` is either a real directory on the corpus disk // (tiered in place) or ONE absolute symlink to `//media` on another // drive, recorded in `config.json` as `mediaDir` (relocated). The text — // transcripts, cues, metadata, every sidecar — stays on the corpus disk. // // What the link buys in call-site churn it owes in one failure mode: an // unmounted drive. A dangling link reads as ENOENT. For the media alone that is // survivable — a reader of the text never touches it — and this module is the // one place that can tell the media's states apart, so the guards that call it // are what make the link safe: `assertChannelMediaReachable` for a job that // opens a big file, `assertChannelTextReadable` for one that reads only text. // // THE RETIRED LAYOUT. Before release 17 a relocation moved the whole `data/` // (an absolute symlink `data -> //data`, `config.dataDir`). Such a // channel is `legacy`: its text is on the far drive too, so it is held by BOTH // guards — every lane, every media job, the index and stats builds — until // `archilyzer storage migrate-tier ` brings its text home. `dataDir` is // still parsed one release for exactly that (channelConfig.ts). // // IT LIVES IN lib/ AND MAY NOT IMPORT controller/ (architecture.test.ts), which // is where readChannelConfig is. Hence the optional `config` argument: a caller // holding a parsed config passes it and pays nothing, and when it is absent this // module reads `/config.json` itself and pulls out the one field it // needs. That is a four-line JSON read, not a second config parser — nothing // here validates or defaults anything else in the file. // Only the paths field this module needs, so a caller (and a test) does not have // to build a whole Paths to ask where a channel's media is. export type ChannelMediaPaths = Pick; // Marker written in the CHANNEL dir (never in data/, which is the thing being // moved) for the duration of a relocation. Its presence means "media is in // transition" to every guard, and its `phase` is what lets an interrupted job // resume rather than restart. export const RELOCATION_MARKER_FILENAME = ".relocating.json"; export type RelocationDirection = "out" | "back"; export type RelocationPhase = "copy" | "swap" | "reclaim"; // What a marker is moving. `media` (or absent): the channel's media tier — its // text stays readable, so only its media writers are held. `tier-migration`: // the one-off migration off the retired layout (common/bin/migrate-media-tier.ts), // which rebuilds `data/` itself, so the text is held too. export type RelocationScope = "media" | "tier-migration"; export type RelocationMarker = { // Absolute path of the relocated media dir: //media (the retired // mover wrote //data). target: string; direction: RelocationDirection; startedAt: string; phase: RelocationPhase; scope?: RelocationScope; }; export type ChannelMediaStatus = // No relocation: no `media` link (a classic channel, or one tiered into a // real `media/` directory on the corpus disk) and no `mediaDir`. | "in-place" // Relocated: the `media` link and `mediaDir` agree, and the target is a // reachable directory. | "ok" // Relocated, but the target is not there — almost always an unmounted drive. | "unreachable" // A relocation is in flight (or was interrupted): the marker is present. | "in-transition" // Disk and config disagree, in either direction. Never guessed past. | "inconsistent" // Relocated onto a storage location whose drive is not answering // (`lib/storageHealth.ts`). Answered from memory, WITHOUT a filesystem call: // a call there would block one of the process's few I/O threads for as long // as the drive takes to come back. Held and refused like `unreachable`. | "stalled" // The RETIRED whole-directory layout: `data/` is a link, or config.json still // records `dataDir`. Its text is not on the corpus disk, so it is held by the // text guard as well as the media one until `archilyzer storage migrate-tier` // runs. Answered without a call to the far drive. | "legacy"; export type ChannelMediaLocation = { // Always channelDir/data — the real text dir every reader uses (a link only // on a `legacy` channel). dataDir: string; // Always channelDir/media — the media tier's one name. mediaLink: string; // Whether config.json records a relocation target (`mediaDir`, or the // retired `dataDir` on a legacy channel). relocated: boolean; // config.mediaDir (or, mid-transition with no config yet, the marker's // target; on a legacy channel the retired `dataDir`). target?: string; // The text tier: `data/`, and whether a reader may walk it. Not readable on a // `legacy` channel (its text is on the far drive), when `data/` is something // other than a directory, or mid tier-migration. A channel with no `data/` // yet is readable (it has downloaded nothing). text: { dir: string; readable: boolean }; status: ChannelMediaStatus; // Operator-readable reason, set for every status except "in-place" and "ok". detail?: string; // The in-flight marker, when one is present. marker?: RelocationMarker; }; // Thrown by assertChannelMediaReachable. A distinct class so a caller can tell // "this channel's drive is not mounted" from any other I/O failure and skip // rather than fail the whole lane. export class ChannelMediaUnreachableError extends Error { readonly slug: string; readonly status: ChannelMediaStatus; readonly location: ChannelMediaLocation; constructor(slug: string, location: ChannelMediaLocation) { super( `Channel "${slug}": media is not reachable — ${ location.detail ?? location.status }`, ); this.name = "ChannelMediaUnreachableError"; this.slug = slug; this.status = location.status; this.location = location; } } // Thrown by assertChannelTextReadable: the channel's TEXT cannot be walked — a // `legacy` channel, a `data/` that is not a directory, a tier migration in // flight. Same shape as the media error so a lane can skip on either. export class ChannelTextUnreadableError extends Error { readonly slug: string; readonly status: ChannelMediaStatus; readonly location: ChannelMediaLocation; constructor(slug: string, location: ChannelMediaLocation, detail: string) { super(`Channel "${slug}": its text is not readable — ${detail}`); this.name = "ChannelTextUnreadableError"; this.slug = slug; this.status = location.status; this.location = location; } } // THE RETIRED layout's target shape, `//data`. Nothing moves a // channel to it any more (the mover, the re-point and the rename use // `relocatedMediaDir`, lib/mediaTier-server.ts); it names what a `legacy` // channel's tree is, for the tier migration and the tests that build one. export function relocatedDataDir(root: string, slug: string): string { return path.join(root.trim(), slug, "data"); } // WHETHER A MARKER HOLDS THE TEXT TOO. A media move (`scope` "media", or none // with a `//media` target) carries `media/` only, so its channel's // text stays readable. A tier migration rebuilds `data/` itself; and a marker // with no scope whose target is the retired `//data` shape is the // old whole-directory mover's, copying `data/` — both hold the text. export function markerHoldsText( marker: Pick, ): boolean { if (marker.scope === "tier-migration") return true; if (marker.scope === "media") return false; return path.basename(marker.target.replace(/\/+$/, "")) === "data"; } // The sentence a legacy channel is refused with, naming the way out. export function legacyDetail(slug: string): string { return ( `its media layout is the retired whole-directory one — run ` + `archilyzer storage migrate-tier ${slug}` ); } export function channelMediaDir( paths: ChannelMediaPaths, slug: string, ): string { return path.join(paths.channelsDir, slug, "data"); } export function relocationMarkerPath( paths: ChannelMediaPaths, slug: string, ): string { return path.join(paths.channelsDir, slug, RELOCATION_MARKER_FILENAME); } function parseMarker(raw: unknown): RelocationMarker | null { if (!raw || typeof raw !== "object") return null; const r = raw as Record; if (typeof r.target !== "string" || r.target.trim() === "") return null; const direction = r.direction === "back" ? "back" : "out"; const phase = r.phase === "swap" || r.phase === "reclaim" ? r.phase : "copy"; const marker: RelocationMarker = { target: r.target, direction, startedAt: typeof r.startedAt === "string" ? r.startedAt : "", phase, }; if (r.scope === "media" || r.scope === "tier-migration") marker.scope = r.scope; return marker; } export async function readRelocationMarker( paths: ChannelMediaPaths, slug: string, ): Promise { try { const raw = await readFile(relocationMarkerPath(paths, slug), "utf8"); return parseMarker(JSON.parse(raw)); } catch { return null; } } // THE OPERATOR'S LAST RESORT, and the only writer in this module. // // A channel carrying a marker is "in-transition" to every guard, which is // correct while a move is running and a dead end once one is not: the runners // skip the channel, runManagedFunction refuses its media jobs, and the snapshot // will not regenerate. The relocate job itself resumes from a marker and clears // it on success, so this is not the normal way out — it is for a marker whose // run is gone (a killed process, a container replaced mid-copy) and whose state // on disk the operator has looked at. // // It removes the marker and NOTHING else: no link is touched, no config is // rewritten, nothing is deleted. Whatever inspect() says afterwards is the truth // the disk was already telling, with the transition claim taken off the top. export async function clearRelocationMarker( paths: ChannelMediaPaths, slug: string, ): Promise { await rm(relocationMarkerPath(paths, slug), { force: true }); forgetChannelMedia(slug); } // The two fields this module needs, the way a ChannelConfig carries them. export type ChannelMediaConfig = Pick; type Configured = { mediaDir?: string; dataDir?: string }; function trimmed(v: unknown): string | undefined { if (typeof v !== "string") return undefined; const t = v.trim(); return t === "" ? undefined : t; } function configuredOf(config: ChannelMediaConfig | null | undefined): Configured { return { mediaDir: trimmed(config?.mediaDir), dataDir: trimmed(config?.dataDir) }; } // `mediaDir` and the retired `dataDir`, read straight off config.json. // Deliberately NOT parseChannelConfig: this runs in guards on hot paths and // must not depend on the controller that owns the rest of the schema. async function readConfigured( paths: ChannelMediaPaths, slug: string, ): Promise { try { const raw = await readFile( path.join(paths.channelsDir, slug, "config.json"), "utf8", ); const parsed = JSON.parse(raw) as { mediaDir?: unknown; dataDir?: unknown }; return { mediaDir: trimmed(parsed.mediaDir), dataDir: trimmed(parsed.dataDir) }; } catch { return {}; } } // The configured target's drive is not answering: the answer, from memory. The // detail names the location and when it stopped, never a path — the badge's // title shows it, and /storage has the paths. export function stalledMediaLocation( dataDir: string, configured: string, // Null when the drive is on no location the health state knows (a root typed // by hand) and the watchdog found it not answering, or when the watchdog // found it slow rather than stalled. health: LocationHealth | null, // The watchdog's own words, for a refusal that marked no location. detail?: string, ): ChannelMediaLocation { return { dataDir, mediaLink: path.join(path.dirname(dataDir), MEDIA_LINK_NAME), relocated: true, target: configured, // The text is on the corpus disk: a stalled MEDIA drive does not hold it. text: { dir: dataDir, readable: true }, status: "stalled", detail: health ? `${NOT_ANSWERING} (location "${health.label}", ${sinceText(health.since)})` : (detail ?? `${NOT_ANSWERING} (a read did not answer within ${secondsText(healthTimings().budgetMs)})`), }; } // THE STALL, for a caller holding a parsed config: the stalled location the // channel's MEDIA is on, or null. No I/O — the question a page or poll that // opens a big file asks before it does. Keys on `mediaDir`; on a legacy // channel (no `mediaDir` yet) on its retired `dataDir`, where everything is. export function channelMediaStall( config: ChannelMediaConfig | null | undefined, ): LocationHealth | null { const c = configuredOf(config); const dir = c.mediaDir ?? c.dataDir; return dir ? stalledLocationForPath(dir) : null; } // THE STALL OF THE TEXT: non-null only for a legacy channel whose retired // `dataDir` is on a stalled location — the one layout whose text is on another // drive. A page that reads only text (a listing, a transcript) asks this, not // `channelMediaStall`: a stalled media drive never holds the text. export function channelTextStall( config: ChannelMediaConfig | null | undefined, ): LocationHealth | null { const dir = configuredOf(config).dataDir; return dir ? stalledLocationForPath(dir) : null; } // --------------------------------------------------------------------------- // The memo // --------------------------------------------------------------------------- // // FIVE SECONDS, PER CHANNEL, KEYED BY SLUG AND THE CONFIGURED TARGETS. The home // page, /channels and the auto-queue status poll (every three seconds, four // lanes) each inspect every channel; without this each of them costs a few // syscalls a relocated channel, every time, on a drive that may be the slow // one. The key carries the configured `mediaDir` and `dataDir`, so a move that // rewrites either is a new key at once, and the movers clear the memo outright // (`forgetChannelMedia`) whenever a marker is written or removed. // // THE STALL GATE IS ASKED BEFORE A REMEMBERED ANSWER IS GIVEN, so a drive that // stops answering is seen on the next call even when an `ok` from four seconds // ago is remembered. A remembered `in-transition` is given as it is: the // marker is asked before the gate (below), and the movers forget the channel // whenever they write or remove it. // // `fresh: true` BYPASSES IT, and every caller that decides something from the // answer passes it: the start-of-work guard (`assertChannelMediaReachable`), // the movers, the index and stats builds, the storage watch, eviction and the // re-point preflight. The memo is for pages and polls. // // ONE MAP PER PROCESS (on `globalThis`), for the reason `storageHealth.ts` // gives: the movers that clear it run in one bundle layer and the pages that // read it in another. export const CHANNEL_MEDIA_MEMO_MS = 5_000; export type InspectOptions = { // Skip the memo: take a fresh answer, and do not remember it. fresh?: boolean; // Test seam for the clock. now?: number; }; type MediaMemo = Map; declare global { // eslint-disable-next-line no-var var __yttChannelMediaMemo__: MediaMemo | undefined; } function mediaMemo(): MediaMemo { if (!globalThis.__yttChannelMediaMemo__) { globalThis.__yttChannelMediaMemo__ = new Map(); } return globalThis.__yttChannelMediaMemo__; } // Forget what the memo holds: for one channel, or for every channel. The // movers call it whenever the disk changes under a channel (a marker written or // removed, a link swapped, a location re-pointed), so the next page sees it. export function forgetChannelMedia(slug?: string): void { const memo = mediaMemo(); if (slug === undefined) { memo.clear(); return; } for (const key of [...memo.keys()]) { if (key.split("\u0000")[1] === slug) memo.delete(key); } } function memoKey(paths: ChannelMediaPaths, slug: string, c: Configured): string { return `${paths.channelsDir}\u0000${slug}\u0000${c.mediaDir ?? ""}\u0000${c.dataDir ?? ""}`; } // Past this many entries, expired ones are swept on insert. A corpus has tens // of channels; this only matters to a process that inspects many corpora. const MEMO_SWEEP_AT = 512; // A few lstats and (at most) one small JSON read. Render-safe: nothing here // walks a directory, so calling it per channel on a listing page costs a few // syscalls a row — or none, for five seconds after the last answer (see the // memo above). On a stalled location the one call that reaches the drive (the // media target's `stat`) is not made; the marker, the links and config.json are // on the corpus disk and are read as usual. export async function inspectChannelMedia( paths: ChannelMediaPaths, slug: string, config?: ChannelMediaConfig | null, opts: InspectOptions = {}, ): Promise { const configured = config === undefined ? await readConfigured(paths, slug) : configuredOf(config); const now = opts.now ?? Date.now(); const key = memoKey(paths, slug, configured); const memo = mediaMemo(); if (!opts.fresh) { const hit = memo.get(key); if (hit && now - hit.at < CHANNEL_MEDIA_MEMO_MS) { const st = hit.location.status; if (st !== "in-transition" && st !== "legacy" && configured.mediaDir) { const stall = stalledLocationForPath(configured.mediaDir); if (stall) { return stalledMediaLocation(hit.location.dataDir, configured.mediaDir, stall); } } return cloneLocation(hit.location); } } const location = await inspectOnDisk(paths, slug, configured); // A stall is not remembered: the health state is already its memory. if (!opts.fresh && location.status !== "stalled") { if (memo.size >= MEMO_SWEEP_AT) { for (const [k, v] of memo) { if (now - v.at >= CHANNEL_MEDIA_MEMO_MS) memo.delete(k); } } memo.set(key, { at: now, location }); } return cloneLocation(location); } function cloneLocation(l: ChannelMediaLocation): ChannelMediaLocation { return { ...l, text: { ...l.text } }; } // `data/` on the corpus disk: absent (nothing downloaded yet — readable), a // directory (readable), a link (the retired layout), or something else. async function textState( dataDir: string, ): Promise<"absent" | "dir" | "link" | "other"> { try { const l = await lstat(dataDir); if (l.isSymbolicLink()) return "link"; return l.isDirectory() ? "dir" : "other"; } catch { return "absent"; } } async function inspectOnDisk( paths: ChannelMediaPaths, slug: string, configured: Configured, ): Promise { const dataDir = channelMediaDir(paths, slug); const mediaLink = path.join(paths.channelsDir, slug, MEDIA_LINK_NAME); const { mediaDir } = configured; const text = await textState(dataDir); const textReadable = text === "absent" || text === "dir"; const base = { dataDir, mediaLink }; // THE MARKER FIRST. It is in the channel dir, on the corpus disk, so reading // it costs the drive nothing — and a channel mid-move reads `in-transition` // whatever its drive is doing, which is what a resumed move and every guard // key off. A media move leaves the text readable; a tier migration does not. const marker = await readRelocationMarker(paths, slug); if (marker) { return { ...base, relocated: Boolean(mediaDir ?? configured.dataDir), target: mediaDir ?? marker.target, text: { dir: dataDir, readable: textReadable && !markerHoldsText(marker), }, status: "in-transition", detail: `a media relocation (${marker.direction}) is in progress or was ` + `interrupted at phase "${marker.phase}" — target ${marker.target}`, marker, }; } // THE RETIRED LAYOUT, from the corpus disk alone: a `data` link or a // recorded `dataDir`. Never a call to the far drive — the migration is what // reads it, with the editor stopped. if (text === "link" || configured.dataDir) { let linkTarget: string | undefined; if (text === "link") { try { linkTarget = await readlink(dataDir); } catch { /* unreadable: the config's value, if any, stands */ } } return { ...base, relocated: true, target: configured.dataDir ?? linkTarget, text: { dir: dataDir, readable: false }, status: "legacy", detail: legacyDetail(slug), }; } const textRecord = { dir: dataDir, readable: textReadable }; if (!textReadable) { return { ...base, relocated: Boolean(mediaDir), target: mediaDir, text: textRecord, status: "inconsistent", detail: `${dataDir} is neither a directory nor a symlink`, }; } // THE GATE: a channel whose media is on a location whose drive is not // answering is answered from memory, before the link is looked at and before // the target's stat, which would hold an I/O thread for as long as the drive // takes. Its text stays readable. if (mediaDir) { const stall = stalledLocationForPath(mediaDir); if (stall) return stalledMediaLocation(dataDir, mediaDir, stall); } let link: Awaited> | null = null; try { link = await lstat(mediaLink); } catch { // No media link at all. With no configured target that is a classic // channel — its media are real files in `data//` — the overwhelmingly // common case, and not an error. With one, the link is gone. if (!mediaDir) { return { ...base, relocated: false, text: textRecord, status: "in-place" }; } return { ...base, relocated: true, target: mediaDir, text: textRecord, status: "inconsistent", detail: `config.json records mediaDir ${mediaDir} but ${mediaLink} does not ` + `exist — the symlink is missing`, }; } if (link.isSymbolicLink()) { let linkTarget = ""; try { linkTarget = await readlink(mediaLink); } catch { /* readlink of a link we just lstat'd: treat as unreadable below */ } if (!mediaDir) { return { ...base, relocated: false, target: linkTarget || undefined, text: textRecord, status: "inconsistent", detail: `${mediaLink} is a symlink to ${linkTarget || "(unreadable)"} but ` + `config.json records no mediaDir`, }; } if (path.resolve(linkTarget) !== path.resolve(mediaDir)) { return { ...base, relocated: true, target: mediaDir, text: textRecord, status: "inconsistent", detail: `${mediaLink} points at ${linkTarget || "(unreadable)"} but ` + `config.json records ${mediaDir}`, }; } // The link points at a DEEP path (//media), so an unmounted // root gives ENOENT here. An empty mountpoint can never be mistaken for the // media, which is the whole reason the suffix is fixed. // // THE ONE CALL HERE THAT REACHES THE DRIVE, so it goes through the // watchdog: not made while the location is stalled, and a stat that has // not answered within the budget (`storage.health.budgetMs`, 3 s by // default) marks it stalled and answers `stalled` now. try { const st = await onDrive(mediaDir, () => stat(mediaDir)); if (!st.isDirectory()) { return { ...base, relocated: true, target: mediaDir, text: textRecord, status: "unreachable", detail: `${mediaDir} exists but is not a directory`, }; } } catch (err) { if (isDriveNotAnswering(err)) { return stalledMediaLocation(dataDir, mediaDir, err.health, err.message); } return { ...base, relocated: true, target: mediaDir, text: textRecord, status: "unreachable", detail: `${mediaDir} does not exist (drive not mounted?)`, }; } return { ...base, relocated: true, target: mediaDir, text: textRecord, status: "ok" }; } if (!link.isDirectory()) { return { ...base, relocated: Boolean(mediaDir), target: mediaDir, text: textRecord, status: "inconsistent", detail: `${mediaLink} is neither a directory nor a symlink`, }; } if (mediaDir) { return { ...base, relocated: true, target: mediaDir, text: textRecord, status: "inconsistent", detail: `config.json records mediaDir ${mediaDir} but ${mediaLink} is a real ` + `directory — the media was never moved, or was moved back by hand`, }; } // Tiered in place: `media/` is a real directory on the corpus disk. return { ...base, relocated: false, text: textRecord, status: "in-place" }; } // THE MEDIA GUARD, for a job that opens or writes a BIG file. // "ok" and "in-place" pass; everything else throws. An in-transition or // inconsistent channel is refused for the same reason an unreachable one is: // the caller would otherwise read a half-populated or empty dir as the truth. // A stalled one is refused because the work would block on the drive. // // ALWAYS FRESH: this is the start-of-work guard, and a remembered "ok" from a // few seconds ago is not what a job about to read `data/` should be told. export async function assertChannelMediaReachable( paths: ChannelMediaPaths, slug: string, config?: ChannelMediaConfig | null, ): Promise { const location = await inspectChannelMedia(paths, slug, config, { fresh: true, }); if (location.status === "ok" || location.status === "in-place") { return location; } throw new ChannelMediaUnreachableError(slug, location); } // THE TEXT GUARD (release 17): passes for every channel whose `data/` is a // readable directory on the corpus disk — whatever its MEDIA is doing. A // stalled, unreachable, inconsistent or moving media tier does not hold a // reader of the text: the index and stats builds, the snapshot, the digests, // normalize. Refused: a `legacy` channel (its text is on the far drive), a // `data/` that is not a directory, a tier migration in flight. // // ALWAYS FRESH, for the reason the media guard is. // // THIS CHECK HAS A TWIN. `checkChannelReachable` in // `umtool/report-to-video/cues.mjs` repeats it in plain `.mjs` (refuse a // legacy channel — a `data` link or a recorded `dataDir`; refuse a marker only // when its `scope` is `tier-migration`), because umtool's bins run under bare // node with no `tsx` and cannot import this module. The cue resolver reads // TEXT; on a legacy channel whose drive is not mounted the cues read as // ENOENT, and it would otherwise answer from the published archive — cutting // clips from a snapshot's cues instead of the corpus's. Change the checks here // and change them there. export async function assertChannelTextReadable( paths: ChannelMediaPaths, slug: string, config?: ChannelMediaConfig | null, ): Promise { const location = await inspectChannelMedia(paths, slug, config, { fresh: true, }); if (location.text.readable) return location; const detail = location.status === "legacy" ? (location.detail ?? legacyDetail(slug)) : location.status === "in-transition" ? `its media layout is being migrated (${location.detail ?? "a marker is present"})` : `${location.dataDir} is not a readable directory`; throw new ChannelTextUnreadableError(slug, location, detail); }