// THE MEDIA TIER'S CLASSIFIER — which files in a video dir are big and cold, // which are the hot text, and which are somebody's scratch. // // Release 17 splits a channel's files across two tiers: the TEXT (transcripts, // cues, `metadata.info.json`, every sidecar) stays in `channels//data//` // on the corpus disk, and the MEDIA (the audio, a persisted source container, // the raw live-chat replay) may live under `channels//media//` — a // real directory, or one absolute symlink to `//media` on another // drive — with a RELATIVE per-file link left in `data//` so every reader // keeps opening the same path (`lib/mediaTier-server.ts` holds the links). // // BY NAME, NEVER BY SIZE. A file's tier is a function of its name alone, over // `mediaFiles.ts`'s anchored predicates, so the answer is the same for a file // half-written, a file on an unmounted drive (whose size nobody can read) and a // file in a listing a bundle tool reads off another machine. Pure: no fs, no // import that brings one in — the channel export/import bundle (the slice after // release 17) reuses this module, and a client may too. // // `transcript.live_chat.json` IS MEDIA. It is read once, by // `normalizeLiveChat`, which derives the small `live_chat.cues.json` every // other reader uses; the raw replay is tens of GB on the big channels. The // `clips/` cache is NEVER tiered: it stays on the corpus disk and is evicted by // age (`evictClipWindows`). import { isPartAudioFile, isRealAudioFile, isSourceMediaFile, } from "./mediaFiles"; // The raw live-chat replay's name. `videoStatus.ts` exports the same string as // `LIVE_CHAT_FILENAME`, but that module imports `node:fs`; mediaTier.test.ts // pins that the two agree. export const LIVE_CHAT_MEDIA_FILENAME = "transcript.live_chat.json"; // The clip-window cache dir (`clipWindow.ts`'s `CLIPS_DIR_NAME`, pinned by the // test for the same reason). Never tiered, never classified as media. export const CLIPS_DIR = "clips"; export type MediaTierKind = "media" | "text" | "scratch"; // Somebody's in-flight bytes — a downloader's partial, a transcoder's temp, a // transcriber's window dir. Never tiered (the writer is about to rename it, or // to resume it), never counted as text, and never carried by a copy that // rebuilds a `data/` (the migration lists text and scratch separately so a // verify can say which is which). const SCRATCH_PATTERNS: ReadonlyArray = [ // This app's transcode temp: `audio.tmp-.` (controller/transcode.ts). /^audio\.tmp-\d+\./, // parakeet's per-file window scratch dir: `.audio..parakeet`. /^\.audio\..*\.parakeet$/, // yt-dlp's fragment downloads: `.part-Frag`. /\.part-Frag\d+$/, // yt-dlp's postprocessor temp: `audio.temp.mp3`, `source-media.temp.mp4`. /\.temp\./, // The audio check's snapshots of a partial: `audio.m4a.part.good`, `.part.testing`. /\.part\.(good|testing)$/, // Any other downloader partial (`transcript.live_chat.json.part`) and // yt-dlp's resume-state file beside one (`audio.m4a.ytdl`). /\.part$/, /\.ytdl$/, // The media-tier hook's own temps (`lib/mediaTier-server.ts`): the link // beside the name, and the bytes being placed in `media//`. /\.tierlink-\d+$/, /\.tiering-\d+$/, // Syncthing's in-flight temp (`.syncthing.audio.mp3.tmp`): a transfer the // sync tool never finished. /^\.syncthing\..*\.tmp$/, ]; export function isScratchEntry(name: string): boolean { if (isPartAudioFile(name)) return true; return SCRATCH_PATTERNS.some((re) => re.test(name)); } // Which tier a video-dir entry belongs to. Scratch first: `source-media.temp.mp4` // and `audio.tmp-2760235.mp3` are already rejected by the anchored media // predicates, and a scratch rule that matched a finalized name would be a bug // the test table catches. export function classifyEntry(name: string): MediaTierKind { if (name === CLIPS_DIR) return "text"; if (isScratchEntry(name)) return "scratch"; if ( isRealAudioFile(name) || isSourceMediaFile(name) || name === LIVE_CHAT_MEDIA_FILENAME ) { return "media"; } return "text"; } // What the hook actually moves into the media tier: NARROWER than "media". // // - `source-media.*` stays a real file in `data//`: `persistSourceVideo` // `rename`s it into the saved-video store, and a rename of a LINK would move // the link, leaving the bytes behind in `media/` and a dangling pointer in // the store. The store is already its own per-object tier. // - `audio.*.part` stays real: it is yt-dlp's resumable partial, which yt-dlp // appends to and renames. export function isTierable(name: string): boolean { return isRealAudioFile(name) || name === LIVE_CHAT_MEDIA_FILENAME; } export type ClassifiedVideoDir = { media: string[]; text: string[]; scratch: string[]; }; // One video dir's entries (a `readdir`'s names), by tier, each list in the // order given. export function classifyVideoDir(entries: Iterable): ClassifiedVideoDir { const out: ClassifiedVideoDir = { media: [], text: [], scratch: [] }; for (const name of entries) out[classifyEntry(name)].push(name); return out; }