commit 0fd12e4122724c390aa0f16b3b9ccec84775ed51
parent 9babe0ee3ff820042c1b64655f8e298ea1a60877
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 1 Oct 2026 16:32:31 -0400
common: the media tier's classifier and its hook (mediaTier.ts, mediaTier-server.ts)
classifyEntry/isTierable/classifyVideoDir by name over mediaFiles' anchored
predicates; tierMediaFile/tierVideoDir/tierChannelMedia (relative per-file
links into channels/<slug>/media, EXDEV-safe, non-recursive per-video mkdir,
never throws into a download) and removeMediaFile/removeVideoDirMedia.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
4 files changed, 804 insertions(+), 0 deletions(-)
diff --git a/common/lib/mediaTier-server.test.ts b/common/lib/mediaTier-server.test.ts
@@ -0,0 +1,259 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ lstat,
+ mkdir,
+ mkdtemp,
+ readFile,
+ readlink,
+ readdir,
+ rm,
+ symlink,
+ utimes,
+ writeFile,
+} from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import {
+ channelMediaLink,
+ relocatedMediaDir,
+ removeMediaFile,
+ removeVideoDirMedia,
+ tierChannelMedia,
+ tierLinkTarget,
+ tierMediaFile,
+ tierVideoDir,
+} from "./mediaTier-server";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test lib/mediaTier-server.test.ts
+//
+// Everything happens inside one mkdtemp; nothing reads a real corpus.
+
+async function fixture() {
+ const root = await mkdtemp(path.join(tmpdir(), "media-tier-"));
+ const channelsDir = path.join(root, "channels");
+ const slug = "chan";
+ const videoDir = path.join(channelsDir, slug, "data", "vid1");
+ await mkdir(videoDir, { recursive: true });
+ await writeFile(path.join(videoDir, "audio.mp3"), "AUDIO");
+ await writeFile(path.join(videoDir, "transcript.json"), "{}");
+ await writeFile(path.join(videoDir, "transcript.live_chat.json"), "CHAT");
+ await writeFile(path.join(videoDir, "source-media.mp4"), "VIDEO");
+ return { root, channelsDir, slug, videoDir, paths: { channelsDir } };
+}
+
+test("names: the link, the relocated root, the relative link target", () => {
+ assert.equal(channelMediaLink({ channelsDir: "/c" }, "s"), "/c/s/media");
+ assert.equal(relocatedMediaDir(" /mnt/p ", "s"), "/mnt/p/s/media");
+ assert.equal(tierLinkTarget("v", "audio.mp3"), "../../media/v/audio.mp3");
+});
+
+test("a classic channel (no media/) leaves every file real", async () => {
+ const f = await fixture();
+ assert.equal(await tierMediaFile(f.videoDir, "audio.mp3"), "left");
+ assert.ok((await lstat(path.join(f.videoDir, "audio.mp3"))).isFile());
+ assert.equal(existsSync(channelMediaLink(f.paths, f.slug)), false);
+ await rm(f.root, { recursive: true, force: true });
+});
+
+test("tiered in place: the bytes move to media/<id>, a relative link stays", async () => {
+ const f = await fixture();
+ await mkdir(channelMediaLink(f.paths, f.slug));
+ assert.equal(await tierMediaFile(f.videoDir, "audio.mp3"), "tiered");
+ const link = path.join(f.videoDir, "audio.mp3");
+ assert.ok((await lstat(link)).isSymbolicLink());
+ assert.equal(await readlink(link), "../../media/vid1/audio.mp3");
+ assert.equal(await readFile(link, "utf8"), "AUDIO");
+ assert.equal(
+ await readFile(path.join(f.channelsDir, f.slug, "media", "vid1", "audio.mp3"), "utf8"),
+ "AUDIO",
+ );
+ // Called again: already a link.
+ assert.equal(await tierMediaFile(f.videoDir, "audio.mp3"), "already");
+ // No temp link left behind.
+ assert.deepEqual(
+ (await readdir(f.videoDir)).filter((n) => n.includes("tierlink")),
+ [],
+ );
+ await rm(f.root, { recursive: true, force: true });
+});
+
+test("a missing name is already; text and source-media are left", async () => {
+ const f = await fixture();
+ await mkdir(channelMediaLink(f.paths, f.slug));
+ assert.equal(await tierMediaFile(f.videoDir, "audio.m4a"), "already");
+ assert.equal(await tierMediaFile(f.videoDir, "transcript.json"), "left");
+ assert.equal(await tierMediaFile(f.videoDir, "source-media.mp4"), "left");
+ assert.ok((await lstat(path.join(f.videoDir, "source-media.mp4"))).isFile());
+ await rm(f.root, { recursive: true, force: true });
+});
+
+test("tierVideoDir tiers the audio and the raw live chat, nothing else", async () => {
+ const f = await fixture();
+ await mkdir(channelMediaLink(f.paths, f.slug));
+ const c = await tierVideoDir(f.videoDir);
+ assert.deepEqual(c, { tiered: 2, left: 0, already: 0 });
+ assert.ok((await lstat(path.join(f.videoDir, "transcript.live_chat.json"))).isSymbolicLink());
+ assert.ok((await lstat(path.join(f.videoDir, "transcript.json"))).isFile());
+ assert.ok((await lstat(path.join(f.videoDir, "source-media.mp4"))).isFile());
+ assert.deepEqual(await tierVideoDir(f.videoDir), { tiered: 0, left: 0, already: 2 });
+ await rm(f.root, { recursive: true, force: true });
+});
+
+test("relocated: the media link points at another root; EXDEV copies, then links", async () => {
+ const f = await fixture();
+ const platter = path.join(f.root, "platter");
+ const target = relocatedMediaDir(platter, f.slug);
+ await mkdir(target, { recursive: true });
+ await symlink(target, channelMediaLink(f.paths, f.slug));
+ let copies = 0;
+ const result = await tierMediaFile(f.videoDir, "audio.mp3", {
+ fs: {
+ rename: async () => {
+ const err = new Error("cross-device link not permitted") as NodeJS.ErrnoException;
+ err.code = "EXDEV";
+ throw err;
+ },
+ copyFileAtomic: async (src, dest) => {
+ copies += 1;
+ await writeFile(dest, await readFile(src));
+ },
+ },
+ });
+ assert.equal(result, "tiered");
+ assert.equal(copies, 1);
+ const link = path.join(f.videoDir, "audio.mp3");
+ assert.ok((await lstat(link)).isSymbolicLink());
+ assert.equal(await readFile(link, "utf8"), "AUDIO");
+ assert.equal(await readFile(path.join(target, "vid1", "audio.mp3"), "utf8"), "AUDIO");
+ await rm(f.root, { recursive: true, force: true });
+});
+
+test("a failed move is undone: the name is a real file again, no link", async () => {
+ const f = await fixture();
+ await mkdir(channelMediaLink(f.paths, f.slug));
+ const logs: string[] = [];
+ const result = await tierMediaFile(f.videoDir, "audio.mp3", {
+ onLog: (s) => logs.push(s),
+ fs: {
+ rename: async () => {
+ const err = new Error("no space left on device") as NodeJS.ErrnoException;
+ err.code = "ENOSPC";
+ throw err;
+ },
+ },
+ });
+ assert.equal(result, "left");
+ assert.ok((await lstat(path.join(f.videoDir, "audio.mp3"))).isFile());
+ assert.match(logs.join(""), /left in place: no space left/);
+ assert.deepEqual(
+ (await readdir(f.videoDir)).filter((n) => n.includes("tierlink")),
+ [],
+ );
+ await rm(f.root, { recursive: true, force: true });
+});
+
+test("a dangling media link (unmounted drive) creates nothing and leaves the file", async () => {
+ const f = await fixture();
+ const platter = path.join(f.root, "platter-unmounted");
+ const target = relocatedMediaDir(platter, f.slug);
+ await symlink(target, channelMediaLink(f.paths, f.slug));
+ assert.equal(await tierMediaFile(f.videoDir, "audio.mp3"), "left");
+ assert.deepEqual(await tierVideoDir(f.videoDir), { tiered: 0, left: 2, already: 0 });
+ assert.deepEqual(await tierChannelMedia(f.paths, f.slug), { tiered: 0, left: 0, already: 0 });
+ // Nothing materialised under the missing mountpoint.
+ assert.equal(existsSync(platter), false);
+ assert.ok((await lstat(path.join(f.videoDir, "audio.mp3"))).isFile());
+ await rm(f.root, { recursive: true, force: true });
+});
+
+test("the per-video mkdir is not recursive: a media root that vanished is not rebuilt", async () => {
+ const f = await fixture();
+ const platter = path.join(f.root, "platter");
+ const target = relocatedMediaDir(platter, f.slug);
+ await mkdir(target, { recursive: true });
+ await symlink(target, channelMediaLink(f.paths, f.slug));
+ const calls: unknown[][] = [];
+ // The drive goes away between the readiness check and the mkdir.
+ const result = await tierMediaFile(f.videoDir, "audio.mp3", {
+ fs: {
+ mkdir: async (...args: unknown[]) => {
+ calls.push(args);
+ await rm(platter, { recursive: true, force: true });
+ return mkdir(args[0] as string);
+ },
+ },
+ });
+ assert.equal(result, "left");
+ assert.equal(calls.length, 1);
+ assert.equal(calls[0].length, 1, "mkdir is called without options");
+ assert.equal(existsSync(platter), false);
+ assert.ok((await lstat(path.join(f.videoDir, "audio.mp3"))).isFile());
+ await rm(f.root, { recursive: true, force: true });
+});
+
+test("tierChannelMedia: createMediaDir makes media/ real; since skips old dirs", async () => {
+ const f = await fixture();
+ const old = path.join(f.channelsDir, f.slug, "data", "vid0");
+ await mkdir(old);
+ await writeFile(path.join(old, "audio.mp3"), "OLD");
+ const past = new Date(Date.now() - 3_600_000);
+ await utimes(old, past, past);
+ const since = Date.now() - 60_000;
+ // Without createMediaDir a classic channel tiers nothing.
+ assert.deepEqual(await tierChannelMedia(f.paths, f.slug, { since }), {
+ tiered: 0,
+ left: 0,
+ already: 0,
+ });
+ const c = await tierChannelMedia(f.paths, f.slug, { since, createMediaDir: true });
+ assert.deepEqual(c, { tiered: 2, left: 0, already: 0 });
+ assert.ok((await lstat(channelMediaLink(f.paths, f.slug))).isDirectory());
+ assert.ok((await lstat(path.join(old, "audio.mp3"))).isFile());
+ // Without since, the old dir too.
+ assert.deepEqual(await tierChannelMedia(f.paths, f.slug), { tiered: 1, left: 0, already: 2 });
+ await rm(f.root, { recursive: true, force: true });
+});
+
+test("removeMediaFile derefs a link: the bytes in media/<id> go, then the link", async () => {
+ const f = await fixture();
+ await mkdir(channelMediaLink(f.paths, f.slug));
+ await tierVideoDir(f.videoDir);
+ const bytes = path.join(f.channelsDir, f.slug, "media", "vid1", "audio.mp3");
+ assert.ok(existsSync(bytes));
+ await removeMediaFile(f.videoDir, "audio.mp3");
+ assert.equal(existsSync(bytes), false);
+ await assert.rejects(lstat(path.join(f.videoDir, "audio.mp3")));
+ // A real file, and a missing one.
+ await removeMediaFile(f.videoDir, "transcript.json");
+ await assert.rejects(lstat(path.join(f.videoDir, "transcript.json")));
+ await removeMediaFile(f.videoDir, "nope");
+ await rm(f.root, { recursive: true, force: true });
+});
+
+test("removeMediaFile never follows a link out of the channel's media/<id>", async () => {
+ const f = await fixture();
+ const outside = path.join(f.root, "precious.mp3");
+ await writeFile(outside, "KEEP");
+ await rm(path.join(f.videoDir, "audio.mp3"));
+ await symlink(outside, path.join(f.videoDir, "audio.mp3"));
+ await removeMediaFile(f.videoDir, "audio.mp3");
+ assert.equal(await readFile(outside, "utf8"), "KEEP");
+ await assert.rejects(lstat(path.join(f.videoDir, "audio.mp3")));
+ await rm(f.root, { recursive: true, force: true });
+});
+
+test("removeVideoDirMedia clears every link's bytes and the video's media dir", async () => {
+ const f = await fixture();
+ await mkdir(channelMediaLink(f.paths, f.slug));
+ await tierVideoDir(f.videoDir);
+ const tierDir = path.join(f.channelsDir, f.slug, "media", "vid1");
+ await writeFile(path.join(tierDir, "metadata.info.json"), "{}"); // a leftover copy
+ await removeVideoDirMedia(f.videoDir);
+ assert.equal(existsSync(tierDir), false);
+ // The text is the caller's to remove with the dir.
+ assert.ok(existsSync(path.join(f.videoDir, "transcript.json")));
+ await rm(f.root, { recursive: true, force: true });
+});
diff --git a/common/lib/mediaTier-server.ts b/common/lib/mediaTier-server.ts
@@ -0,0 +1,346 @@
+// THE MEDIA TIER ON DISK — the hook that moves a finished media file out of
+// `data/<id>/` and leaves a link behind, and the one way to remove one.
+//
+// The layout (plans/release-17.md, "The model (A′)"):
+//
+// channels/<slug>/data/<id>/audio.mp3 -> ../../media/<id>/audio.mp3 (a RELATIVE link)
+// channels/<slug>/media a real directory on the corpus disk (tiered in place),
+// or ONE absolute symlink to <root>/<slug>/media (relocated),
+// or absent (classic: the media is real files in data/<id>/)
+//
+// Readers never change: they open `data/<id>/<name>` and the kernel follows the
+// link. Writers change in exactly two ways, and both are here:
+//
+// - EVERY SITE THAT FINALISES A MEDIA FILE calls the hook (`tierMediaFile`,
+// `tierVideoDir`, `tierChannelMedia`) once it has renamed its temp into
+// place. yt-dlp's postprocessor and this app's transcode both `rename()` a
+// temp OVER the final name, which replaces a link with a real file — the
+// hook then moves that file into the tier, over the stale copy there.
+// - EVERY SITE THAT DELETES ONE calls `removeMediaFile` / `removeVideoDirMedia`.
+// A plain `rm` of `data/<id>/audio.mp3` removes the LINK and orphans the
+// bytes on the media drive, which no sweep would ever find again.
+//
+// THE HOOK NEVER THROWS INTO A DOWNLOAD. A classic channel (no `media/`), a
+// relocated one whose drive is unmounted (the link dangles), a stalled drive,
+// a full disk: the file stays a real file in `data/<id>/`, readers are
+// unaffected, and the next sweep that calls the hook tiers it.
+//
+// lib/, so no controller import (architecture.test.ts).
+
+import path from "node:path";
+import {
+ lstat,
+ mkdir,
+ readdir,
+ readlink,
+ rename,
+ rm,
+ stat,
+ symlink,
+} from "node:fs/promises";
+import type { Paths } from "./paths";
+import { copyFileAtomic } from "./jsonFile-server";
+import { isTierable } from "./mediaTier";
+import { onDrive, stalledLocationForPath } from "./storageHealth";
+
+// The one name a channel's media tier is reached by: `channels/<slug>/media`.
+export const MEDIA_LINK_NAME = "media";
+
+export function channelMediaLink(
+ paths: Pick<Paths, "channelsDir">,
+ slug: string,
+): string {
+ return path.join(paths.channelsDir, slug, MEDIA_LINK_NAME);
+}
+
+// A relocated channel's media root: `<root>/<slug>/media`. The suffix is fixed,
+// not configurable, so an empty mountpoint can never be mistaken for the media
+// and the movers can recognise a target by its shape (the same reason
+// `relocatedDataDir` fixed `<slug>/data`).
+export function relocatedMediaDir(root: string, slug: string): string {
+ return path.join(root.trim(), slug, MEDIA_LINK_NAME);
+}
+
+// The link a tiered file leaves in `data/<id>/`: RELATIVE, so it survives a
+// channel rename, `reconcileVideoDirs`' renames of a video dir (the link and
+// its target move together only when the target's `<id>` is renamed too — open
+// question 3 of the plan) and a move back, which makes `media/` a real
+// directory without touching a single link.
+export function tierLinkTarget(id: string, name: string): string {
+ return path.join("..", "..", MEDIA_LINK_NAME, id, name);
+}
+
+// `data/<id>` → `channels/<slug>/media/<id>`, by shape (the video dir is
+// always `channels/<slug>/data/<id>`).
+function mediaDirOfVideoDir(videoDir: string): string {
+ const id = path.basename(videoDir);
+ return path.join(path.dirname(path.dirname(videoDir)), MEDIA_LINK_NAME, id);
+}
+
+function errCode(err: unknown): string | undefined {
+ return (err as NodeJS.ErrnoException | null)?.code;
+}
+
+// Test seam: the filesystem calls a test needs to fail on cue (an injected
+// `rename` throwing EXDEV, the copy the fallback makes).
+export type TierFs = {
+ rename: typeof rename;
+ copyFileAtomic: (src: string, dest: string) => Promise<void>;
+ // Called with ONE argument — never `{ recursive: true }` (below).
+ mkdir: (dir: string) => Promise<unknown>;
+};
+
+const DEFAULT_FS: TierFs = {
+ rename,
+ copyFileAtomic: (s, d) => copyFileAtomic(s, d),
+ mkdir: (d) => mkdir(d),
+};
+
+export type TierOptions = {
+ onLog?: (line: string) => void;
+ fs?: Partial<TierFs>;
+};
+
+// Whether the channel's media tier can take a file now: `channels/<slug>/media`
+// resolves to a directory, on a drive that is not known to be stalled and that
+// answers within the watchdog's budget. False for a classic channel (no
+// `media`), a dangling link (an unmounted drive), a stalled drive.
+async function mediaTierReady(mediaRoot: string): Promise<boolean> {
+ let target = mediaRoot;
+ try {
+ const l = await lstat(mediaRoot);
+ if (l.isSymbolicLink()) {
+ target = path.resolve(path.dirname(mediaRoot), await readlink(mediaRoot));
+ if (stalledLocationForPath(target)) return false;
+ } else if (!l.isDirectory()) {
+ return false;
+ }
+ } catch {
+ return false;
+ }
+ try {
+ const st = await onDrive(target, () => stat(mediaRoot));
+ return st.isDirectory();
+ } catch {
+ return false;
+ }
+}
+
+export type TierResult = "tiered" | "left" | "already";
+
+// MOVE ONE FINISHED MEDIA FILE INTO THE TIER and leave a relative link behind.
+//
+// "already" — the name is a link already, or there is no such file;
+// "left" — it stays a real file (not tierable, no media tier, the drive is
+// not there or not answering, or a step failed and was undone);
+// "tiered" — the bytes are in `media/<id>/<name>` and `data/<id>/<name>` is
+// the link.
+//
+// `mkdir(media/<id>)` is NOT recursive: `media/` was just seen to be a
+// directory, and a recursive mkdir aimed at a mountpoint that went away in
+// between would build the path on the root filesystem and fill it.
+//
+// The link REPLACES the file atomically (a temp link renamed over the name),
+// so a reader opening the name between the two steps finds the file or the
+// link, never nothing. Same filesystem: the file is renamed into the tier
+// first (instant), the link replaces the now-missing name. Across filesystems
+// (EXDEV — a relocated channel): the bytes are copied atomically into the tier,
+// then the link replaces the original, which is then gone.
+export async function tierMediaFile(
+ videoDir: string,
+ name: string,
+ opts: TierOptions = {},
+): Promise<TierResult> {
+ const fsx: TierFs = { ...DEFAULT_FS, ...opts.fs };
+ const file = path.join(videoDir, name);
+ let st;
+ try {
+ st = await lstat(file);
+ } catch {
+ return "already";
+ }
+ if (st.isSymbolicLink()) return "already";
+ if (!st.isFile() || !isTierable(name)) return "left";
+
+ const mediaRoot = path.dirname(mediaDirOfVideoDir(videoDir));
+ if (!(await mediaTierReady(mediaRoot))) return "left";
+
+ const id = path.basename(videoDir);
+ const destDir = mediaDirOfVideoDir(videoDir);
+ const dest = path.join(destDir, name);
+ try {
+ await fsx.mkdir(destDir);
+ } catch (err) {
+ if (errCode(err) !== "EEXIST") {
+ opts.onLog?.(`[media tier] ${id}/${name} left in place: ${(err as Error).message}\n`);
+ return "left";
+ }
+ }
+
+ const tmpLink = path.join(videoDir, `.${name}.tierlink-${process.pid}`);
+ let moved = false;
+ try {
+ // The link first (dangling until the bytes land), so the move and the
+ // replace are two consecutive renames.
+ await rm(tmpLink, { force: true });
+ await symlink(tierLinkTarget(id, name), tmpLink);
+ try {
+ await fsx.rename(file, dest);
+ moved = true;
+ } catch (err) {
+ if (errCode(err) !== "EXDEV") throw err;
+ await fsx.copyFileAtomic(file, dest);
+ }
+ await rename(tmpLink, file);
+ return "tiered";
+ } catch (err) {
+ await rm(tmpLink, { force: true }).catch(() => {});
+ // Undo a same-filesystem move so the name is never left missing. A failed
+ // cross-filesystem copy left the original where it was.
+ if (moved) {
+ await rename(dest, file).catch(() => {});
+ }
+ opts.onLog?.(`[media tier] ${id}/${name} left in place: ${(err as Error).message}\n`);
+ return "left";
+ }
+}
+
+export type TierCounts = { tiered: number; left: number; already: number };
+
+function emptyCounts(): TierCounts {
+ return { tiered: 0, left: 0, already: 0 };
+}
+
+// Every tierable file in one video dir. A missing dir counts nothing.
+export async function tierVideoDir(
+ videoDir: string,
+ opts: TierOptions = {},
+): Promise<TierCounts> {
+ const counts = emptyCounts();
+ let names: string[];
+ try {
+ names = await readdir(videoDir);
+ } catch {
+ return counts;
+ }
+ for (const name of names) {
+ if (!isTierable(name)) continue;
+ counts[await tierMediaFile(videoDir, name, opts)] += 1;
+ }
+ return counts;
+}
+
+export type TierChannelOptions = TierOptions & {
+ // Only video dirs whose mtime is at or after this instant (ms since epoch) —
+ // a batch download's run start: a dir yt-dlp wrote into had an entry added
+ // or renamed, which moves its mtime.
+ since?: number;
+ // Make `channels/<slug>/media` a real directory when neither a link nor a
+ // directory is there (the mover's preflight tiers a classic channel in
+ // place, same filesystem, before it copies `media/`).
+ createMediaDir?: boolean;
+};
+
+// Every video dir of a channel. A classic channel without `createMediaDir`
+// tiers nothing (every file is "left" — no work is attempted, and none is
+// counted). Never throws.
+export async function tierChannelMedia(
+ paths: Pick<Paths, "channelsDir">,
+ slug: string,
+ opts: TierChannelOptions = {},
+): Promise<TierCounts> {
+ const counts = emptyCounts();
+ const mediaRoot = channelMediaLink(paths, slug);
+ if (opts.createMediaDir) {
+ try {
+ await lstat(mediaRoot);
+ } catch (err) {
+ if (errCode(err) === "ENOENT") {
+ await mkdir(mediaRoot).catch(() => {});
+ }
+ }
+ }
+ if (!(await mediaTierReady(mediaRoot))) return counts;
+ const dataDir = path.join(paths.channelsDir, slug, "data");
+ let ids: string[];
+ try {
+ ids = await readdir(dataDir);
+ } catch {
+ return counts;
+ }
+ for (const id of ids) {
+ const videoDir = path.join(dataDir, id);
+ if (opts.since !== undefined) {
+ try {
+ const st = await stat(videoDir);
+ if (!st.isDirectory() || st.mtimeMs < opts.since) continue;
+ } catch {
+ continue;
+ }
+ }
+ const c = await tierVideoDir(videoDir, opts);
+ counts.tiered += c.tiered;
+ counts.left += c.left;
+ counts.already += c.already;
+ }
+ return counts;
+}
+
+// REMOVE ONE FILE FROM A VIDEO DIR, through its link when it is one: the link's
+// target is removed first (only when it resolves inside this channel's
+// `media/<id>/` — a link pointing anywhere else is removed alone, never
+// followed out), then the name. Missing is not an error.
+//
+// THE ONE `rm` OF A VIDEO-DIR ENTRY. Every deleter in common/ and the editor
+// goes through here (the grep gate in plans/release-17.md), text sidecars
+// included, so no call site has to know which of its names may be a link.
+export async function removeMediaFile(videoDir: string, name: string): Promise<void> {
+ const file = path.join(videoDir, name);
+ let st;
+ try {
+ st = await lstat(file);
+ } catch {
+ return;
+ }
+ if (st.isSymbolicLink()) {
+ const tierDir = mediaDirOfVideoDir(videoDir);
+ let target = "";
+ try {
+ target = path.resolve(videoDir, await readlink(file));
+ } catch {
+ /* unreadable link: removed alone below */
+ }
+ if (target && path.dirname(target) === tierDir) {
+ await rm(target, { force: true });
+ }
+ }
+ await rm(file, { force: true });
+}
+
+// Before a whole video dir is deleted: every link's target in it, then the
+// video's own `media/<id>/` (whatever else is left there — the migration's
+// platter copies before a `--reclaim`). The caller removes the dir itself.
+export async function removeVideoDirMedia(videoDir: string): Promise<void> {
+ let names: string[] = [];
+ try {
+ names = await readdir(videoDir);
+ } catch {
+ /* no dir: only the tier's side to clear */
+ }
+ for (const name of names) {
+ let st;
+ try {
+ st = await lstat(path.join(videoDir, name));
+ } catch {
+ continue;
+ }
+ if (st.isSymbolicLink()) await removeMediaFile(videoDir, name);
+ }
+ const tierDir = mediaDirOfVideoDir(videoDir);
+ try {
+ const l = await lstat(tierDir);
+ if (l.isDirectory()) await rm(tierDir, { recursive: true, force: true });
+ } catch {
+ /* no tier dir for this video */
+ }
+}
diff --git a/common/lib/mediaTier.test.ts b/common/lib/mediaTier.test.ts
@@ -0,0 +1,87 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ CLIPS_DIR,
+ LIVE_CHAT_MEDIA_FILENAME,
+ classifyEntry,
+ classifyVideoDir,
+ isTierable,
+} from "./mediaTier";
+import { LIVE_CHAT_FILENAME } from "./videoStatus";
+import { CLIPS_DIR_NAME } from "./clipWindow";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test lib/mediaTier.test.ts
+
+test("the classifier's two copied names agree with their owners", () => {
+ assert.equal(LIVE_CHAT_MEDIA_FILENAME, LIVE_CHAT_FILENAME);
+ assert.equal(CLIPS_DIR, CLIPS_DIR_NAME);
+});
+
+// THE TABLE. By name, never by size; every row is a name this corpus holds.
+const TABLE: ReadonlyArray<[string, "media" | "text" | "scratch", boolean]> = [
+ // [name, tier, tierable]
+ ["audio.mp3", "media", true],
+ ["audio.m4a", "media", true],
+ ["audio.opus", "media", true],
+ ["audio.mp4", "media", true],
+ ["audio.webm", "media", true],
+ ["source-media.mp4", "media", false],
+ ["source-media.webm", "media", false],
+ ["transcript.live_chat.json", "media", true],
+ // A subtitle named like audio is TEXT (the isRealAudioFile anchoring).
+ ["audio.en-orig.vtt", "text", false],
+ ["audio.en.vtt", "text", false],
+ // Somebody's scratch.
+ ["audio.tmp-2760235.mp3", "scratch", false],
+ ["source-media.temp.mp4", "scratch", false],
+ ["audio.temp.mp3", "scratch", false],
+ ["audio.m4a.part", "scratch", false],
+ ["audio.m4a.part.good", "scratch", false],
+ ["audio.m4a.part.testing", "scratch", false],
+ ["audio.live_chat.json.part-Frag114", "scratch", false],
+ ["transcript.live_chat.json.part", "scratch", false],
+ ["audio.m4a.ytdl", "scratch", false],
+ [".audio.mp3.parakeet", "scratch", false],
+ [".audio.mp3.tierlink-4242", "scratch", false],
+ // The hot text.
+ ["transcript.json", "text", false],
+ ["transcript.en.vtt", "text", false],
+ ["transcript.en-orig.vtt", "text", false],
+ ["transcript.cues.json", "text", false],
+ ["live_chat.cues.json", "text", false],
+ ["metadata.info.json", "text", false],
+ ["metadata.history.json", "text", false],
+ ["diarization.json", "text", false],
+ ["saved-video.json", "text", false],
+ ["clips", "text", false],
+ ["thumbnail.jpg", "text", false],
+];
+
+for (const [name, tier, tierable] of TABLE) {
+ test(`classifyEntry(${name}) = ${tier}, tierable ${tierable}`, () => {
+ assert.equal(classifyEntry(name), tier);
+ assert.equal(isTierable(name), tierable);
+ });
+}
+
+test("classifyVideoDir splits a listing by tier, in order", () => {
+ const out = classifyVideoDir([
+ "metadata.info.json",
+ "audio.mp3",
+ "audio.tmp-1.mp3",
+ "transcript.json",
+ "transcript.live_chat.json",
+ ]);
+ assert.deepEqual(out, {
+ media: ["audio.mp3", "transcript.live_chat.json"],
+ text: ["metadata.info.json", "transcript.json"],
+ scratch: ["audio.tmp-1.mp3"],
+ });
+});
+
+test("every tierable name is media (tierable is narrower)", () => {
+ for (const [name] of TABLE) {
+ if (isTierable(name)) assert.equal(classifyEntry(name), "media", name);
+ }
+});
diff --git a/common/lib/mediaTier.ts b/common/lib/mediaTier.ts
@@ -0,0 +1,112 @@
+// THE MEDIA TIER'S CLASSIFIER — which files in a video dir are big and cold,
+// which are the hot text, and which are somebody's scratch.
+//
+// Release 17 splits a channel's files across two tiers: the TEXT (transcripts,
+// cues, `metadata.info.json`, every sidecar) stays in `channels/<slug>/data/<id>/`
+// on the corpus disk, and the MEDIA (the audio, a persisted source container,
+// the raw live-chat replay) may live under `channels/<slug>/media/<id>/` — a
+// real directory, or one absolute symlink to `<root>/<slug>/media` on another
+// drive — with a RELATIVE per-file link left in `data/<id>/` so every reader
+// keeps opening the same path (`lib/mediaTier-server.ts` holds the links).
+//
+// BY NAME, NEVER BY SIZE. A file's tier is a function of its name alone, over
+// `mediaFiles.ts`'s anchored predicates, so the answer is the same for a file
+// half-written, a file on an unmounted drive (whose size nobody can read) and a
+// file in a listing a bundle tool reads off another machine. Pure: no fs, no
+// import that brings one in — the channel export/import bundle (the slice after
+// release 17) reuses this module, and a client may too.
+//
+// `transcript.live_chat.json` IS MEDIA. It is read once, by
+// `normalizeLiveChat`, which derives the small `live_chat.cues.json` every
+// other reader uses; the raw replay is tens of GB on the big channels. The
+// `clips/` cache is NEVER tiered: it stays on the corpus disk and is evicted by
+// age (`evictClipWindows`).
+
+import {
+ isPartAudioFile,
+ isRealAudioFile,
+ isSourceMediaFile,
+} from "./mediaFiles";
+
+// The raw live-chat replay's name. `videoStatus.ts` exports the same string as
+// `LIVE_CHAT_FILENAME`, but that module imports `node:fs`; mediaTier.test.ts
+// pins that the two agree.
+export const LIVE_CHAT_MEDIA_FILENAME = "transcript.live_chat.json";
+
+// The clip-window cache dir (`clipWindow.ts`'s `CLIPS_DIR_NAME`, pinned by the
+// test for the same reason). Never tiered, never classified as media.
+export const CLIPS_DIR = "clips";
+
+export type MediaTierKind = "media" | "text" | "scratch";
+
+// Somebody's in-flight bytes — a downloader's partial, a transcoder's temp, a
+// transcriber's window dir. Never tiered (the writer is about to rename it, or
+// to resume it), never counted as text, and never carried by a copy that
+// rebuilds a `data/` (the migration lists text and scratch separately so a
+// verify can say which is which).
+const SCRATCH_PATTERNS: ReadonlyArray<RegExp> = [
+ // This app's transcode temp: `audio.tmp-<pid>.<ext>` (controller/transcode.ts).
+ /^audio\.tmp-\d+\./,
+ // parakeet's per-file window scratch dir: `.audio.<ext>.parakeet`.
+ /^\.audio\..*\.parakeet$/,
+ // yt-dlp's fragment downloads: `<name>.part-Frag<n>`.
+ /\.part-Frag\d+$/,
+ // yt-dlp's postprocessor temp: `audio.temp.mp3`, `source-media.temp.mp4`.
+ /\.temp\./,
+ // The audio check's snapshots of a partial: `audio.m4a.part.good`, `.part.testing`.
+ /\.part\.(good|testing)$/,
+ // Any other downloader partial (`transcript.live_chat.json.part`) and
+ // yt-dlp's resume-state file beside one (`audio.m4a.ytdl`).
+ /\.part$/,
+ /\.ytdl$/,
+ // The media-tier hook's own temp link (`lib/mediaTier-server.ts`).
+ /\.tierlink-\d+$/,
+];
+
+export function isScratchEntry(name: string): boolean {
+ if (isPartAudioFile(name)) return true;
+ return SCRATCH_PATTERNS.some((re) => re.test(name));
+}
+
+// Which tier a video-dir entry belongs to. Scratch first: `source-media.temp.mp4`
+// and `audio.tmp-2760235.mp3` are already rejected by the anchored media
+// predicates, and a scratch rule that matched a finalized name would be a bug
+// the test table catches.
+export function classifyEntry(name: string): MediaTierKind {
+ if (name === CLIPS_DIR) return "text";
+ if (isScratchEntry(name)) return "scratch";
+ if (
+ isRealAudioFile(name) ||
+ isSourceMediaFile(name) ||
+ name === LIVE_CHAT_MEDIA_FILENAME
+ ) {
+ return "media";
+ }
+ return "text";
+}
+
+// What the hook actually moves into the media tier: NARROWER than "media".
+//
+// - `source-media.*` stays a real file in `data/<id>/`: `persistSourceVideo`
+// `rename`s it into the saved-video store, and a rename of a LINK would move
+// the link, leaving the bytes behind in `media/` and a dangling pointer in
+// the store. The store is already its own per-object tier.
+// - `audio.*.part` stays real: it is yt-dlp's resumable partial, which yt-dlp
+// appends to and renames.
+export function isTierable(name: string): boolean {
+ return isRealAudioFile(name) || name === LIVE_CHAT_MEDIA_FILENAME;
+}
+
+export type ClassifiedVideoDir = {
+ media: string[];
+ text: string[];
+ scratch: string[];
+};
+
+// One video dir's entries (a `readdir`'s names), by tier, each list in the
+// order given.
+export function classifyVideoDir(entries: Iterable<string>): ClassifiedVideoDir {
+ const out: ClassifiedVideoDir = { media: [], text: [], scratch: [] };
+ for (const name of entries) out[classifyEntry(name)].push(name);
+ return out;
+}
