commit b7b17bc1036f21f09eafc51d727a2f92d7339a16
parent caac7c6a6c02a33ed03a83d5b7bcb4440fe65ef6
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 1 Oct 2026 17:15:44 -0400
common, editor: every media finalisation tiers, every deleter derefs
The hook after transcodeAudio's rename, before both download-outcome writes,
after a batch yt-dlp run (since its start) and after normalizeLiveChat writes
the cues; the tier link carries the file's mtime and the live-chat freshness
check lstats the raw. cleanAudioFromTranscribed, cleanExtraAudioFormats,
removeWrongFormatAudio, purgeSupersededAutoSubs, backfillReacquire, the
source-container discard, the audio-checked source discard,
fixIncompleteTranscript and the video page's three deleters go through
removeMediaFile / removeVideoDirMedia.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
14 files changed, 133 insertions(+), 19 deletions(-)
diff --git a/common/controller/backfillReacquire.ts b/common/controller/backfillReacquire.ts
@@ -59,8 +59,9 @@
// deletes audio after a transcription. The outcome is the good one: a video
// whose only transcript was YouTube ASR gets our own.
+import { removeMediaFile } from "../lib/mediaTier-server";
import path from "node:path";
-import { readdir, rm } from "node:fs/promises";
+import { readdir } from "node:fs/promises";
import type { Paths } from "../lib/paths";
import { getSettings } from "../lib/settings";
import { diskGate, evaluateDiskGate, getFreeBytes } from "../lib/diskSpace";
@@ -503,7 +504,8 @@ function buildCleanup(
}
for (const name of added) {
- await rm(path.join(videoDir, name), { force: true }).catch((err) => {
+ // Through its link when the hook tiered it (release 17).
+ await removeMediaFile(videoDir, name).catch((err) => {
// Reported, never thrown: this runs in a `finally`, and throwing here
// would replace the real outcome of the item with a cleanup error.
log(`Could not remove ${name} for ${videoId}: ${(err as Error).message}`);
diff --git a/common/controller/cleanAudioFromTranscribed.ts b/common/controller/cleanAudioFromTranscribed.ts
@@ -1,3 +1,4 @@
+import { removeMediaFile } from "../lib/mediaTier-server";
import path from "node:path";
import fs from "fs-extra";
import type { Paths } from "../lib/paths";
@@ -10,7 +11,7 @@ import { computeKeptVideoIds } from "./keptVideos";
import { pruneSavedVideos } from "./pruneSavedVideos";
import { excludedIds, verifyBeforeClean } from "./verifyBeforeClean";
-const { pathExists, readdir, remove } = fs;
+const { pathExists, readdir } = fs;
export type CleanAudioOptions = {
channelSlug: string;
@@ -155,7 +156,8 @@ export async function cleanAudioFromTranscribed({
if (signal?.aborted) break;
if (excluded.has(candidate.id)) continue;
for (const f of candidate.audioFiles) {
- await remove(path.join(candidate.videoDir, f));
+ // Through its link when tiered (release 17): the bytes on the media tier go too.
+ await removeMediaFile(candidate.videoDir, f);
log(`Removed ${candidate.id}/${f}`);
removedFiles++;
}
diff --git a/common/controller/cleanExtraAudioFormats.ts b/common/controller/cleanExtraAudioFormats.ts
@@ -1,3 +1,4 @@
+import { removeMediaFile } from "../lib/mediaTier-server";
import path from "node:path";
import fs from "fs-extra";
import type { Paths } from "../lib/paths";
@@ -72,7 +73,7 @@ export async function cleanExtraAudioFormats({
continue;
}
for (const f of extras) {
- await remove(path.join(videoDir, f));
+ await removeMediaFile(videoDir, f); // derefs a tiered link (release 17)
log(`Removed ${id}/${f}`);
removedFiles++;
}
diff --git a/common/controller/normalizeLiveChat.ts b/common/controller/normalizeLiveChat.ts
@@ -4,8 +4,9 @@
// matches NormalizedTranscript with source: "live_chat".
import path from "node:path";
-import { readFile, stat } from "node:fs/promises";
+import { lstat, readFile, stat } from "node:fs/promises";
import { writeJsonAtomic } from "../lib/jsonFile-server";
+import { tierMediaFile } from "../lib/mediaTier-server";
import { parseLiveChat } from "../lib/liveChat";
import type { Cue } from "../lib/vtt";
import { summarize, type RawMetadata } from "../lib/transcripts-server";
@@ -41,6 +42,20 @@ async function mtimeMs(p: string): Promise<number | null> {
}
}
+// THE RAW REPLAY'S mtime, WITHOUT FOLLOWING A LINK (release 17). The raw file
+// is media: on a tiered channel `transcript.live_chat.json` is a relative link
+// into channels/<slug>/media, possibly on another drive. The tier hook copies
+// the file's mtime onto the link (`lutimes`), so the link answers the same
+// freshness question from the corpus disk — the index build asks it per video
+// and must never reach the media drive to do so.
+async function rawMtimeMs(p: string): Promise<number | null> {
+ try {
+ return (await lstat(p)).mtimeMs;
+ } catch {
+ return null;
+ }
+}
+
// TODO: live_chat files can be hundreds of MB. parseLiveChat already splits
// on "\n", so a streaming readline variant is straightforward if we hit a
// memory wall. For now this matches the readFile pattern used by the
@@ -54,7 +69,7 @@ export async function normalizeLiveChat(
const [metaStatMs, rawStatMs, cuesMs] = await Promise.all([
mtimeMs(metaPath),
- mtimeMs(rawPath),
+ rawMtimeMs(rawPath),
mtimeMs(cuesPath),
]);
@@ -101,6 +116,12 @@ export async function normalizeLiveChat(
opts.log?.(
`Normalized live chat ${opts.channelSlug}/${path.basename(opts.videoDir)} (${cues.length} cues)`,
);
+ // THE MEDIA TIER'S HOOK (release 17): the raw replay is read once, here, and
+ // every other reader uses the cues just written — so it moves into
+ // channels/<slug>/media now (a relative link stays). Never throws.
+ await tierMediaFile(opts.videoDir, LIVE_CHAT_FILENAME, {
+ onLog: opts.log ? (line) => opts.log?.(line.trimEnd()) : undefined,
+ });
return { status: "wrote", cuesPath };
}
@@ -130,7 +151,7 @@ export async function isLiveChatCuesFresh(
const [cuesMs, metaMs, rawMs] = await Promise.all([
mtimeMs(cuesPath),
mtimeMs(metaPath),
- mtimeMs(rawPath),
+ rawMtimeMs(rawPath),
]);
if (cuesMs === null || metaMs === null || rawMs === null) {
return { fresh: false, cuesPath };
diff --git a/common/controller/purgeSupersededAutoSubs.ts b/common/controller/purgeSupersededAutoSubs.ts
@@ -1,3 +1,4 @@
+import { removeMediaFile } from "../lib/mediaTier-server";
import path from "node:path";
import fs from "fs-extra";
import type { Paths } from "../lib/paths";
@@ -9,7 +10,7 @@ import {
listTranscriptVtts,
} from "../lib/videoStatus";
-const { pathExists, readdir, remove } = fs;
+const { pathExists, readdir } = fs;
// Delete the YouTube auto-caption VTTs that our own transcript has superseded —
// the manual counterpart to the `supersededAutoSubs` snapshot bucket, and the
@@ -88,7 +89,7 @@ export async function purgeSupersededAutoSubs({
skipped++;
continue;
}
- await remove(path.join(videoDir, name));
+ await removeMediaFile(videoDir, name); // the one rm of a video-dir entry (release 17)
log(`Removed ${id}/${name}`);
removedFiles++;
removedHere++;
diff --git a/common/controller/removeWrongFormatAudio.ts b/common/controller/removeWrongFormatAudio.ts
@@ -1,3 +1,4 @@
+import { removeMediaFile } from "../lib/mediaTier-server";
import path from "node:path";
import fs from "fs-extra";
import type { Paths } from "../lib/paths";
@@ -5,7 +6,7 @@ import { isDoNotClean } from "../lib/doNotClean-server";
import { audioFilesToRemove } from "../lib/videoStatus";
import { readChannelConfig } from "./channels";
-const { pathExists, readdir, remove } = fs;
+const { pathExists, readdir } = fs;
export type RemoveWrongFormatAudioOptions = {
channelSlug: string;
@@ -70,7 +71,7 @@ export async function removeWrongFormatAudio({
continue;
}
for (const f of wrongFormat) {
- await remove(path.join(videoDir, f));
+ await removeMediaFile(videoDir, f); // derefs a tiered link (release 17)
log(`Removed ${id}/${f}`);
removedFiles++;
}
diff --git a/common/controller/transcode.ts b/common/controller/transcode.ts
@@ -3,6 +3,7 @@ import { rename, rm } from "node:fs/promises";
import { execa } from "execa";
import type { Paths } from "../lib/paths";
import type { AudioFormat } from "../lib/channelConfig";
+import { tierMediaFile } from "../lib/mediaTier-server";
const CODEC_ARGS: Record<AudioFormat, string[]> = {
m4a: ["-c:a", "aac"],
@@ -54,4 +55,11 @@ export async function transcodeAudio(opts: TranscodeAudioOptions): Promise<void>
}
await rename(tmp, out);
opts.onLog(`Wrote ${out}\n`);
+ // THE MEDIA TIER'S HOOK (release 17). The rename above put a real file over
+ // the name — over a tiered LINK, when the channel had one — so the fresh
+ // file moves into channels/<slug>/media (over the stale copy there) and the
+ // link is made again. Every finalising transcode is this function: the app
+ // extraction, the audio-checked download, the video page's Transcode. Never
+ // throws; on a classic channel it leaves the file where it is.
+ await tierMediaFile(opts.videoDir, path.basename(out), { onLog: opts.onLog });
}
diff --git a/common/lib/mediaTier-server.test.ts b/common/lib/mediaTier-server.test.ts
@@ -257,3 +257,16 @@ test("removeVideoDirMedia clears every link's bytes and the video's media dir",
assert.ok(existsSync(path.join(f.videoDir, "transcript.json")));
await rm(f.root, { recursive: true, force: true });
});
+
+test("the link carries the file's mtime, so an lstat answers freshness without the media drive", async () => {
+ const f = await fixture();
+ await mkdir(channelMediaLink(f.paths, f.slug));
+ const file = path.join(f.videoDir, "transcript.live_chat.json");
+ const when = new Date("2026-01-02T03:04:05Z");
+ await utimes(file, when, when);
+ assert.equal(await tierMediaFile(f.videoDir, "transcript.live_chat.json"), "tiered");
+ const l = await lstat(file);
+ assert.ok(l.isSymbolicLink());
+ assert.equal(l.mtimeMs, when.getTime());
+ await rm(f.root, { recursive: true, force: true });
+});
diff --git a/common/lib/mediaTier-server.ts b/common/lib/mediaTier-server.ts
@@ -30,6 +30,7 @@
import path from "node:path";
import {
lstat,
+ lutimes,
mkdir,
readdir,
readlink,
@@ -184,6 +185,11 @@ export async function tierMediaFile(
// replace are two consecutive renames.
await rm(tmpLink, { force: true });
await symlink(tierLinkTarget(id, name), tmpLink);
+ // The FILE's times on the LINK, so an `lstat` of the name answers what a
+ // `stat` of the file did before it was tiered — the live-chat freshness
+ // check (normalizeLiveChat.ts) reads it there, on the corpus disk, instead
+ // of reaching the media drive.
+ await lutimes(tmpLink, st.atime, st.mtime).catch(() => {});
try {
await fsx.rename(file, dest);
moved = true;
diff --git a/common/ytdlp/audioCheckedDownload.ts b/common/ytdlp/audioCheckedDownload.ts
@@ -17,6 +17,7 @@
// the probe) and promotes it to `.good` on success. SIGCONT is always sent in a
// finally to avoid orphaning a suspended child.
+import { removeMediaFile } from "../lib/mediaTier-server";
import { constants as fsConstants } from "node:fs";
import { isPartAudioFile, isRealAudioFile } from "../lib/mediaFiles";
import {
@@ -803,7 +804,7 @@ export async function runAudioCheckedYtdlp(
// container if !keepSourceVideo.
await rm(goodPath(partPathFor(finalFile)), { force: true });
if (!opts.channelConfig.keepSourceVideo) {
- await rm(finalFile, { force: true });
+ await removeMediaFile(path.dirname(finalFile), path.basename(finalFile));
}
return buildOutcome("ok");
}
diff --git a/common/ytdlp/downloadOneManaged.ts b/common/ytdlp/downloadOneManaged.ts
@@ -1,3 +1,5 @@
+import { removeMediaFile } from "../lib/mediaTier-server";
+import { tierVideoDir } from "../lib/mediaTier-server";
import path from "node:path";
import { appendFile, mkdir, readdir, readFile, rm, stat } from "node:fs/promises";
import { createWriteStream, type Dirent, type WriteStream } from "node:fs";
@@ -290,7 +292,7 @@ async function finalizeAppExtraction(opts: {
return;
}
if (!opts.persist) {
- await rm(path.join(opts.videoDir, source), { force: true });
+ await removeMediaFile(opts.videoDir, source);
opts.onLog(`Discarded source container ${source} (audio-only).\n`);
return;
}
@@ -998,6 +1000,9 @@ async function runManagedDownload(
} else {
try {
await mkdir(videoDir, { recursive: true });
+ // THE MEDIA TIER'S HOOK (release 17): what this run finalised moves
+ // into channels/<slug>/media when the channel has one. Never throws.
+ await tierVideoDir(videoDir, { onLog: opts.onLog });
await writeDownloadOutcome(videoDir, record);
} catch (err) {
opts.onLog(
@@ -1626,6 +1631,12 @@ async function writeOutcome(
// exist yet; in that case `mkdir -p` it so the sidecar lands somewhere.
try {
await mkdir(videoDir, { recursive: true });
+ // THE MEDIA TIER'S HOOK (release 17), after reconcileVideoDirs: what this
+ // run finalised — yt-dlp's own `-x` output, the app's extraction, the live
+ // chat — moves into channels/<slug>/media when the channel has one, and a
+ // relative link stays. Never throws: on a classic channel, or a media
+ // drive that is not there, the files stay real.
+ await tierVideoDir(videoDir, { onLog: opts.onLog });
await writeDownloadOutcome(videoDir, record);
} catch (err) {
opts.onLog(
diff --git a/common/ytdlp/runYtdlp.ts b/common/ytdlp/runYtdlp.ts
@@ -1,3 +1,4 @@
+import { tierChannelMedia } from "../lib/mediaTier-server";
import path from "node:path";
import { mkdir, readdir, readFile } from "node:fs/promises";
import { writeFileAtomic } from "../lib/jsonFile-server";
@@ -162,10 +163,45 @@ export type RunYtdlpOpts = {
) => void | Promise<void>;
};
+// The batch modes whose child writes media into `data/<id>/` — yt-dlp's own
+// `-x` output, the batch download loops — and so end with the media tier's
+// hook. Not store-playlist (writes `playlist`) and not download-missing-subs
+// (writes subtitles, which are text).
+const MEDIA_WRITING_MODES: ReadonlySet<RunYtdlpOpts["mode"]> = new Set([
+ "download-from-playlist",
+ "download-missing",
+ "download-one-audio",
+ "retry-bucket",
+ "sync",
+]);
+
+// A video dir's mtime is compared with the run's start; a filesystem with
+// coarse timestamps rounds down, so the window opens this much earlier.
+const TIER_SINCE_SLACK_MS = 2_000;
+
export async function runYtdlp(opts: RunYtdlpOpts): Promise<void> {
if (!opts.channelConfig.url) {
throw new Error("Channel has no `url` configured");
}
+ if (!MEDIA_WRITING_MODES.has(opts.mode)) return runYtdlpMode(opts);
+ // THE MEDIA TIER'S HOOK FOR A BATCH (release 17): after the child returns —
+ // done, failed or cancelled, whatever it finished — every video dir this run
+ // touched has its media moved into channels/<slug>/media when the channel
+ // has one. The per-video managed downloads tier as they go
+ // (downloadOneManaged); this catches what a raw yt-dlp run finalised itself.
+ // Never throws, and costs one lstat on a classic channel.
+ const startedAt = Date.now();
+ try {
+ await runYtdlpMode(opts);
+ } finally {
+ await tierChannelMedia(opts.paths, opts.channelSlug, {
+ since: startedAt - TIER_SINCE_SLACK_MS,
+ onLog: opts.onLog,
+ });
+ }
+}
+
+async function runYtdlpMode(opts: RunYtdlpOpts): Promise<void> {
switch (opts.mode) {
case "store-playlist":
await storePlaylist(opts);
diff --git a/editor/app/channels/[slug]/lib/fixIncompleteTranscript.ts b/editor/app/channels/[slug]/lib/fixIncompleteTranscript.ts
@@ -9,8 +9,9 @@
// audio on disk is itself short, so re-running whisper on it just reproduces the
// short transcript. The fix MUST re-fetch the audio first.
+import { removeMediaFile } from "yt-dlp-transcript-common/lib/mediaTier-server";
import path from "node:path";
-import { readdir, rm } from "node:fs/promises";
+import { readdir } from "node:fs/promises";
import type { ChannelConfig } from "yt-dlp-transcript-common/lib/channelConfig";
import { getPaths, type Paths } from "yt-dlp-transcript-common/lib/paths";
import { getSettings } from "yt-dlp-transcript-common/lib/settings";
@@ -94,7 +95,7 @@ export async function fixIncompleteTranscriptOne(opts: {
// than seeing it as already present.
const entries = await readdir(videoDir).catch(() => [] as string[]);
for (const name of entries.filter(isRealAudioFile)) {
- await rm(path.join(videoDir, name), { force: true });
+ await removeMediaFile(videoDir, name);
onLog(`Removed truncated audio ${name}.`);
}
onLog(`Re-downloading audio for ${videoId}…`);
@@ -150,7 +151,8 @@ export async function clearIncompleteTranscriptOne(opts: {
name === "transcript.cues.json",
);
for (const name of toRemove) {
- await rm(path.join(resolved, name), { force: true });
+ // Through its link when tiered (release 17): the bytes go too.
+ await removeMediaFile(resolved, name);
}
return { removed: toRemove.length };
}
diff --git a/editor/app/channels/[slug]/videos/[id]/videoActions.ts b/editor/app/channels/[slug]/videos/[id]/videoActions.ts
@@ -40,6 +40,10 @@ import { ChannelMediaUnreachableError } from "yt-dlp-transcript-common/lib/chann
import { setExcludedFromTruncatedCheck } from "yt-dlp-transcript-common/lib/excludeTruncatedCheck-server";
import { pruneFailedTranscriptions } from "yt-dlp-transcript-common/controller/failedTranscriptions";
import { transcodeAudio } from "yt-dlp-transcript-common/controller/transcode";
+import {
+ removeMediaFile,
+ removeVideoDirMedia,
+} from "yt-dlp-transcript-common/lib/mediaTier-server";
import { transcribeWithWorker } from "yt-dlp-transcript-common/controller/transcribeOne";
import { findVideoSourceUrl } from "yt-dlp-transcript-common/controller/undownloadedVideos";
import {
@@ -499,7 +503,9 @@ export async function deleteVideoFileAction(
if (!s.isFile()) {
return { ok: false, error: `Not a regular file: ${filename}` };
}
- await rm(target, { force: true });
+ // Through its link when the file is tiered (release 17): the bytes on the
+ // media tier go too, never orphaned behind a removed link.
+ await removeMediaFile(videoDir, path.basename(target));
revalidatePath(`/channels/${slug}/videos/${videoId}`);
requestChannelSnapshot(getPaths(), slug);
return { ok: true };
@@ -565,6 +571,9 @@ export async function deleteOneVideoDir(
error: "Refusing to delete: video path resolved outside the data dir",
};
}
+ // The video's media tier first (release 17): every tiered link's bytes and
+ // its `media/<id>/`, which the recursive rm of the text dir cannot reach.
+ await removeVideoDirMedia(resolved);
await rm(resolved, { recursive: true, force: true });
return { ok: true };
}
@@ -595,7 +604,7 @@ export async function removeAudioFilesForVideo(
wrongFormatOnly: opts.wrongFormatOnly,
});
for (const name of toRemove) {
- await rm(path.join(resolved, name), { force: true });
+ await removeMediaFile(resolved, name); // derefs a tiered link (release 17)
}
return { ok: true, removed: toRemove.length };
}
