commit a4b130153366ecedd86cf171d992fb6b26d620a2
parent d4ed8093566f61e5d4e65e69cdde4ef402aa49d2
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 25 Jun 2026 00:58:32 -0400
Phase 3: saved-video store (separate dir/disk) + retention pruning
Move persisted source videos out of the per-video data dir into a separate
saved-video store, leaving a small saved-video.json pointer behind. This keeps
the main data volume holding only audio + transcripts while the large source
videos can live on another disk.
Store layout & config:
- paths.savedVideosDir (env SAVED_VIDEOS_DIR, default <transcripts>/saved-videos)
- ChannelConfig.savedVideosDir per-channel override
- containers land under <root>/<slug>/<videoId>/source-media.<ext>
New common/lib/savedVideo.ts (pure) + savedVideo-server.ts:
- SavedVideoPointer { storedAt, dir, file, bytes, keepReason?, sha256? }
- savedVideoDir/savedVideoRoot/savedVideoPath resolvers + parse
- persistSourceVideo (cross-device-safe move + pointer), resolveSavedVideo,
unpersistSavedVideo (reverse), dropSavedVideo (retention delete), loadSavedVideo
- moveFileCrossDevice: rename within a disk, copy-to-temp + atomic rename +
unlink across an EXDEV boundary
Wiring:
- downloadOneManaged.finalizeAppExtraction now MOVES a persisted container into
the store and writes the pointer (best-effort: a failed move leaves the
container in the data dir as source-media.<ext>; never fails the download).
- transcribeOne.resolveAudioFile falls back to the stored container via the
pointer, returned as a path relative to videoDir so the local engine (cwd) and
the remote uploader (path.join) both read it correctly.
Retention prune (resolves the phase-2 "source media not auto-pruned" caveat):
- persistencePlan gains a stable `category` (keep-latest|pin|override|none),
recorded on the pointer as keepReason.
- new common/controller/pruneSavedVideos.ts: evicts only keep-latest containers
that rolled out of the window; never override (manual archive) or pin/
do-not-clean (irreplaceable) ones. Wired into cleanAudioFromTranscribed so the
existing Clean-audio button also prunes (CleanAudioResult gains
prunedSavedVideos/prunedBytes).
Disk: checkDiskSpaceFor(dir, settings) generalizes the floor check to any
filesystem (the store), with checkDiskSpace delegating for transcriptsDir.
Tests: persistencePlan category assertions; savedVideo store
persist/resolve/unpersist/drop; pruneSavedVideos window-eviction vs.
override/pin/do-not-clean protection. 24 unit tests pass; common + editor
typecheck clean.
Deferred to phase 5: audio-check + persistence; saved-store counts in the
channel snapshot.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Diffstat:
14 files changed, 724 insertions(+), 27 deletions(-)
diff --git a/common/controller/cleanAudioFromTranscribed.ts b/common/controller/cleanAudioFromTranscribed.ts
@@ -4,6 +4,7 @@ import type { Paths } from "../lib/paths";
import { isDoNotClean } from "../lib/doNotClean-server";
import { readChannelConfig } from "./channels";
import { computeKeptVideoIds } from "./keptVideos";
+import { pruneSavedVideos } from "./pruneSavedVideos";
const { pathExists, readdir, remove } = fs;
@@ -19,6 +20,10 @@ export type CleanAudioResult = {
cleanedDirs: number;
removedFiles: number;
skipped: number;
+ // Saved-store containers evicted because they rolled out of the keep-latest
+ // window (the retention prune runs alongside the audio sweep).
+ prunedSavedVideos: number;
+ prunedBytes: number;
};
export async function cleanAudioFromTranscribed({
@@ -31,7 +36,14 @@ export async function cleanAudioFromTranscribed({
const dataDir = path.join(paths.channelsDir, channelSlug, "data");
if (!(await pathExists(dataDir))) {
log(`No data directory for ${channelSlug}`);
- return { inspected: 0, cleanedDirs: 0, removedFiles: 0, skipped: 0 };
+ return {
+ inspected: 0,
+ cleanedDirs: 0,
+ removedFiles: 0,
+ skipped: 0,
+ prunedSavedVideos: 0,
+ prunedBytes: 0,
+ };
}
const dirs = await readdir(dataDir);
@@ -83,5 +95,17 @@ export async function cleanAudioFromTranscribed({
log(
`Cleaned ${removedFiles} audio file(s) from ${cleanedDirs} of ${dirs.length} video dir(s).${skippedNote}`,
);
- return { inspected: dirs.length, cleanedDirs, removedFiles, skipped };
+
+ // Retention prune of the saved-video store: evict any keep-latest containers
+ // that have rolled out of the window (pinned/manually-archived ones survive).
+ const prune = await pruneSavedVideos({ channelSlug, paths, onLog: log, signal });
+
+ return {
+ inspected: dirs.length,
+ cleanedDirs,
+ removedFiles,
+ skipped,
+ prunedSavedVideos: prune.pruned,
+ prunedBytes: prune.bytesFreed,
+ };
}
diff --git a/common/controller/pruneSavedVideos.test.ts b/common/controller/pruneSavedVideos.test.ts
@@ -0,0 +1,107 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import { persistSourceVideo, loadSavedVideo } from "../lib/savedVideo-server";
+import { setDoNotClean } from "../lib/doNotClean-server";
+import { pruneSavedVideos } from "./pruneSavedVideos";
+import type { SavedVideoKeepReason } from "../lib/savedVideo";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/pruneSavedVideos.test.ts
+
+async function withPaths(fn: (paths: Paths) => Promise<void>): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-prune-"));
+ const paths = {
+ channelsDir: path.join(dir, "channels"),
+ savedVideosDir: path.join(dir, "saved"),
+ } as Paths;
+ try {
+ await fn(paths);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+// Seed a data/<id> dir with an upload date + a persisted source container.
+async function seedSaved(
+ paths: Paths,
+ slug: string,
+ id: string,
+ uploadDate: string,
+ keepReason: SavedVideoKeepReason,
+): Promise<void> {
+ const videoDir = path.join(paths.channelsDir, slug, "data", id);
+ await mkdir(videoDir, { recursive: true });
+ await writeFile(
+ path.join(videoDir, "metadata.info.json"),
+ JSON.stringify({ id, upload_date: uploadDate }),
+ );
+ await writeFile(path.join(videoDir, "source-media.mp4"), `${id}-bytes`);
+ await persistSourceVideo({
+ videoDir,
+ sourceFilename: "source-media.mp4",
+ storeDir: path.join(paths.savedVideosDir, slug, id),
+ keepReason,
+ });
+}
+
+async function writeConfig(
+ paths: Paths,
+ slug: string,
+ keepLatest: number,
+): Promise<void> {
+ const dir = path.join(paths.channelsDir, slug);
+ await mkdir(dir, { recursive: true });
+ await writeFile(
+ path.join(dir, "config.json"),
+ JSON.stringify({ handling: "transcribe", keepLatest }),
+ );
+}
+
+test("prunes only keep-latest containers that rolled out of the window", async () => {
+ await withPaths(async (paths) => {
+ await writeConfig(paths, "ch", 2);
+ // Newest two (by date): c, b -> kept. a is the rolled-out keep-latest one.
+ await seedSaved(paths, "ch", "a", "20240101", "keep-latest"); // evict
+ await seedSaved(paths, "ch", "b", "20240201", "keep-latest"); // in window
+ await seedSaved(paths, "ch", "c", "20240301", "keep-latest"); // in window
+
+ const res = await pruneSavedVideos({ channelSlug: "ch", paths });
+ assert.equal(res.scanned, 3);
+ assert.equal(res.pruned, 1);
+ assert.equal(await loadSavedVideo(path.join(paths.channelsDir, "ch", "data", "a")), null);
+ assert.notEqual(await loadSavedVideo(path.join(paths.channelsDir, "ch", "data", "b")), null);
+ assert.notEqual(await loadSavedVideo(path.join(paths.channelsDir, "ch", "data", "c")), null);
+ });
+});
+
+test("never evicts override or pinned containers, even out of window", async () => {
+ await withPaths(async (paths) => {
+ await writeConfig(paths, "ch", 1);
+ await seedSaved(paths, "ch", "newest", "20240301", "keep-latest");
+ // Out-of-window manual archive -> kept.
+ await seedSaved(paths, "ch", "archived", "20240101", "override");
+ // Out-of-window pin-category container -> kept.
+ await seedSaved(paths, "ch", "pinptr", "20240102", "pin");
+ // Out-of-window keep-latest, but separately do-not-clean pinned -> kept.
+ await seedSaved(paths, "ch", "deleted", "20240103", "keep-latest");
+ await setDoNotClean(
+ path.join(paths.channelsDir, "ch", "data", "deleted"),
+ true,
+ "deleted from source",
+ );
+
+ const res = await pruneSavedVideos({ channelSlug: "ch", paths });
+ assert.equal(res.pruned, 0);
+ for (const id of ["archived", "pinptr", "deleted"]) {
+ assert.notEqual(
+ await loadSavedVideo(path.join(paths.channelsDir, "ch", "data", id)),
+ null,
+ `${id} should be kept`,
+ );
+ }
+ });
+});
diff --git a/common/controller/pruneSavedVideos.ts b/common/controller/pruneSavedVideos.ts
@@ -0,0 +1,79 @@
+import path from "node:path";
+import fs from "fs-extra";
+import type { Paths } from "../lib/paths";
+import { isDoNotClean } from "../lib/doNotClean-server";
+import { readChannelConfig } from "./channels";
+import { computeKeptVideoIds } from "./keptVideos";
+import { dropSavedVideo, loadSavedVideo } from "../lib/savedVideo-server";
+
+const { pathExists, readdir } = fs;
+
+export type PruneSavedVideosResult = {
+ // Dirs carrying a saved-video pointer that were considered.
+ scanned: number;
+ // Window-persisted containers dropped (rolled out, unpinned).
+ pruned: number;
+ bytesFreed: number;
+};
+
+// Retention prune of the saved-video store. A persisted source video is removed
+// from the store only when ALL of these hold:
+// - it was persisted by the rolling keep-latest rule (pointer keepReason
+// "keep-latest") — NOT a manual archive ("override") or a pin ("pin"),
+// - its id has rolled OUT of the channel's current keep-latest window, and
+// - it is not pinned via do-not-clean (a kept video later deleted-from-source
+// is pinned and must survive forever).
+// This is what bounds the store's growth as new uploads displace old ones, and it
+// is the Phase-3 resolution of the Phase-2 "source media not auto-pruned" caveat.
+// Manually-archived and pinned containers are never auto-evicted.
+export async function pruneSavedVideos({
+ channelSlug,
+ paths,
+ onLog,
+ signal,
+}: {
+ channelSlug: string;
+ paths: Paths;
+ onLog?: (msg: string) => void;
+ signal?: AbortSignal;
+}): Promise<PruneSavedVideosResult> {
+ const log = onLog ?? ((m: string) => console.log(m));
+ const dataDir = path.join(paths.channelsDir, channelSlug, "data");
+ if (!(await pathExists(dataDir))) {
+ return { scanned: 0, pruned: 0, bytesFreed: 0 };
+ }
+ const config = await readChannelConfig(paths, channelSlug);
+ const keptIds = await computeKeptVideoIds({
+ paths,
+ channelSlug,
+ keepLatest: config?.keepLatest ?? 0,
+ });
+ const dirs = await readdir(dataDir);
+
+ let scanned = 0;
+ let pruned = 0;
+ let bytesFreed = 0;
+ for (const id of dirs) {
+ if (signal?.aborted) break;
+ const videoDir = path.join(dataDir, id);
+ const pointer = await loadSavedVideo(videoDir);
+ if (!pointer) continue;
+ scanned++;
+ // Only the rolling keep-latest containers are eligible for auto-eviction.
+ if (pointer.keepReason !== "keep-latest") continue;
+ if (keptIds.has(id)) continue; // still inside the window
+ if (await isDoNotClean(videoDir)) continue; // pinned -> keep forever
+ const bytes = await dropSavedVideo(videoDir);
+ bytesFreed += bytes;
+ pruned++;
+ log(
+ `Pruned saved video ${id} (rolled out of keep-latest window; freed ${bytes} bytes)`,
+ );
+ }
+ if (pruned > 0) {
+ log(
+ `Pruned ${pruned} saved video(s) from the store (freed ${bytesFreed} bytes).`,
+ );
+ }
+ return { scanned, pruned, bytesFreed };
+}
diff --git a/common/controller/transcribeOne.ts b/common/controller/transcribeOne.ts
@@ -11,6 +11,7 @@ import { normalizeTranscript } from "./normalizeTranscript";
import { pingRemoteHealth, transcribeViaRemote } from "./remoteTranscribe";
import { TranscribeError } from "./transcribeError";
import { findSourceMedia, isRealAudioFile } from "../lib/videoStatus";
+import { resolveSavedVideo } from "../lib/savedVideo-server";
import { writeTranscribeOutcome } from "../lib/transcribeOutcome-server";
const { pathExists, readdir, rename, writeFile } = fs;
@@ -38,6 +39,12 @@ async function resolveAudioFile(
// redownload-to-archive that only fetched the container, still transcribable.
const container = findSourceMedia(entries);
if (container) return container;
+ // The container may have been moved into the saved-video store (Phase 3),
+ // leaving a saved-video.json pointer. Resolve it and return a path relative to
+ // videoDir so the engine (cwd = videoDir) and the remote uploader
+ // (path.join(videoDir, …)) both read the stored file correctly.
+ const saved = await resolveSavedVideo(videoDir);
+ if (saved) return path.relative(videoDir, saved);
return null;
}
diff --git a/common/lib/channelConfig.ts b/common/lib/channelConfig.ts
@@ -52,6 +52,12 @@ export type ChannelConfig = {
// keep-latest persistence rule forces "app" for the videos it persists,
// regardless of this setting.
extractionMode?: ExtractionMode;
+ // Per-channel override for the saved-video store root (global default is
+ // paths.savedVideosDir / SAVED_VIDEOS_DIR). When set, this channel's persisted
+ // source videos live under <savedVideosDir>/<slug>/<videoId>/. Lets a single
+ // channel's large videos land on a different disk than the rest. Resolved by
+ // savedVideoDir() in common/lib/savedVideo.ts. Empty/whitespace = use global.
+ savedVideosDir?: string;
ytdlpExtraArgs?: string[];
subLangs?: string;
lastSyncedAt?: string;
@@ -158,6 +164,9 @@ export function parseChannelConfig(raw: unknown): ChannelConfig | null {
if (r.extractionMode === "ytdlp" || r.extractionMode === "app") {
config.extractionMode = r.extractionMode;
}
+ if (typeof r.savedVideosDir === "string" && r.savedVideosDir.trim() !== "") {
+ config.savedVideosDir = r.savedVideosDir.trim();
+ }
if (
Array.isArray(r.ytdlpExtraArgs) &&
r.ytdlpExtraArgs.every((x) => typeof x === "string")
diff --git a/common/lib/diskSpace.ts b/common/lib/diskSpace.ts
@@ -29,26 +29,33 @@ export type DiskSpaceStatus = {
ok: boolean;
};
-// Measure free space on the transcripts data directory (where all downloads
-// land) and compare it against the configured floor. When the gate is disabled
-// (minFreeDiskGB === 0) this always reports ok.
-export async function checkDiskSpace(
- paths: Paths,
+// Compare free space on the filesystem holding `dir` against the configured
+// floor. When the gate is disabled (minFreeDiskGB === 0) this always reports ok
+// and skips the statfs syscall entirely (it runs before every video in a batch
+// and on every active-jobs poll, and a disabled gate hides the indicator anyway).
+export async function checkDiskSpaceFor(
+ dir: string,
settings: SiteSettings,
): Promise<DiskSpaceStatus> {
const enabled = settings.minFreeDiskGB > 0;
const thresholdBytes = settings.minFreeDiskGB * BYTES_PER_GB;
- // Skip the statfs syscall entirely when the gate is off — this runs before
- // every video in a batch and on every active-jobs poll, and a disabled gate
- // hides the indicator anyway.
if (!enabled) {
- return { enabled, freeBytes: Number.POSITIVE_INFINITY, thresholdBytes, ok: true };
+ return {
+ enabled,
+ freeBytes: Number.POSITIVE_INFINITY,
+ thresholdBytes,
+ ok: true,
+ };
}
- const freeBytes = await getFreeBytes(paths.transcriptsDir);
- return {
- enabled,
- freeBytes,
- thresholdBytes,
- ok: freeBytes >= thresholdBytes,
- };
+ const freeBytes = await getFreeBytes(dir);
+ return { enabled, freeBytes, thresholdBytes, ok: freeBytes >= thresholdBytes };
+}
+
+// Measure free space on the transcripts data directory (where all downloads
+// land) and compare it against the configured floor.
+export async function checkDiskSpace(
+ paths: Paths,
+ settings: SiteSettings,
+): Promise<DiskSpaceStatus> {
+ return checkDiskSpaceFor(paths.transcriptsDir, settings);
}
diff --git a/common/lib/paths.ts b/common/lib/paths.ts
@@ -6,6 +6,14 @@ export type Paths = {
monorepoRoot: string;
transcriptsDir: string;
channelsDir: string;
+ // Root of the saved-video store: persisted source-video containers (the
+ // keep-latest persistence rule) are MOVED out of the per-video data dir into
+ // savedVideosDir/<slug>/<videoId>/, leaving only a small saved-video.json
+ // pointer in the data dir. Defaults under transcriptsDir but is overridable
+ // via SAVED_VIDEOS_DIR so the (large) source videos can live on a separate
+ // disk. A channel may further override the root via ChannelConfig.savedVideosDir.
+ // See common/lib/savedVideo.ts.
+ savedVideosDir: string;
// Per-site config lives under sitesDir/<siteId>/site.json (+ chart-templates.json).
// See common/lib/site.ts. A "site" is a selection + presentation layer over the
// single global channel pool; channel downloads are never duplicated per site.
@@ -98,6 +106,8 @@ export function getPaths(): Paths {
monorepoRoot,
transcriptsDir,
channelsDir: path.join(transcriptsDir, "channels"),
+ savedVideosDir:
+ process.env.SAVED_VIDEOS_DIR ?? path.join(transcriptsDir, "saved-videos"),
sitesDir,
homepageDir,
homepageConfigFile: path.join(homepageDir, "homepage.json"),
diff --git a/common/lib/savedVideo-server.ts b/common/lib/savedVideo-server.ts
@@ -0,0 +1,142 @@
+import path from "node:path";
+import {
+ copyFile,
+ mkdir,
+ readFile,
+ rename,
+ rm,
+ stat,
+ writeFile,
+} from "node:fs/promises";
+import {
+ SAVED_VIDEO_POINTER_FILENAME,
+ parseSavedVideoPointer,
+ savedVideoPath,
+ type SavedVideoKeepReason,
+ type SavedVideoPointer,
+} from "./savedVideo";
+
+// Filesystem side of the saved-video store. See savedVideo.ts for the layout.
+
+export function savedVideoPointerPath(videoDir: string): string {
+ return path.join(videoDir, SAVED_VIDEO_POINTER_FILENAME);
+}
+
+export async function loadSavedVideo(
+ videoDir: string,
+): Promise<SavedVideoPointer | null> {
+ try {
+ const raw = await readFile(savedVideoPointerPath(videoDir), "utf8");
+ return parseSavedVideoPointer(JSON.parse(raw));
+ } catch {
+ return null;
+ }
+}
+
+export async function isSavedVideo(videoDir: string): Promise<boolean> {
+ return (await loadSavedVideo(videoDir)) !== null;
+}
+
+async function writePointer(
+ videoDir: string,
+ pointer: SavedVideoPointer,
+): Promise<void> {
+ const file = savedVideoPointerPath(videoDir);
+ const tmp = `${file}.tmp-${process.pid}`;
+ await writeFile(tmp, JSON.stringify(pointer, null, 2) + "\n");
+ await rename(tmp, file);
+}
+
+// Move a file, crossing device boundaries safely. A plain rename() works within
+// one filesystem; EXDEV (the store is on a different disk) falls back to a
+// copy-to-temp + atomic rename + unlink so a crash mid-copy never leaves a
+// partial file under the final name.
+async function moveFileCrossDevice(src: string, dest: string): Promise<void> {
+ await mkdir(path.dirname(dest), { recursive: true });
+ try {
+ await rename(src, dest);
+ return;
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code !== "EXDEV") throw err;
+ }
+ const tmp = `${dest}.tmp-${process.pid}`;
+ await copyFile(src, tmp);
+ await rename(tmp, dest);
+ await rm(src, { force: true });
+}
+
+// Resolve a usable absolute path to a video's persisted source container, or
+// null when there's no pointer or the stored file has gone missing.
+export async function resolveSavedVideo(
+ videoDir: string,
+): Promise<string | null> {
+ const pointer = await loadSavedVideo(videoDir);
+ if (!pointer) return null;
+ const file = savedVideoPath(pointer);
+ try {
+ await stat(file);
+ return file;
+ } catch {
+ return null;
+ }
+}
+
+// Move a downloaded source container OUT of the main data dir into the saved
+// store and write a pointer sidecar back into the data dir. Returns the pointer.
+// Overwrites any existing stored container for this video (the redownload case).
+export async function persistSourceVideo(opts: {
+ videoDir: string;
+ sourceFilename: string;
+ storeDir: string;
+ keepReason?: SavedVideoKeepReason;
+}): Promise<SavedVideoPointer> {
+ const src = path.join(opts.videoDir, opts.sourceFilename);
+ const dest = path.join(opts.storeDir, opts.sourceFilename);
+ const st = await stat(src);
+ await moveFileCrossDevice(src, dest);
+ const pointer: SavedVideoPointer = {
+ storedAt: new Date().toISOString(),
+ dir: opts.storeDir,
+ file: opts.sourceFilename,
+ bytes: st.size,
+ ...(opts.keepReason ? { keepReason: opts.keepReason } : {}),
+ };
+ await writePointer(opts.videoDir, pointer);
+ return pointer;
+}
+
+// Best-effort removal of a now-empty store dir (and its empty <slug> parent).
+async function pruneEmptyStoreDirs(storeDir: string): Promise<void> {
+ await rm(storeDir, { recursive: false, force: true }).catch(() => {});
+ await rm(path.dirname(storeDir), { recursive: false, force: true }).catch(
+ () => {},
+ );
+}
+
+// Reverse persistSourceVideo: move the stored container back into the data dir
+// and remove the pointer. Returns false when there was no pointer to reverse.
+export async function unpersistSavedVideo(videoDir: string): Promise<boolean> {
+ const pointer = await loadSavedVideo(videoDir);
+ if (!pointer) return false;
+ const src = savedVideoPath(pointer);
+ const dest = path.join(videoDir, pointer.file);
+ try {
+ await moveFileCrossDevice(src, dest);
+ } catch {
+ // The stored container is already gone; just drop the dangling pointer.
+ }
+ await rm(savedVideoPointerPath(videoDir), { force: true });
+ await pruneEmptyStoreDirs(pointer.dir);
+ return true;
+}
+
+// Permanently delete a video's persisted container + pointer (retention prune of
+// a video that rolled out of the keep-latest window). Returns bytes freed.
+export async function dropSavedVideo(videoDir: string): Promise<number> {
+ const pointer = await loadSavedVideo(videoDir);
+ if (!pointer) return 0;
+ await rm(savedVideoPath(pointer), { force: true });
+ await rm(savedVideoPointerPath(videoDir), { force: true });
+ await pruneEmptyStoreDirs(pointer.dir);
+ return pointer.bytes;
+}
diff --git a/common/lib/savedVideo.test.ts b/common/lib/savedVideo.test.ts
@@ -0,0 +1,155 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, stat, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "./paths";
+import {
+ parseSavedVideoPointer,
+ savedVideoDir,
+ savedVideoRoot,
+} from "./savedVideo";
+import {
+ dropSavedVideo,
+ loadSavedVideo,
+ persistSourceVideo,
+ resolveSavedVideo,
+ unpersistSavedVideo,
+} from "./savedVideo-server";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test lib/savedVideo.test.ts
+
+function fakePaths(savedVideosDir: string): Paths {
+ return { savedVideosDir } as Paths;
+}
+
+async function withTmp(fn: (root: string) => Promise<void>): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-saved-"));
+ try {
+ await fn(dir);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+test("savedVideoRoot prefers the per-channel override, else the global", () => {
+ const paths = fakePaths("/global/store");
+ assert.equal(savedVideoRoot(paths, undefined), "/global/store");
+ assert.equal(savedVideoRoot(paths, { savedVideosDir: " " }), "/global/store");
+ assert.equal(
+ savedVideoRoot(paths, { savedVideosDir: "/chan/store" }),
+ "/chan/store",
+ );
+});
+
+test("savedVideoDir composes <root>/<slug>/<videoId>", () => {
+ const paths = fakePaths("/global/store");
+ assert.equal(
+ savedVideoDir(paths, undefined, "chan", "vid1"),
+ path.join("/global/store", "chan", "vid1"),
+ );
+});
+
+test("parseSavedVideoPointer rejects malformed and reads keepReason", () => {
+ assert.equal(parseSavedVideoPointer(null), null);
+ assert.equal(parseSavedVideoPointer({ dir: "/a" }), null); // no file/bytes
+ const ok = parseSavedVideoPointer({
+ storedAt: "t",
+ dir: "/a",
+ file: "source-media.mp4",
+ bytes: 10,
+ keepReason: "keep-latest",
+ sha256: "abc",
+ });
+ assert.deepEqual(ok, {
+ storedAt: "t",
+ dir: "/a",
+ file: "source-media.mp4",
+ bytes: 10,
+ keepReason: "keep-latest",
+ sha256: "abc",
+ });
+ // An unknown keepReason is dropped rather than carried through.
+ const noReason = parseSavedVideoPointer({
+ dir: "/a",
+ file: "f.mp4",
+ bytes: 1,
+ keepReason: "bogus",
+ });
+ assert.equal(noReason?.keepReason, undefined);
+});
+
+test("persist moves the container into the store and writes a pointer", async () => {
+ await withTmp(async (root) => {
+ const videoDir = path.join(root, "data", "vid1");
+ const storeDir = path.join(root, "store", "chan", "vid1");
+ await mkdir(videoDir, { recursive: true });
+ await writeFile(path.join(videoDir, "source-media.mp4"), "video-bytes");
+
+ const pointer = await persistSourceVideo({
+ videoDir,
+ sourceFilename: "source-media.mp4",
+ storeDir,
+ keepReason: "keep-latest",
+ });
+
+ assert.equal(pointer.file, "source-media.mp4");
+ assert.equal(pointer.dir, storeDir);
+ assert.equal(pointer.keepReason, "keep-latest");
+ assert.equal(pointer.bytes, "video-bytes".length);
+ // Container moved OUT of the data dir, into the store.
+ await assert.rejects(stat(path.join(videoDir, "source-media.mp4")));
+ await stat(path.join(storeDir, "source-media.mp4"));
+ // Pointer readable + resolvable.
+ const loaded = await loadSavedVideo(videoDir);
+ assert.equal(loaded?.keepReason, "keep-latest");
+ assert.equal(
+ await resolveSavedVideo(videoDir),
+ path.join(storeDir, "source-media.mp4"),
+ );
+ });
+});
+
+test("unpersist returns the container to the data dir and drops the pointer", async () => {
+ await withTmp(async (root) => {
+ const videoDir = path.join(root, "data", "vid1");
+ const storeDir = path.join(root, "store", "chan", "vid1");
+ await mkdir(videoDir, { recursive: true });
+ await writeFile(path.join(videoDir, "source-media.webm"), "abc");
+ await persistSourceVideo({
+ videoDir,
+ sourceFilename: "source-media.webm",
+ storeDir,
+ keepReason: "override",
+ });
+
+ assert.equal(await unpersistSavedVideo(videoDir), true);
+ await stat(path.join(videoDir, "source-media.webm"));
+ assert.equal(await loadSavedVideo(videoDir), null);
+ assert.equal(await resolveSavedVideo(videoDir), null);
+ // Reversing again is a no-op.
+ assert.equal(await unpersistSavedVideo(videoDir), false);
+ });
+});
+
+test("drop deletes the stored container and reports bytes freed", async () => {
+ await withTmp(async (root) => {
+ const videoDir = path.join(root, "data", "vid1");
+ const storeDir = path.join(root, "store", "chan", "vid1");
+ await mkdir(videoDir, { recursive: true });
+ await writeFile(path.join(videoDir, "source-media.mkv"), "0123456789");
+ await persistSourceVideo({
+ videoDir,
+ sourceFilename: "source-media.mkv",
+ storeDir,
+ keepReason: "keep-latest",
+ });
+
+ const freed = await dropSavedVideo(videoDir);
+ assert.equal(freed, 10);
+ assert.equal(await loadSavedVideo(videoDir), null);
+ await assert.rejects(stat(path.join(storeDir, "source-media.mkv")));
+ assert.equal(await dropSavedVideo(videoDir), 0); // nothing left
+ });
+});
diff --git a/common/lib/savedVideo.ts b/common/lib/savedVideo.ts
@@ -0,0 +1,92 @@
+import path from "node:path";
+import type { Paths } from "./paths";
+import type { ChannelConfig } from "./channelConfig";
+
+// The saved-video store (Phase 3 of the video-persistence feature).
+//
+// When the per-download persistence rule decides to keep a source video, the
+// downloaded container (data/<id>/source-media.<ext> from Phase 2) is MOVED out
+// of the per-video data dir into a separate store and a small pointer sidecar is
+// left behind in the data dir. This keeps the main data volume holding only
+// audio + transcripts while the (large) source videos can live on another disk.
+//
+// Layout:
+// <root>/<slug>/<videoId>/source-media.<ext> (the moved container)
+// channels/<slug>/data/<videoId>/saved-video.json (the pointer back to it)
+//
+// <root> = ChannelConfig.savedVideosDir (per-channel override) || Paths.savedVideosDir.
+//
+// This module is pure (path math + pointer parse) so it's safe to import from
+// both client and server; the filesystem operations live in savedVideo-server.ts.
+
+export const SAVED_VIDEO_POINTER_FILENAME = "saved-video.json";
+
+// Why this source was persisted (mirrors persistencePlan's PersistCategory, minus
+// "none"). The retention prune only evicts "keep-latest" containers once they
+// roll out of the window; "pin" (irreplaceable) and "override" (manually
+// archived) containers are kept until explicitly unpersisted.
+export type SavedVideoKeepReason = "keep-latest" | "pin" | "override";
+
+export type SavedVideoPointer = {
+ // ISO timestamp the container was persisted into the store.
+ storedAt: string;
+ // Absolute store dir that holds the container (savedVideoDir() result).
+ dir: string;
+ // Container basename within `dir` (e.g. "source-media.mp4").
+ file: string;
+ // Size of the stored container in bytes (for backup manifests + disk reports).
+ bytes: number;
+ // Why it was persisted (governs retention pruning). Absent on legacy pointers,
+ // which the prune treats conservatively as non-evictable.
+ keepReason?: SavedVideoKeepReason;
+ // Optional content hash, populated by the backup/verify step (Phase 4).
+ sha256?: string;
+};
+
+function isKeepReason(v: unknown): v is SavedVideoKeepReason {
+ return v === "keep-latest" || v === "pin" || v === "override";
+}
+
+// The store root for a channel: the per-channel override when set, else the
+// global default. Whitespace-only overrides fall back to the global default.
+export function savedVideoRoot(
+ paths: Paths,
+ config: Pick<ChannelConfig, "savedVideosDir"> | null | undefined,
+): string {
+ const override = config?.savedVideosDir?.trim();
+ return override ? override : paths.savedVideosDir;
+}
+
+// The store dir for a single video: <root>/<slug>/<videoId>/.
+export function savedVideoDir(
+ paths: Paths,
+ config: Pick<ChannelConfig, "savedVideosDir"> | null | undefined,
+ channelSlug: string,
+ videoId: string,
+): string {
+ return path.join(savedVideoRoot(paths, config), channelSlug, videoId);
+}
+
+// Absolute path to the stored container a pointer references.
+export function savedVideoPath(pointer: SavedVideoPointer): string {
+ return path.join(pointer.dir, pointer.file);
+}
+
+export function parseSavedVideoPointer(raw: unknown): SavedVideoPointer | null {
+ if (!raw || typeof raw !== "object") return null;
+ const r = raw as Record<string, unknown>;
+ if (typeof r.dir !== "string" || r.dir === "") return null;
+ if (typeof r.file !== "string" || r.file === "") return null;
+ if (typeof r.bytes !== "number" || !Number.isFinite(r.bytes)) return null;
+ const pointer: SavedVideoPointer = {
+ storedAt: typeof r.storedAt === "string" ? r.storedAt : "",
+ dir: r.dir,
+ file: r.file,
+ bytes: r.bytes,
+ };
+ if (isKeepReason(r.keepReason)) pointer.keepReason = r.keepReason;
+ if (typeof r.sha256 === "string" && r.sha256 !== "") {
+ pointer.sha256 = r.sha256;
+ }
+ return pointer;
+}
diff --git a/common/ytdlp/downloadOneManaged.ts b/common/ytdlp/downloadOneManaged.ts
@@ -23,6 +23,8 @@ import {
import { isDoNotClean } from "../lib/doNotClean-server";
import { transcodeAudio } from "../controller/transcode";
import { findSourceMedia } from "../lib/videoStatus";
+import { savedVideoDir } from "../lib/savedVideo";
+import { persistSourceVideo } from "../lib/savedVideo-server";
import {
type AudioCheckAttemptStats,
type DownloadAttempt,
@@ -155,16 +157,25 @@ function transcribeMediaArgs(
}
// After an app-mode transcribe download, produce audio.<fmt> from the downloaded
-// source-media.<ext> container via ffmpeg. When the video is not persisted the
-// container is then removed (the extract-now / save-disk path). Best-effort: a
-// failed extraction leaves the container in place (so a later pass or the
-// transcribe-from-container fallback can still recover) and is reported rather
-// than failing the whole download.
+// source-media.<ext> container via ffmpeg. When the video is persisted the
+// container is MOVED into the saved-video store (Phase 3) and a pointer is left
+// in the data dir; otherwise the container is removed (the extract-now /
+// save-disk path). Best-effort throughout: a failed extraction leaves the
+// container in place (so a later pass or the transcribe-from-container fallback
+// can still recover), and a failed store-move leaves the container in the data
+// dir as source-media.<ext> (still persisted, just not relocated). Neither fails
+// the whole download.
async function finalizeAppExtraction(opts: {
paths: Paths;
+ channelSlug: string;
+ channelConfig: ChannelConfig;
videoDir: string;
+ videoId: string;
fmt: AudioFormat;
persist: boolean;
+ // The persistence cause, recorded on the saved-video pointer (governs the
+ // retention prune). Only meaningful when persist is true.
+ category: PersistenceDecision["category"];
onLog: (s: string) => void;
signal: AbortSignal;
}): Promise<void> {
@@ -191,11 +202,32 @@ async function finalizeAppExtraction(opts: {
);
return;
}
- if (opts.persist) {
- opts.onLog(`Kept source container ${source} (persisted).\n`);
- } else {
+ if (!opts.persist) {
await rm(path.join(opts.videoDir, source), { force: true });
opts.onLog(`Discarded source container ${source} (audio-only).\n`);
+ return;
+ }
+ // Persist: move the container into the saved-video store + write a pointer.
+ const storeDir = savedVideoDir(
+ opts.paths,
+ opts.channelConfig,
+ opts.channelSlug,
+ opts.videoId,
+ );
+ try {
+ const pointer = await persistSourceVideo({
+ videoDir: opts.videoDir,
+ sourceFilename: source,
+ storeDir,
+ keepReason: opts.category === "none" ? undefined : opts.category,
+ });
+ opts.onLog(
+ `Persisted source video to ${path.join(pointer.dir, pointer.file)} (${pointer.bytes} bytes).\n`,
+ );
+ } catch (err) {
+ opts.onLog(
+ `Failed to move source container into the saved-video store (${(err as Error).message}); leaving ${source} in the data dir.\n`,
+ );
}
}
@@ -709,9 +741,13 @@ async function runManagedDownload(
) {
await finalizeAppExtraction({
paths: opts.paths,
+ channelSlug: opts.channelSlug,
+ channelConfig: opts.channelConfig,
videoDir,
+ videoId,
fmt,
persist: plan.persist,
+ category: plan.category,
onLog: opts.onLog,
signal: opts.signal,
});
@@ -782,9 +818,13 @@ async function runManagedDownload(
if (plan.extractionMode === "app") {
await finalizeAppExtraction({
paths: opts.paths,
+ channelSlug: opts.channelSlug,
+ channelConfig: opts.channelConfig,
videoDir,
+ videoId,
fmt,
persist: plan.persist,
+ category: plan.category,
onLog: opts.onLog,
signal: opts.signal,
});
diff --git a/common/ytdlp/persistencePlan.test.ts b/common/ytdlp/persistencePlan.test.ts
@@ -25,12 +25,19 @@ test("inside the keep-latest window persists via app extraction", () => {
const d = resolvePersistenceDecision({ inWindow: true, pinned: false });
assert.equal(d.persist, true);
assert.equal(d.extractionMode, "app");
+ assert.equal(d.category, "keep-latest");
});
test("a do-not-clean pin persists even outside the window", () => {
const d = resolvePersistenceDecision({ inWindow: false, pinned: true });
assert.equal(d.persist, true);
assert.equal(d.extractionMode, "app");
+ assert.equal(d.category, "pin");
+});
+
+test("category is 'none' for an audio-only outcome", () => {
+ const d = resolvePersistenceDecision({ inWindow: false, pinned: false });
+ assert.equal(d.category, "none");
});
test("extractImmediately overrides the keep-latest window", () => {
@@ -50,6 +57,7 @@ test("keepSourceVideoOverride=true persists an out-of-window video", () => {
});
assert.equal(d.persist, true);
assert.equal(d.extractionMode, "app");
+ assert.equal(d.category, "override");
});
test("keepSourceVideoOverride=false beats the window and extractImmediately", () => {
diff --git a/common/ytdlp/persistencePlan.ts b/common/ytdlp/persistencePlan.ts
@@ -30,11 +30,20 @@ export type PersistenceDecisionInput = {
overrides?: PersistRunOverrides;
};
+// Why a video's source is persisted — recorded on the saved-video pointer so the
+// retention prune (Phase 3) can tell a rolling keep-latest container (evictable
+// once it leaves the window) apart from a manually-archived one ("override") or a
+// pinned/irreplaceable one ("pin"), which must never be auto-pruned. "none" when
+// the video is not persisted.
+export type PersistCategory = "keep-latest" | "pin" | "override" | "none";
+
export type PersistenceDecision = {
// Keep the downloaded source video container (Phase 3 moves it to the store;
// Phase 2 leaves it in the data dir as source-media.<ext>).
persist: boolean;
extractionMode: ExtractionMode;
+ // Coarse cause of the decision, used by the saved-store retention prune.
+ category: PersistCategory;
// Human-readable justification, surfaced in the download log.
reason: string;
};
@@ -53,28 +62,35 @@ export function resolvePersistenceDecision(
): PersistenceDecision {
const o = input.overrides ?? {};
let persist: boolean;
+ let category: PersistCategory;
let reason: string;
if (o.keepSourceVideoOverride === true) {
persist = true;
+ category = "override";
reason = "per-run keep-source-video override";
} else if (o.keepSourceVideoOverride === false) {
persist = false;
+ category = "none";
reason = "per-run no-keep override";
} else if (o.extractImmediately) {
persist = false;
+ category = "none";
reason = "per-run extract-immediately override";
} else if (input.pinned) {
persist = true;
+ category = "pin";
reason = "do-not-clean pin";
} else if (input.inWindow) {
persist = true;
+ category = "keep-latest";
reason = "keep-latest window";
} else {
persist = false;
+ category = "none";
reason = "outside keep-latest window";
}
const extractionMode: ExtractionMode = persist
? "app"
: (input.channelExtractionMode ?? "ytdlp");
- return { persist, extractionMode, reason };
+ return { persist, extractionMode, category, reason };
}
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,7 @@
# Changelog
## [Unreleased]
+- **Saved-video store: persisted source videos move to a separate dir/disk, with retention pruning (phase 3 of the video-persistence subsystem; backend, UI lands later).** When the per-download persistence rule (phase 2) keeps a source video, the downloaded container is now **moved out of the per-video data dir into a separate saved-video store** — leaving only a small `saved-video.json` pointer behind — so the main data volume holds just audio + transcripts while the (large) source videos can live on another disk. The store root defaults to `<transcripts>/saved-videos`, is overridable globally via the **`SAVED_VIDEOS_DIR`** env var, and can be further overridden **per channel** (`savedVideosDir` in `config.json`); a video's container lands under `<root>/<slug>/<videoId>/`. The move is **cross-device-safe** (rename within a disk, copy-to-temp + atomic rename + unlink across disks) and **best-effort** — a failed move leaves the container in the data dir as `source-media.<ext>` (still persisted, just not relocated) rather than failing the download. **Transcription resolves from the store**: when no extracted `audio.*` exists, the transcribe fallback follows the pointer to the stored container (returned as a path relative to the video dir so both the local engine and the remote uploader read it correctly), so a kept-but-cleaned or archive-only video still transcribes. **Retention pruning** (the phase-2 follow-up) now bounds the store: the Clean-audio sweep also evicts any *keep-latest* container that has rolled out of the window — but **never** a manually-archived (`override`) or pinned/irreplaceable (`pin`/do-not-clean) one, distinguished by a `keepReason` recorded on each pointer. Reversible helpers ship for the upcoming UI: `unpersistSavedVideo` (move the container back) and `dropSavedVideo` (delete it). A reusable `checkDiskSpaceFor(dir, …)` lands so disk gating can target the store filesystem (used by the UI/backup phases). Note: source-video persistence is still skipped for audio-check channels (deferred), and saved-store counts aren't yet surfaced in the channel snapshot (lands with the phase-5 UI). See the new `common/lib/savedVideo.ts` (+ `savedVideo-server.ts` + tests), `common/controller/pruneSavedVideos.ts` (+ tests), `common/lib/paths.ts` (`savedVideosDir`), `common/lib/channelConfig.ts` (`savedVideosDir`), `common/lib/diskSpace.ts`, `common/ytdlp/{persistencePlan,downloadOneManaged}.ts`, `common/controller/{transcribeOne,cleanAudioFromTranscribed}.ts`.
- **Per-download persistence rule + app-side audio extraction (phase 2 of the video-persistence subsystem; backend, UI lands later).** Each individual download now consults the channel's keep-latest rule (plus any per-run overrides) to decide *what to keep*: a video inside the keep-latest window — or one carrying a `do-not-clean` pin — downloads its **full source video** (`bestvideo*+bestaudio/best`) and the app extracts `audio.<fmt>` from it with ffmpeg, keeping the container as `source-media.<ext>` (a deliberately distinct name from `audio.<ext>` so it's never mistaken for cleanable audio); everything else stays audio-only as before. Crucially the keep decision is made per video by its **upload date against the channel's Nth-newest cutoff** (computed once per run via the new `computeKeepWindow`/`isInKeepWindow`), so the newest videos — which aren't on disk yet at download time — are correctly persisted. A new **`extractionMode`** channel setting (`"ytdlp"` default | `"app"`) selects who extracts audio for audio-only downloads; persisting always forces app-side extraction. **Per-run overrides** thread through `download-missing` (and the shared managed-download path): `keepSourceVideoOverride` (force keep/discard), `extractImmediately` (extract now + discard the container even on a keep channel — the disk-saving backfill case), and `audioFormatOverride`. **Transcription falls back to the source container** when no extracted `audio.*` exists (parakeet ffmpeg-slices any container), so a kept-but-cleaned video or an archive-only download is still transcribable. New per-video **"Archive source video"** action (`redownloadToArchiveAction`) re-fetches an existing video purely to grab + keep its source container without disturbing the transcript. Legacy `"ytdlp"`-mode downloads produce byte-identical yt-dlp args to before (no behavior change for existing channels). Note: persisted source containers currently remain in the data dir and are not auto-pruned when they roll out of the window, and source-video persistence is skipped (with a log note) for audio-check channels — both addressed by the saved-video store in phase 3. See `common/ytdlp/persistencePlan.ts` (+ tests), `common/controller/keptVideos.ts` (keep-window), `common/ytdlp/downloadOneManaged.ts`, `common/ytdlp/runYtdlp.ts`, `common/controller/transcribeOne.ts`, `common/lib/videoStatus.ts` (`source-media`/`isVideoContainer`), and `editor/app/channels/[slug]/pipelineActions.ts` / `videos/[id]/videoActions.ts`.
- **New per-channel "keep latest N" retention rule that protects recent videos from the Clean-audio sweep and pins any that get deleted from their source (backend; UI lands in a later change).** A channel can set `keepLatest` (in `config.json` for now) to shield its newest N videos — by upload date, a rolling window — from the **Clean audio** cleanup: those dirs are skipped just like a `do-not-clean` marker, and the snapshot's reclaim estimates/cleanup buckets exclude them (a new `keptCount` is recorded). Because a kept video can later be **deleted from its source** (YouTube etc.) and become irreplaceable, a new **kept-deletion check** re-probes just the kept window's availability (reusing `runAvailabilityCheck` with `onlyIds` + `recheck-non-deleted`) and **permanently pins** any video found `deleted`/`private`/`members_only` with a `do-not-clean` marker, so it survives even after it rolls out of the window. The check runs on demand via the new `check-kept-deleted` managed job (`checkKeptDeletedAction`, re-runnable/bookmarkable) and automatically from the sync scheduler on its own cadence (new `syncScheduler.keepLatestCheckIntervalMinutes`, default daily; per-channel `lastKeptCheckAt` state; suppressed during quiet hours, capped per tick, and skipped for a channel just synced this tick). This is phase 1 of a larger **video-persistence** subsystem (per-download persistence rules, a separate saved-video store, and backups follow). See `common/lib/channelConfig.ts` (`keepLatest`), the new `common/controller/keptVideos.ts` (`computeKeptVideoIds` + tests) and `common/controller/checkKeptDeleted.ts`, `common/controller/cleanAudioFromTranscribed.ts`, `common/controller/channelSnapshot.ts`, and the scheduler wiring in `common/lib/settings.ts`, `common/jobs/syncSchedulerState.ts`, and `editor/app/scheduler/runTick.ts`.
- **The monitor widget can now carry optional control buttons (Pause/Resume Transcriptions, Drain all).** The `/widget` view stays read-only by default, but a new **Show control buttons** option in the widget builder (URL flag `controls=1`) adds an interactive row at the top with the same **Pause Transcriptions** toggle (between-segment GPU release) and **Drain all** as the full app — so a pinned/iframe monitor can pause the GPU or wind work down without opening the editor. The controls stay visible even when the widget is otherwise idle (so you can pause preemptively), and the widget refetches its worker payload on a pause/resume so the toggle flips immediately instead of waiting for the next poll. Defaults keep the widget control-free, so existing links render unchanged. See `editor/app/widget/lib/config.ts`, the new `editor/app/widget/components/WidgetControls.tsx`, `editor/app/widget/components/MonitorWidget.tsx`, and `editor/app/widget/builder/components/WidgetBuilder.tsx`.
