commit 2a25799aeecfae4ccabc3b11673eced949576b34
parent 43620a86ed8c1fa649143891e0445161906aaced
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 25 Jun 2026 08:21:12 -0400
Merge feat/video-persistence: keep-latest retention, saved-video store, backups & first-class UI
Five-phase video-persistence subsystem:
1. keep-latest retention window + deletion pinning (cleanup protection)
2. per-download persistence rule + app-side ffmpeg extraction + run overrides
3. saved-video store (separate dir/disk; move + pointer)
4. rsync-mirror backups + manifest + drift verification
5. first-class UI (Saved Videos area, channel retention controls, per-video persist/unpersist)
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Diffstat:
48 files changed, 3791 insertions(+), 57 deletions(-)
diff --git a/common/controller/backupSavedVideos.test.ts b/common/controller/backupSavedVideos.test.ts
@@ -0,0 +1,157 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import {
+ appendFile,
+ mkdir,
+ mkdtemp,
+ readFile,
+ rm,
+ stat,
+ writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import { persistSourceVideo } from "../lib/savedVideo-server";
+import { loadSavedVideo } from "../lib/savedVideo-server";
+import {
+ BACKUP_MANIFEST_FILENAME,
+ parseBackupManifest,
+} from "../lib/savedVideoBackup";
+import {
+ backupSavedVideos,
+ verifySavedVideoBackup,
+} from "./backupSavedVideos";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/backupSavedVideos.test.ts
+//
+// These exercise the real rsync binary (resolved via paths.rsyncBin -> "rsync").
+
+async function withPaths(
+ fn: (paths: Paths, dest: string) => Promise<void>,
+): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-backup-"));
+ const paths = {
+ channelsDir: path.join(dir, "channels"),
+ savedVideosDir: path.join(dir, "saved"),
+ rsyncBin: "rsync",
+ } as Paths;
+ const dest = path.join(dir, "backup");
+ try {
+ await fn(paths, dest);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+async function seed(
+ paths: Paths,
+ slug: string,
+ id: string,
+ content: string,
+): Promise<void> {
+ const videoDir = path.join(paths.channelsDir, slug, "data", id);
+ await mkdir(videoDir, { recursive: true });
+ await writeFile(path.join(videoDir, "source-media.mp4"), content);
+ await persistSourceVideo({
+ videoDir,
+ sourceFilename: "source-media.mp4",
+ storeDir: path.join(paths.savedVideosDir, slug, id),
+ keepReason: "keep-latest",
+ });
+}
+
+function sha256(s: string): string {
+ return createHash("sha256").update(s).digest("hex");
+}
+
+test("backup mirrors the store + writes a manifest with sizes and hashes", async () => {
+ await withPaths(async (paths, dest) => {
+ await seed(paths, "alpha", "v1", "hello video one");
+ await seed(paths, "beta", "v2", "second");
+
+ const res = await backupSavedVideos({ paths, dest });
+ assert.equal(res.entries, 2);
+ assert.equal(res.backedUp, 2);
+
+ // Containers landed at <dest>/<slug>/<id>/source-media.mp4.
+ assert.equal(
+ await readFile(path.join(dest, "alpha", "v1", "source-media.mp4"), "utf8"),
+ "hello video one",
+ );
+ assert.equal(
+ await readFile(path.join(dest, "beta", "v2", "source-media.mp4"), "utf8"),
+ "second",
+ );
+
+ // Manifest records each entry with the right size + streamed sha256.
+ const manifest = parseBackupManifest(
+ JSON.parse(
+ await readFile(path.join(dest, BACKUP_MANIFEST_FILENAME), "utf8"),
+ ),
+ );
+ assert.ok(manifest);
+ assert.equal(manifest.entries.length, 2);
+ const alpha = manifest.entries.find((e) => e.slug === "alpha");
+ assert.ok(alpha);
+ assert.equal(alpha.bytes, "hello video one".length);
+ assert.equal(alpha.sha256, sha256("hello video one"));
+
+ // The checksum was cached back onto the live pointer.
+ const pointer = await loadSavedVideo(
+ path.join(paths.channelsDir, "alpha", "data", "v1"),
+ );
+ assert.equal(pointer?.sha256, sha256("hello video one"));
+
+ // A fresh verify is clean.
+ const verify = await verifySavedVideoBackup({ paths, dest });
+ assert.equal(verify.ok, true);
+ assert.equal(verify.checked, 2);
+ });
+});
+
+test("verify reports size, checksum, missing, and extra drift", async () => {
+ await withPaths(async (paths, dest) => {
+ await seed(paths, "alpha", "v1", "original-one");
+ await seed(paths, "alpha", "v2", "original-two");
+ await seed(paths, "beta", "v3", "original-three");
+ await backupSavedVideos({ paths, dest });
+
+ // Grow one dest file -> size mismatch.
+ await appendFile(path.join(dest, "alpha", "v1", "source-media.mp4"), "X");
+ // Same-size edit of another -> checksum mismatch (replace with equal length).
+ const v2 = path.join(dest, "alpha", "v2", "source-media.mp4");
+ const v2bytes = (await stat(v2)).size;
+ await writeFile(v2, "Z".repeat(v2bytes));
+ // Delete the third -> missing.
+ await rm(path.join(dest, "beta", "v3", "source-media.mp4"));
+ // Drop an unknown container into the dest -> extra.
+ await mkdir(path.join(dest, "ghost", "v9"), { recursive: true });
+ await writeFile(path.join(dest, "ghost", "v9", "source-media.mp4"), "boo");
+
+ const drift = await verifySavedVideoBackup({ paths, dest });
+ assert.equal(drift.ok, false);
+ assert.deepEqual(
+ drift.sizeMismatch.map((e) => e.videoId),
+ ["v1"],
+ );
+ assert.deepEqual(
+ drift.checksumMismatch.map((e) => e.videoId),
+ ["v2"],
+ );
+ assert.deepEqual(
+ drift.missing.map((e) => e.videoId),
+ ["v3"],
+ );
+ assert.deepEqual(drift.extra, [path.join("ghost", "v9", "source-media.mp4")]);
+ });
+});
+
+test("verify throws when no manifest is present", async () => {
+ await withPaths(async (paths, dest) => {
+ await mkdir(dest, { recursive: true });
+ await assert.rejects(verifySavedVideoBackup({ paths, dest }));
+ });
+});
diff --git a/common/controller/backupSavedVideos.ts b/common/controller/backupSavedVideos.ts
@@ -0,0 +1,258 @@
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { mkdir, readFile, readdir, rename, stat, writeFile } from "node:fs/promises";
+import { execa } from "execa";
+import type { Paths } from "../lib/paths";
+import {
+ BACKUP_MANIFEST_FILENAME,
+ backupEntryRelDir,
+ backupEntryRelPath,
+ parseBackupManifest,
+ type BackupEntry,
+ type BackupManifest,
+} from "../lib/savedVideoBackup";
+import { listSavedVideos } from "./savedVideoInventory";
+import { updateSavedVideoChecksum } from "../lib/savedVideo-server";
+
+// Backup tooling for the saved-video store (Phase 4 of the video-persistence
+// feature). `dest` is treated as a local filesystem path (typically a mounted
+// backup disk): containers are mirrored into it with rsync — incremental and
+// resumable, so re-running only transfers changed/new files — and a manifest
+// recording each container's size + streamed sha256 is written at the dest root.
+// The verify step reads that manifest back and reports drift against the dest.
+
+// Stream a file through sha256 without buffering it whole (containers are large).
+async function sha256File(file: string): Promise<string> {
+ const hash = createHash("sha256");
+ await new Promise<void>((resolve, reject) => {
+ const stream = createReadStream(file);
+ stream.on("error", reject);
+ stream.on("data", (chunk) => hash.update(chunk));
+ stream.on("end", () => resolve());
+ });
+ return hash.digest("hex");
+}
+
+export type BackupSavedVideosResult = {
+ // Saved containers considered (one per data-dir pointer).
+ entries: number;
+ // Containers rsynced to the destination (entries minus any that errored).
+ backedUp: number;
+ // Total bytes across the backed-up containers.
+ bytes: number;
+ // Absolute path to the manifest written at the destination root.
+ manifestPath: string;
+};
+
+export async function backupSavedVideos({
+ paths,
+ dest,
+ onLog,
+ signal,
+}: {
+ paths: Paths;
+ dest: string;
+ onLog?: (msg: string) => void;
+ signal?: AbortSignal;
+}): Promise<BackupSavedVideosResult> {
+ const log = onLog ?? ((m: string) => console.log(m));
+ const destRoot = dest.trim();
+ if (!destRoot) throw new Error("No backup destination configured");
+
+ const saved = await listSavedVideos({ paths });
+ log(`Backing up ${saved.length} saved video(s) to ${destRoot}`);
+ await mkdir(destRoot, { recursive: true });
+
+ const manifestEntries: BackupEntry[] = [];
+ let backedUp = 0;
+ let bytes = 0;
+ for (const entry of saved) {
+ if (signal?.aborted) break;
+ const destDir = path.join(destRoot, backupEntryRelDir(entry));
+ try {
+ const st = await stat(entry.storedPath);
+ await mkdir(destDir, { recursive: true });
+ // Trailing slash on destDir keeps rsync placing the file inside it.
+ const args = ["-a", "--partial", entry.storedPath, `${destDir}/`];
+ log(`$ ${paths.rsyncBin} ${args.join(" ")}`);
+ const child = execa(paths.rsyncBin, args, {
+ cancelSignal: signal,
+ all: true,
+ buffer: false,
+ reject: false,
+ });
+ child.all?.on("data", (c: Buffer) => log(c.toString("utf8")));
+ const result = await child;
+ if (result.exitCode !== 0) {
+ if (signal?.aborted) break;
+ log(
+ `rsync failed for ${entry.slug}/${entry.videoId} (exit ${result.exitCode}); skipping`,
+ );
+ continue;
+ }
+ const sha256 = await sha256File(entry.storedPath);
+ // Cache the checksum back onto the pointer for the store/UI.
+ await updateSavedVideoChecksum(entry.videoDir, sha256).catch(() => {});
+ manifestEntries.push({
+ slug: entry.slug,
+ videoId: entry.videoId,
+ file: entry.pointer.file,
+ bytes: st.size,
+ sha256,
+ });
+ backedUp++;
+ bytes += st.size;
+ } catch (err) {
+ log(
+ `Failed to back up ${entry.slug}/${entry.videoId}: ${(err as Error).message}`,
+ );
+ }
+ }
+
+ const manifest: BackupManifest = {
+ backedUpAt: new Date().toISOString(),
+ entries: manifestEntries,
+ };
+ const manifestPath = path.join(destRoot, BACKUP_MANIFEST_FILENAME);
+ const tmp = `${manifestPath}.tmp-${process.pid}`;
+ await writeFile(tmp, JSON.stringify(manifest, null, 2) + "\n");
+ await rename(tmp, manifestPath);
+ log(
+ `Backup complete: ${backedUp}/${saved.length} container(s), ${bytes} bytes. Manifest at ${manifestPath}`,
+ );
+
+ return { entries: saved.length, backedUp, bytes, manifestPath };
+}
+
+export type BackupDrift = {
+ // Manifest entries whose file is absent from the destination.
+ missing: BackupEntry[];
+ // Present at the destination but a different size than the manifest records.
+ sizeMismatch: BackupEntry[];
+ // Present and right-sized but whose content hash no longer matches (only
+ // computed when `checksum` is true).
+ checksumMismatch: BackupEntry[];
+ // Container files at the destination that the manifest doesn't know about
+ // (relative <slug>/<videoId>/<file> paths).
+ extra: string[];
+};
+
+export type VerifyBackupResult = BackupDrift & {
+ // True when every drift bucket is empty.
+ ok: boolean;
+ backedUpAt: string;
+ checked: number;
+};
+
+// Walk the destination two levels deep (<slug>/<videoId>) collecting the
+// relative paths of stored container files, ignoring the manifest itself.
+async function listDestEntries(destRoot: string): Promise<Set<string>> {
+ const out = new Set<string>();
+ let slugs: string[];
+ try {
+ slugs = (await readdir(destRoot, { withFileTypes: true }))
+ .filter((e) => e.isDirectory())
+ .map((e) => e.name);
+ } catch {
+ return out;
+ }
+ for (const slug of slugs) {
+ const slugDir = path.join(destRoot, slug);
+ const ids = (await readdir(slugDir, { withFileTypes: true }).catch(() => []))
+ .filter((e) => e.isDirectory())
+ .map((e) => e.name);
+ for (const id of ids) {
+ const files = await readdir(path.join(slugDir, id)).catch(
+ () => [] as string[],
+ );
+ for (const f of files) out.add(path.join(slug, id, f));
+ }
+ }
+ return out;
+}
+
+export async function verifySavedVideoBackup({
+ paths: _paths,
+ dest,
+ onLog,
+ signal,
+ checksum = true,
+}: {
+ paths: Paths;
+ dest: string;
+ onLog?: (msg: string) => void;
+ signal?: AbortSignal;
+ // Re-hash each present, right-sized file and compare to the manifest. Off
+ // skips the (potentially expensive) hashing and only checks presence + size.
+ checksum?: boolean;
+}): Promise<VerifyBackupResult> {
+ const log = onLog ?? ((m: string) => console.log(m));
+ const destRoot = dest.trim();
+ if (!destRoot) throw new Error("No backup destination configured");
+
+ let manifest: BackupManifest | null = null;
+ try {
+ manifest = parseBackupManifest(
+ JSON.parse(
+ await readFile(path.join(destRoot, BACKUP_MANIFEST_FILENAME), "utf8"),
+ ),
+ );
+ } catch {
+ manifest = null;
+ }
+ if (!manifest) {
+ throw new Error(`No readable backup manifest at ${destRoot}`);
+ }
+
+ const missing: BackupEntry[] = [];
+ const sizeMismatch: BackupEntry[] = [];
+ const checksumMismatch: BackupEntry[] = [];
+ const known = new Set<string>();
+ let checked = 0;
+
+ for (const entry of manifest.entries) {
+ if (signal?.aborted) break;
+ const rel = backupEntryRelPath(entry);
+ known.add(rel);
+ const file = path.join(destRoot, rel);
+ let st;
+ try {
+ st = await stat(file);
+ } catch {
+ missing.push(entry);
+ continue;
+ }
+ checked++;
+ if (st.size !== entry.bytes) {
+ sizeMismatch.push(entry);
+ continue;
+ }
+ if (checksum) {
+ const got = await sha256File(file);
+ if (got !== entry.sha256) checksumMismatch.push(entry);
+ }
+ }
+
+ const destEntries = await listDestEntries(destRoot);
+ const extra = [...destEntries].filter((rel) => !known.has(rel)).sort();
+
+ const ok =
+ missing.length === 0 &&
+ sizeMismatch.length === 0 &&
+ checksumMismatch.length === 0 &&
+ extra.length === 0;
+ log(
+ `Verify: checked ${checked}/${manifest.entries.length}; ${missing.length} missing, ${sizeMismatch.length} size-mismatch, ${checksumMismatch.length} checksum-mismatch, ${extra.length} extra.`,
+ );
+
+ return {
+ ok,
+ backedUpAt: manifest.backedUpAt,
+ checked,
+ missing,
+ sizeMismatch,
+ checksumMismatch,
+ extra,
+ };
+}
diff --git a/common/controller/channelSnapshot.ts b/common/controller/channelSnapshot.ts
@@ -26,6 +26,7 @@ import { extractVideoId } from "../ytdlp/runYtdlp";
import { reconcileVideoDirs } from "./reconcileVideoDirs";
import { loadFailedTranscriptions } from "./failedTranscriptions";
import { readChannelConfig } from "./channels";
+import { computeKeptVideoIds } from "./keptVideos";
import { loadMaybeMissing } from "./quickAvailabilityCheck";
export type AvailabilitySnapshot = {
@@ -82,6 +83,11 @@ export type ChannelSnapshot = {
};
undownloadedIds: string[];
excludedFromDownload?: ExcludedFromDownload;
+ // Count of videos in the channel's keep-latest window (ChannelConfig.keepLatest).
+ // These are protected from the Clean-audio sweep (folded into the cleanup-
+ // exclusion set alongside do-not-clean markers) and targeted for source-video
+ // persistence. Optional: older snapshots lack it; readers must default to 0.
+ keptCount?: number;
availability?: AvailabilitySnapshot;
// Known videos absent from the channel's most recent fresh flat-playlist
// fetch (the quick availability check). The maybe-missing.json sidecar is the
@@ -214,6 +220,13 @@ export async function generateChannelSnapshot(
const targetAudioFile = config?.audioFormat
? `audio.${config.audioFormat}`
: null;
+ // The keep-latest window: newest N videos protected from cleanup. Computed
+ // once and folded into the cleanup-exclusion set below.
+ const keptIds = await computeKeptVideoIds({
+ paths,
+ channelSlug: slug,
+ keepLatest: config?.keepLatest ?? 0,
+ });
const videoDirNames = dirEntries
.filter((d) => d.isDirectory())
.map((d) => d.name);
@@ -307,6 +320,11 @@ export async function generateChannelSnapshot(
for (const v of perVideo) {
if (v.doNotClean) doNotCleanIds.add(v.id);
}
+ // Everything shielded from the Clean-audio sweep: explicit do-not-clean markers
+ // plus the rolling keep-latest window. The cleanup buckets/reclaim estimates
+ // below exclude this whole set so they match what cleanAudioFromTranscribed
+ // will actually remove.
+ const protectedFromCleanup = new Set<string>([...doNotCleanIds, ...keptIds]);
const noTranscript: string[] = [];
const downloadedNoTranscript: string[] = [];
@@ -355,7 +373,7 @@ export async function generateChannelSnapshot(
// Reclaim estimate for the wrong-format sweep: every non-target audio file
// in a cleanable (not do-not-clean) dir. Covers orphans (no target) and the
// extras counted in multipleAudioFormats — the sweep removes them all.
- if (targetAudioFile && !doNotCleanIds.has(id)) {
+ if (targetAudioFile && !protectedFromCleanup.has(id)) {
for (const name of files.audioFiles) {
if (name !== targetAudioFile) {
foreignAudioBytes += audioSizes[name] ?? 0;
@@ -366,7 +384,7 @@ export async function generateChannelSnapshot(
targetAudioFile &&
files.audioFiles.includes(targetAudioFile) &&
files.audioFiles.length > 1 &&
- !doNotCleanIds.has(id)
+ !protectedFromCleanup.has(id)
) {
multipleAudioFormats.push(id);
for (const name of files.audioFiles) {
@@ -378,7 +396,7 @@ export async function generateChannelSnapshot(
if (
files.hasWhisper &&
files.audioFiles.length > 0 &&
- !doNotCleanIds.has(id)
+ !protectedFromCleanup.has(id)
) {
transcribedWithAudio.push(id);
for (const name of files.audioFiles) {
@@ -490,6 +508,7 @@ export async function generateChannelSnapshot(
},
undownloadedIds,
excludedFromDownload,
+ keptCount: keptIds.size,
availability,
...(maybeMissing ? { maybeMissing } : {}),
cleanupBytes: {
diff --git a/common/controller/checkKeptDeleted.ts b/common/controller/checkKeptDeleted.ts
@@ -0,0 +1,97 @@
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import { EXCLUDED_FROM_DOWNLOAD, type Availability } from "../lib/availability";
+import { resolveEffectiveAvailability } from "../lib/availability-server";
+import { loadDoNotClean, setDoNotClean } from "../lib/doNotClean-server";
+import { readChannelConfig } from "./channels";
+import { computeKeptVideoIds } from "./keptVideos";
+import { runAvailabilityCheck } from "./checkAvailability";
+
+// Periodic "are my kept videos still there?" pass. For a channel's keep-latest
+// window, re-probe source availability and pin (do-not-clean) any video found
+// permanently gone (deleted / private / members_only). A pinned video survives
+// even after it rolls out of the rolling window — it's now irreplaceable, so we
+// keep its media forever. Reuses runAvailabilityCheck (onlyIds = the kept set)
+// for the actual yt-dlp probing.
+
+export type CheckKeptDeletedOptions = {
+ channelSlug: string;
+ paths: Paths;
+ onLog?: (msg: string) => void;
+ signal?: AbortSignal;
+ concurrency?: number;
+};
+
+export type CheckKeptDeletedResult = {
+ // Size of the keep-latest window inspected.
+ kept: number;
+ // Videos that probed as permanently gone (EXCLUDED_FROM_DOWNLOAD).
+ deleted: number;
+ // Videos newly pinned with the do-not-clean marker this run.
+ pinned: number;
+};
+
+function isGone(a: Availability | null): boolean {
+ return (
+ a !== null &&
+ (EXCLUDED_FROM_DOWNLOAD as ReadonlyArray<Availability>).includes(a)
+ );
+}
+
+export async function checkKeptDeleted({
+ channelSlug,
+ paths,
+ onLog,
+ signal,
+ concurrency,
+}: CheckKeptDeletedOptions): Promise<CheckKeptDeletedResult> {
+ const log = onLog ?? ((m: string) => console.log(m));
+ const config = await readChannelConfig(paths, channelSlug);
+ const keepLatest = config?.keepLatest ?? 0;
+ if (keepLatest <= 0) {
+ log(`${channelSlug}: keep-latest disabled — nothing to check.`);
+ return { kept: 0, deleted: 0, pinned: 0 };
+ }
+
+ const keptIds = await computeKeptVideoIds({ paths, channelSlug, keepLatest });
+ if (keptIds.size === 0) {
+ log(`${channelSlug}: no videos in the keep-latest window.`);
+ return { kept: 0, deleted: 0, pinned: 0 };
+ }
+ const onlyIds = [...keptIds];
+ log(`${channelSlug}: checking ${onlyIds.length} kept video(s) for deletion…`);
+
+ // Probe only the kept set, re-checking everything not already known-gone.
+ await runAvailabilityCheck({
+ channelSlug,
+ paths,
+ mode: "recheck-non-deleted",
+ onlyIds,
+ concurrency,
+ onLog: log,
+ signal,
+ });
+
+ const dataDir = path.join(paths.channelsDir, channelSlug, "data");
+ let deleted = 0;
+ let pinned = 0;
+ for (const id of onlyIds) {
+ if (signal?.aborted) break;
+ const videoDir = path.join(dataDir, id);
+ const availability = await resolveEffectiveAvailability(videoDir);
+ if (!isGone(availability)) continue;
+ deleted++;
+ // Only count/log a NEW pin — leave an existing marker (and its note) intact.
+ const already = await loadDoNotClean(videoDir);
+ if (already) continue;
+ const note = `deleted from source (keep-latest): ${availability}`;
+ await setDoNotClean(videoDir, true, note);
+ pinned++;
+ log(`Pinned ${id} as do-not-clean (${availability}).`);
+ }
+
+ log(
+ `${channelSlug}: ${deleted} of ${onlyIds.length} kept video(s) gone from source; pinned ${pinned}.`,
+ );
+ return { kept: onlyIds.length, deleted, pinned };
+}
diff --git a/common/controller/cleanAudioFromTranscribed.ts b/common/controller/cleanAudioFromTranscribed.ts
@@ -2,6 +2,9 @@ import path from "node:path";
import fs from "fs-extra";
import type { Paths } from "../lib/paths";
import { isDoNotClean } from "../lib/doNotClean-server";
+import { readChannelConfig } from "./channels";
+import { computeKeptVideoIds } from "./keptVideos";
+import { pruneSavedVideos } from "./pruneSavedVideos";
const { pathExists, readdir, remove } = fs;
@@ -17,6 +20,10 @@ export type CleanAudioResult = {
cleanedDirs: number;
removedFiles: number;
skipped: number;
+ // Saved-store containers evicted because they rolled out of the keep-latest
+ // window (the retention prune runs alongside the audio sweep).
+ prunedSavedVideos: number;
+ prunedBytes: number;
};
export async function cleanAudioFromTranscribed({
@@ -29,10 +36,26 @@ export async function cleanAudioFromTranscribed({
const dataDir = path.join(paths.channelsDir, channelSlug, "data");
if (!(await pathExists(dataDir))) {
log(`No data directory for ${channelSlug}`);
- return { inspected: 0, cleanedDirs: 0, removedFiles: 0, skipped: 0 };
+ return {
+ inspected: 0,
+ cleanedDirs: 0,
+ removedFiles: 0,
+ skipped: 0,
+ prunedSavedVideos: 0,
+ prunedBytes: 0,
+ };
}
const dirs = await readdir(dataDir);
+ // The rolling keep-latest window is protected from cleanup just like the
+ // explicit do-not-clean marker (the source media of recent videos is kept).
+ const config = await readChannelConfig(paths, channelSlug);
+ const keptIds = await computeKeptVideoIds({
+ paths,
+ channelSlug,
+ keepLatest: config?.keepLatest ?? 0,
+ });
+
let cleanedDirs = 0;
let removedFiles = 0;
let skipped = 0;
@@ -50,6 +73,11 @@ export async function cleanAudioFromTranscribed({
!e.endsWith(".part"),
);
if (audioFiles.length === 0) continue;
+ if (keptIds.has(id)) {
+ log(`Skipped ${id} (in keep-latest window)`);
+ skipped++;
+ continue;
+ }
if (await isDoNotClean(videoDir)) {
log(`Skipped ${id} (marked do not clean)`);
skipped++;
@@ -67,5 +95,17 @@ export async function cleanAudioFromTranscribed({
log(
`Cleaned ${removedFiles} audio file(s) from ${cleanedDirs} of ${dirs.length} video dir(s).${skippedNote}`,
);
- return { inspected: dirs.length, cleanedDirs, removedFiles, skipped };
+
+ // Retention prune of the saved-video store: evict any keep-latest containers
+ // that have rolled out of the window (pinned/manually-archived ones survive).
+ const prune = await pruneSavedVideos({ channelSlug, paths, onLog: log, signal });
+
+ return {
+ inspected: dirs.length,
+ cleanedDirs,
+ removedFiles,
+ skipped,
+ prunedSavedVideos: prune.pruned,
+ prunedBytes: prune.bytesFreed,
+ };
}
diff --git a/common/controller/keptVideos.test.ts b/common/controller/keptVideos.test.ts
@@ -0,0 +1,168 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import {
+ computeKeepWindow,
+ computeKeptVideoIds,
+ isInKeepWindow,
+} from "./keptVideos";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test common/controller/keptVideos.test.ts
+
+// Seed channels/<slug>/data/<id>/metadata.info.json for each [id, uploadDate].
+// A null uploadDate omits the metadata file so the dir-name fallback is exercised.
+async function seedChannel(
+ channelsDir: string,
+ slug: string,
+ videos: ReadonlyArray<[string, string | null]>,
+): Promise<void> {
+ for (const [id, uploadDate] of videos) {
+ const dir = path.join(channelsDir, slug, "data", id);
+ await mkdir(dir, { recursive: true });
+ if (uploadDate !== null) {
+ await writeFile(
+ path.join(dir, "metadata.info.json"),
+ JSON.stringify({ id, upload_date: uploadDate }),
+ );
+ }
+ }
+}
+
+async function withPaths(fn: (paths: Paths) => Promise<void>): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-kept-"));
+ const paths = { channelsDir: path.join(dir, "channels") } as Paths;
+ try {
+ await fn(paths);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+test("keeps the newest N by upload_date", async () => {
+ await withPaths(async (paths) => {
+ await seedChannel(paths.channelsDir, "ch", [
+ ["a", "20240101"],
+ ["b", "20240301"],
+ ["c", "20240201"],
+ ["d", "20231231"],
+ ]);
+ const kept = await computeKeptVideoIds({
+ paths,
+ channelSlug: "ch",
+ keepLatest: 2,
+ });
+ assert.deepEqual([...kept].sort(), ["b", "c"]);
+ });
+});
+
+test("keepLatest <= 0 keeps nothing", async () => {
+ await withPaths(async (paths) => {
+ await seedChannel(paths.channelsDir, "ch", [["a", "20240101"]]);
+ assert.equal(
+ (await computeKeptVideoIds({ paths, channelSlug: "ch", keepLatest: 0 }))
+ .size,
+ 0,
+ );
+ });
+});
+
+test("keepLatest larger than the catalog keeps all", async () => {
+ await withPaths(async (paths) => {
+ await seedChannel(paths.channelsDir, "ch", [
+ ["a", "20240101"],
+ ["b", "20240201"],
+ ]);
+ const kept = await computeKeptVideoIds({
+ paths,
+ channelSlug: "ch",
+ keepLatest: 10,
+ });
+ assert.deepEqual([...kept].sort(), ["a", "b"]);
+ });
+});
+
+test("falls back to a YYYYMMDD_ dir-name prefix when metadata is absent", async () => {
+ await withPaths(async (paths) => {
+ await seedChannel(paths.channelsDir, "ch", [
+ ["20240301_newest", null],
+ ["20240101_older", null],
+ ["nodate", null],
+ ]);
+ const kept = await computeKeptVideoIds({
+ paths,
+ channelSlug: "ch",
+ keepLatest: 1,
+ });
+ assert.deepEqual([...kept], ["20240301_newest"]);
+ });
+});
+
+test("missing channel data dir yields an empty set", async () => {
+ await withPaths(async (paths) => {
+ assert.equal(
+ (
+ await computeKeptVideoIds({
+ paths,
+ channelSlug: "ghost",
+ keepLatest: 5,
+ })
+ ).size,
+ 0,
+ );
+ });
+});
+
+test("keep window: an under-full channel admits any candidate", async () => {
+ await withPaths(async (paths) => {
+ await seedChannel(paths.channelsDir, "ch", [["a", "20240101"]]);
+ const w = await computeKeepWindow({
+ paths,
+ channelSlug: "ch",
+ keepLatest: 3,
+ });
+ assert.equal(w.full, false);
+ assert.equal(w.cutoffKey, null);
+ // A brand-new (not-yet-on-disk) video is admitted regardless of its date.
+ assert.equal(isInKeepWindow("20200101", w), true);
+ });
+});
+
+test("keep window: a full channel admits only candidates >= the cutoff", async () => {
+ await withPaths(async (paths) => {
+ await seedChannel(paths.channelsDir, "ch", [
+ ["a", "20240101"],
+ ["b", "20240301"],
+ ["c", "20240201"],
+ ]);
+ const w = await computeKeepWindow({
+ paths,
+ channelSlug: "ch",
+ keepLatest: 2,
+ });
+ assert.equal(w.full, true);
+ // Newest two are 20240301, 20240201 -> the 2nd-newest (cutoff) is 20240201.
+ assert.equal(w.cutoffKey, "20240201");
+ // A newer upload displaces the boundary video -> kept.
+ assert.equal(isInKeepWindow("20240401", w), true);
+ // A tie with the cutoff is kept.
+ assert.equal(isInKeepWindow("20240201", w), true);
+ // An older upload stays out of the window.
+ assert.equal(isInKeepWindow("20240101", w), false);
+ });
+});
+
+test("keep window: keepLatest <= 0 is inert", async () => {
+ await withPaths(async (paths) => {
+ await seedChannel(paths.channelsDir, "ch", [["a", "20240101"]]);
+ const w = await computeKeepWindow({
+ paths,
+ channelSlug: "ch",
+ keepLatest: 0,
+ });
+ assert.equal(isInKeepWindow("99999999", w), false);
+ });
+});
diff --git a/common/controller/keptVideos.ts b/common/controller/keptVideos.ts
@@ -0,0 +1,129 @@
+import path from "node:path";
+import { readdir } from "node:fs/promises";
+import type { Paths } from "../lib/paths";
+import { loadRawMetadataFromDir } from "../lib/transcripts-server";
+
+// Rolling "keep-latest" window computation. Given a channel's keepLatest config,
+// returns the ids of the newest N videos (by upload date). Used by both the
+// cleanup protection (cleanAudioFromTranscribed / channelSnapshot) and the
+// per-download persistence rule (downloadOneManaged). Kept pure-ish (just fs
+// reads) so it can be reused everywhere the kept set is needed.
+
+export type ComputeKeptOptions = {
+ paths: Paths;
+ channelSlug: string;
+ keepLatest: number;
+};
+
+// List a channel's data-dir video ids (directories, skipping dotfiles). Mirrors
+// editor's readDataDirVideoIds, duplicated here so common doesn't depend on the
+// editor's server-only module.
+export async function listChannelVideoIds(
+ paths: Paths,
+ channelSlug: string,
+): Promise<string[]> {
+ const dataDir = path.join(paths.channelsDir, channelSlug, "data");
+ try {
+ const entries = await readdir(dataDir, { withFileTypes: true });
+ return entries
+ .filter((e) => e.isDirectory() && !e.name.startsWith("."))
+ .map((e) => e.name);
+ } catch {
+ return [];
+ }
+}
+
+// Recency sort key for a video given its already-known upload_date: the
+// YYYYMMDD string, falling back to a YYYYMMDD_ prefix on the id, else "" (sorts
+// oldest). Shared so the live cutoff test for a not-yet-downloaded video keys it
+// the same way the on-disk window does.
+export function uploadKeyFor(
+ uploadDate: string | undefined,
+ id: string,
+): string {
+ if (uploadDate) return uploadDate;
+ return id.match(/^(\d{8})(?:_|$)/)?.[1] ?? "";
+}
+
+// Recency sort key for a video: upload_date (YYYYMMDD) from metadata.info.json,
+// falling back to a YYYYMMDD_ prefix on the dir name, else "" (sorts oldest).
+async function uploadKey(videoDir: string, id: string): Promise<string> {
+ const meta = await loadRawMetadataFromDir(videoDir);
+ return uploadKeyFor(meta?.upload_date, id);
+}
+
+// Sort a channel's data-dir videos newest-first by upload key (ties broken by id
+// descending for determinism). Shared by computeKeptVideoIds (the on-disk window)
+// and computeKeepWindow (the live cutoff used at download time).
+async function keyedVideosNewestFirst(
+ paths: Paths,
+ channelSlug: string,
+): Promise<Array<{ id: string; key: string }>> {
+ const ids = await listChannelVideoIds(paths, channelSlug);
+ if (ids.length === 0) return [];
+ const dataDir = path.join(paths.channelsDir, channelSlug, "data");
+ const keyed = await Promise.all(
+ ids.map(async (id) => ({
+ id,
+ key: await uploadKey(path.join(dataDir, id), id),
+ })),
+ );
+ keyed.sort((a, b) =>
+ a.key === b.key ? b.id.localeCompare(a.id) : b.key.localeCompare(a.key),
+ );
+ return keyed;
+}
+
+// The set of the newest `keepLatest` video ids for a channel (by upload date,
+// newest first; ties broken by id descending for determinism). Empty when
+// keepLatest <= 0 or the channel has no data dir. Used by cleanup/snapshot,
+// which see every kept video already on disk.
+export async function computeKeptVideoIds({
+ paths,
+ channelSlug,
+ keepLatest,
+}: ComputeKeptOptions): Promise<Set<string>> {
+ if (!Number.isFinite(keepLatest) || keepLatest <= 0) return new Set();
+ const keyed = await keyedVideosNewestFirst(paths, channelSlug);
+ return new Set(keyed.slice(0, Math.floor(keepLatest)).map((k) => k.id));
+}
+
+// The keep-latest window expressed as a CUTOFF rather than a fixed id set. This
+// is what the per-download rule needs: at download time the newest videos aren't
+// on disk yet, so membership can't come from listing dirs. Instead we capture
+// - cutoffKey: the upload key of the Nth-newest video currently on disk
+// - full: whether the channel already holds >= keepLatest videos
+// and test a candidate by its own upload key (see isInKeepWindow). A candidate
+// newer than the current Nth would displace it, so it belongs in the window.
+export type KeepWindow = {
+ keepLatest: number;
+ cutoffKey: string | null;
+ full: boolean;
+};
+
+export async function computeKeepWindow({
+ paths,
+ channelSlug,
+ keepLatest,
+}: ComputeKeptOptions): Promise<KeepWindow> {
+ const n = Number.isFinite(keepLatest) ? Math.floor(keepLatest) : 0;
+ if (n <= 0) return { keepLatest: 0, cutoffKey: null, full: false };
+ const keyed = await keyedVideosNewestFirst(paths, channelSlug);
+ const full = keyed.length >= n;
+ const cutoffKey = full ? keyed[n - 1].key : null;
+ return { keepLatest: n, cutoffKey, full };
+}
+
+// Whether a candidate video (identified by its own upload key) belongs in the
+// keep-latest window. With fewer than N videos on disk the window isn't full, so
+// any candidate is kept; once full, a candidate is kept iff it's at least as new
+// as the current Nth-newest (a tie or newer displaces the boundary video).
+export function isInKeepWindow(
+ uploadKey: string,
+ window: KeepWindow | undefined,
+): boolean {
+ if (!window || window.keepLatest <= 0) return false;
+ if (!window.full) return true;
+ if (window.cutoffKey == null) return true;
+ return uploadKey >= window.cutoffKey;
+}
diff --git a/common/controller/persistKept.test.ts b/common/controller/persistKept.test.ts
@@ -0,0 +1,84 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import { persistSourceVideo } from "../lib/savedVideo-server";
+import { persistKept } from "./persistKept";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/persistKept.test.ts
+//
+// These cover the orchestration paths that don't reach downloadOneManaged (no
+// network): keep-latest off, and an all-already-saved window. The download path
+// is exercised via redownloadToArchiveAction in the editor e2e suite.
+
+async function withPaths(fn: (paths: Paths) => Promise<void>): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-persistkept-"));
+ const paths = {
+ channelsDir: path.join(dir, "channels"),
+ savedVideosDir: path.join(dir, "saved"),
+ } as Paths;
+ try {
+ await fn(paths);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+// Seed a data dir whose dir name carries a YYYYMMDD_ recency prefix (so
+// computeKeptVideoIds can order it without metadata) and persist its container.
+async function seedSaved(
+ paths: Paths,
+ slug: string,
+ id: string,
+): Promise<void> {
+ const videoDir = path.join(paths.channelsDir, slug, "data", id);
+ await mkdir(videoDir, { recursive: true });
+ await writeFile(path.join(videoDir, "source-media.mp4"), id);
+ await persistSourceVideo({
+ videoDir,
+ sourceFilename: "source-media.mp4",
+ storeDir: path.join(paths.savedVideosDir, slug, id),
+ keepReason: "keep-latest",
+ });
+}
+
+test("persistKept is a no-op when keep-latest is off", async () => {
+ await withPaths(async (paths) => {
+ await seedSaved(paths, "alpha", "20240101_a");
+ const res = await persistKept({
+ paths,
+ channelSlug: "alpha",
+ channelConfig: { handling: "transcribe" },
+ });
+ assert.deepEqual(res, {
+ kept: 0,
+ alreadySaved: 0,
+ persisted: 0,
+ failed: 0,
+ skippedNoUrl: 0,
+ });
+ });
+});
+
+test("persistKept counts in-window videos already saved without re-downloading", async () => {
+ await withPaths(async (paths) => {
+ await seedSaved(paths, "alpha", "20240101_a");
+ await seedSaved(paths, "alpha", "20240202_b");
+ await seedSaved(paths, "alpha", "20240303_c");
+ // Window of 2 -> the two newest (c, b) are kept; both are already saved, so
+ // no download is attempted (which would fail in this offline test).
+ const res = await persistKept({
+ paths,
+ channelSlug: "alpha",
+ channelConfig: { handling: "transcribe", keepLatest: 2 },
+ });
+ assert.equal(res.kept, 2);
+ assert.equal(res.alreadySaved, 2);
+ assert.equal(res.persisted, 0);
+ assert.equal(res.failed, 0);
+ assert.equal(res.skippedNoUrl, 0);
+ });
+});
diff --git a/common/controller/persistKept.ts b/common/controller/persistKept.ts
@@ -0,0 +1,119 @@
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import type { ChannelConfig } from "../lib/channelConfig";
+import { getSettings } from "../lib/settings";
+import { isSavedVideo } from "../lib/savedVideo-server";
+import { computeKeptVideoIds } from "./keptVideos";
+import { findVideoSourceUrl } from "./undownloadedVideos";
+import { downloadOneManaged } from "../ytdlp/downloadOneManaged";
+
+// Bulk "persist kept now" pass (Phase 5). Ensures every video currently in the
+// channel's keep-latest window has its source container saved to the store, for
+// the catch-up case where the window was widened (or persistence was enabled)
+// after those videos had already been downloaded audio-only.
+//
+// For each kept video whose source isn't saved yet, it re-fetches the container
+// via downloadOneManaged with keepSourceVideoOverride (which app-extracts audio
+// and moves the container into the saved store), without disturbing the existing
+// transcript. Mirrors redownloadToArchiveAction (videoActions.ts) but loops over
+// the whole window. Videos already saved, or whose URL can't be resolved, are
+// skipped rather than failing the pass.
+
+export type PersistKeptResult = {
+ // Total videos in the keep-latest window.
+ kept: number;
+ // Already had a saved-video pointer; left untouched.
+ alreadySaved: number;
+ // Newly persisted this pass.
+ persisted: number;
+ // Re-download attempted but failed.
+ failed: number;
+ // No resolvable source URL (no metadata + not in the playlist).
+ skippedNoUrl: number;
+};
+
+export async function persistKept({
+ paths,
+ channelSlug,
+ channelConfig,
+ onLog,
+ signal,
+}: {
+ paths: Paths;
+ channelSlug: string;
+ channelConfig: ChannelConfig;
+ onLog?: (line: string) => void;
+ signal?: AbortSignal;
+}): Promise<PersistKeptResult> {
+ const log = (line: string) => onLog?.(line.endsWith("\n") ? line : `${line}\n`);
+ // downloadOneManaged requires a non-optional onLog/signal; supply inert
+ // fallbacks so persistKept stays callable without a managed-job context.
+ const downloadLog = (line: string) => onLog?.(line);
+ const downloadSignal = signal ?? new AbortController().signal;
+ const keepLatest = channelConfig.keepLatest ?? 0;
+ const keptIds = await computeKeptVideoIds({
+ paths,
+ channelSlug,
+ keepLatest,
+ });
+ const result: PersistKeptResult = {
+ kept: keptIds.size,
+ alreadySaved: 0,
+ persisted: 0,
+ failed: 0,
+ skippedNoUrl: 0,
+ };
+ if (keptIds.size === 0) {
+ log(
+ keepLatest > 0
+ ? "Persist kept: no videos in the keep-latest window yet."
+ : "Persist kept: keep-latest is off for this channel; nothing to do.",
+ );
+ return result;
+ }
+ const settings = getSettings();
+ const dataDir = path.join(paths.channelsDir, channelSlug, "data");
+ // Newest-first so the freshest videos are secured even if the pass is cancelled
+ // partway through. computeKeptVideoIds returns an unordered set; sort by id desc
+ // as a stable proxy (YYYYMMDD_-prefixed ids sort by recency, like the window).
+ const ordered = [...keptIds].sort((a, b) => b.localeCompare(a));
+ for (const videoId of ordered) {
+ if (signal?.aborted) break;
+ const videoDir = path.join(dataDir, videoId);
+ if (await isSavedVideo(videoDir)) {
+ result.alreadySaved += 1;
+ continue;
+ }
+ const url = await findVideoSourceUrl(paths, channelSlug, videoId, channelConfig);
+ if (!url) {
+ result.skippedNoUrl += 1;
+ log(` ${videoId}: no resolvable source URL, skipping.`);
+ continue;
+ }
+ log(` ${videoId}: re-fetching source container to persist…`);
+ try {
+ await downloadOneManaged({
+ channelSlug,
+ channelConfig,
+ paths,
+ videoUrl: url,
+ onLog: downloadLog,
+ signal: downloadSignal,
+ globalCookiesFromBrowser: settings.cookiesFromBrowser || undefined,
+ inlineTranscribeOnFallback: settings.inlineTranscribeOnFallback,
+ globalSkipLiveDownloads: settings.skipLiveDownloads,
+ appendArchive: true,
+ keepSourceVideoOverride: true,
+ });
+ result.persisted += 1;
+ } catch (e) {
+ result.failed += 1;
+ log(` ${videoId}: persist failed — ${(e as Error).message}`);
+ }
+ }
+ log(
+ `Persist kept: ${result.persisted} persisted, ${result.alreadySaved} already saved, ` +
+ `${result.failed} failed, ${result.skippedNoUrl} skipped (no URL) of ${result.kept} kept.`,
+ );
+ return result;
+}
diff --git a/common/controller/pruneSavedVideos.test.ts b/common/controller/pruneSavedVideos.test.ts
@@ -0,0 +1,107 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import { persistSourceVideo, loadSavedVideo } from "../lib/savedVideo-server";
+import { setDoNotClean } from "../lib/doNotClean-server";
+import { pruneSavedVideos } from "./pruneSavedVideos";
+import type { SavedVideoKeepReason } from "../lib/savedVideo";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/pruneSavedVideos.test.ts
+
+async function withPaths(fn: (paths: Paths) => Promise<void>): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-prune-"));
+ const paths = {
+ channelsDir: path.join(dir, "channels"),
+ savedVideosDir: path.join(dir, "saved"),
+ } as Paths;
+ try {
+ await fn(paths);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+// Seed a data/<id> dir with an upload date + a persisted source container.
+async function seedSaved(
+ paths: Paths,
+ slug: string,
+ id: string,
+ uploadDate: string,
+ keepReason: SavedVideoKeepReason,
+): Promise<void> {
+ const videoDir = path.join(paths.channelsDir, slug, "data", id);
+ await mkdir(videoDir, { recursive: true });
+ await writeFile(
+ path.join(videoDir, "metadata.info.json"),
+ JSON.stringify({ id, upload_date: uploadDate }),
+ );
+ await writeFile(path.join(videoDir, "source-media.mp4"), `${id}-bytes`);
+ await persistSourceVideo({
+ videoDir,
+ sourceFilename: "source-media.mp4",
+ storeDir: path.join(paths.savedVideosDir, slug, id),
+ keepReason,
+ });
+}
+
+async function writeConfig(
+ paths: Paths,
+ slug: string,
+ keepLatest: number,
+): Promise<void> {
+ const dir = path.join(paths.channelsDir, slug);
+ await mkdir(dir, { recursive: true });
+ await writeFile(
+ path.join(dir, "config.json"),
+ JSON.stringify({ handling: "transcribe", keepLatest }),
+ );
+}
+
+test("prunes only keep-latest containers that rolled out of the window", async () => {
+ await withPaths(async (paths) => {
+ await writeConfig(paths, "ch", 2);
+ // Newest two (by date): c, b -> kept. a is the rolled-out keep-latest one.
+ await seedSaved(paths, "ch", "a", "20240101", "keep-latest"); // evict
+ await seedSaved(paths, "ch", "b", "20240201", "keep-latest"); // in window
+ await seedSaved(paths, "ch", "c", "20240301", "keep-latest"); // in window
+
+ const res = await pruneSavedVideos({ channelSlug: "ch", paths });
+ assert.equal(res.scanned, 3);
+ assert.equal(res.pruned, 1);
+ assert.equal(await loadSavedVideo(path.join(paths.channelsDir, "ch", "data", "a")), null);
+ assert.notEqual(await loadSavedVideo(path.join(paths.channelsDir, "ch", "data", "b")), null);
+ assert.notEqual(await loadSavedVideo(path.join(paths.channelsDir, "ch", "data", "c")), null);
+ });
+});
+
+test("never evicts override or pinned containers, even out of window", async () => {
+ await withPaths(async (paths) => {
+ await writeConfig(paths, "ch", 1);
+ await seedSaved(paths, "ch", "newest", "20240301", "keep-latest");
+ // Out-of-window manual archive -> kept.
+ await seedSaved(paths, "ch", "archived", "20240101", "override");
+ // Out-of-window pin-category container -> kept.
+ await seedSaved(paths, "ch", "pinptr", "20240102", "pin");
+ // Out-of-window keep-latest, but separately do-not-clean pinned -> kept.
+ await seedSaved(paths, "ch", "deleted", "20240103", "keep-latest");
+ await setDoNotClean(
+ path.join(paths.channelsDir, "ch", "data", "deleted"),
+ true,
+ "deleted from source",
+ );
+
+ const res = await pruneSavedVideos({ channelSlug: "ch", paths });
+ assert.equal(res.pruned, 0);
+ for (const id of ["archived", "pinptr", "deleted"]) {
+ assert.notEqual(
+ await loadSavedVideo(path.join(paths.channelsDir, "ch", "data", id)),
+ null,
+ `${id} should be kept`,
+ );
+ }
+ });
+});
diff --git a/common/controller/pruneSavedVideos.ts b/common/controller/pruneSavedVideos.ts
@@ -0,0 +1,79 @@
+import path from "node:path";
+import fs from "fs-extra";
+import type { Paths } from "../lib/paths";
+import { isDoNotClean } from "../lib/doNotClean-server";
+import { readChannelConfig } from "./channels";
+import { computeKeptVideoIds } from "./keptVideos";
+import { dropSavedVideo, loadSavedVideo } from "../lib/savedVideo-server";
+
+const { pathExists, readdir } = fs;
+
+export type PruneSavedVideosResult = {
+ // Dirs carrying a saved-video pointer that were considered.
+ scanned: number;
+ // Window-persisted containers dropped (rolled out, unpinned).
+ pruned: number;
+ bytesFreed: number;
+};
+
+// Retention prune of the saved-video store. A persisted source video is removed
+// from the store only when ALL of these hold:
+// - it was persisted by the rolling keep-latest rule (pointer keepReason
+// "keep-latest") — NOT a manual archive ("override") or a pin ("pin"),
+// - its id has rolled OUT of the channel's current keep-latest window, and
+// - it is not pinned via do-not-clean (a kept video later deleted-from-source
+// is pinned and must survive forever).
+// This is what bounds the store's growth as new uploads displace old ones, and it
+// is the Phase-3 resolution of the Phase-2 "source media not auto-pruned" caveat.
+// Manually-archived and pinned containers are never auto-evicted.
+export async function pruneSavedVideos({
+ channelSlug,
+ paths,
+ onLog,
+ signal,
+}: {
+ channelSlug: string;
+ paths: Paths;
+ onLog?: (msg: string) => void;
+ signal?: AbortSignal;
+}): Promise<PruneSavedVideosResult> {
+ const log = onLog ?? ((m: string) => console.log(m));
+ const dataDir = path.join(paths.channelsDir, channelSlug, "data");
+ if (!(await pathExists(dataDir))) {
+ return { scanned: 0, pruned: 0, bytesFreed: 0 };
+ }
+ const config = await readChannelConfig(paths, channelSlug);
+ const keptIds = await computeKeptVideoIds({
+ paths,
+ channelSlug,
+ keepLatest: config?.keepLatest ?? 0,
+ });
+ const dirs = await readdir(dataDir);
+
+ let scanned = 0;
+ let pruned = 0;
+ let bytesFreed = 0;
+ for (const id of dirs) {
+ if (signal?.aborted) break;
+ const videoDir = path.join(dataDir, id);
+ const pointer = await loadSavedVideo(videoDir);
+ if (!pointer) continue;
+ scanned++;
+ // Only the rolling keep-latest containers are eligible for auto-eviction.
+ if (pointer.keepReason !== "keep-latest") continue;
+ if (keptIds.has(id)) continue; // still inside the window
+ if (await isDoNotClean(videoDir)) continue; // pinned -> keep forever
+ const bytes = await dropSavedVideo(videoDir);
+ bytesFreed += bytes;
+ pruned++;
+ log(
+ `Pruned saved video ${id} (rolled out of keep-latest window; freed ${bytes} bytes)`,
+ );
+ }
+ if (pruned > 0) {
+ log(
+ `Pruned ${pruned} saved video(s) from the store (freed ${bytesFreed} bytes).`,
+ );
+ }
+ return { scanned, pruned, bytesFreed };
+}
diff --git a/common/controller/savedVideoInventory.test.ts b/common/controller/savedVideoInventory.test.ts
@@ -0,0 +1,67 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import { persistSourceVideo } from "../lib/savedVideo-server";
+import { listSavedVideos, savedVideoTotals } from "./savedVideoInventory";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/savedVideoInventory.test.ts
+
+async function withPaths(fn: (paths: Paths) => Promise<void>): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-inv-"));
+ const paths = {
+ channelsDir: path.join(dir, "channels"),
+ savedVideosDir: path.join(dir, "saved"),
+ } as Paths;
+ try {
+ await fn(paths);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+async function seed(
+ paths: Paths,
+ slug: string,
+ id: string,
+ content: string,
+): Promise<void> {
+ const videoDir = path.join(paths.channelsDir, slug, "data", id);
+ await mkdir(videoDir, { recursive: true });
+ await writeFile(path.join(videoDir, "source-media.mp4"), content);
+ await persistSourceVideo({
+ videoDir,
+ sourceFilename: "source-media.mp4",
+ storeDir: path.join(paths.savedVideosDir, slug, id),
+ keepReason: "keep-latest",
+ });
+}
+
+test("listSavedVideos enumerates every pointer across channels, sorted", async () => {
+ await withPaths(async (paths) => {
+ await seed(paths, "beta", "v2", "22");
+ await seed(paths, "alpha", "v1", "1");
+ await seed(paths, "alpha", "v0", "000");
+ // A data dir with no pointer is ignored.
+ await mkdir(path.join(paths.channelsDir, "alpha", "data", "plain"), {
+ recursive: true,
+ });
+
+ const all = await listSavedVideos({ paths });
+ assert.deepEqual(
+ all.map((e) => `${e.slug}/${e.videoId}`),
+ ["alpha/v0", "alpha/v1", "beta/v2"],
+ );
+ assert.equal(all[0].pointer.file, "source-media.mp4");
+
+ const onlyAlpha = await listSavedVideos({ paths, channelSlug: "alpha" });
+ assert.equal(onlyAlpha.length, 2);
+
+ const totals = await savedVideoTotals({ paths });
+ assert.equal(totals.count, 3);
+ assert.equal(totals.bytes, "22".length + "1".length + "000".length);
+ });
+});
diff --git a/common/controller/savedVideoInventory.ts b/common/controller/savedVideoInventory.ts
@@ -0,0 +1,81 @@
+import path from "node:path";
+import fs from "fs-extra";
+import type { Paths } from "../lib/paths";
+import { loadSavedVideo } from "../lib/savedVideo-server";
+import { savedVideoPath, type SavedVideoPointer } from "../lib/savedVideo";
+
+const { pathExists, readdir } = fs;
+
+// One persisted source video, located via its data-dir pointer. Shared by the
+// backup/verify controllers (Phase 4) and the saved-videos UI counts (Phase 5).
+export type SavedVideoEntry = {
+ slug: string;
+ videoId: string;
+ // The per-video data dir holding the pointer (channels/<slug>/data/<id>).
+ videoDir: string;
+ // Absolute path to the stored container in the saved-video store.
+ storedPath: string;
+ pointer: SavedVideoPointer;
+};
+
+async function listChannelSlugs(paths: Paths): Promise<string[]> {
+ try {
+ const entries = await readdir(paths.channelsDir, { withFileTypes: true });
+ return entries.filter((e) => e.isDirectory()).map((e) => e.name);
+ } catch {
+ return [];
+ }
+}
+
+// Enumerate every persisted source video, across all channels (or a single
+// channel when channelSlug is given), by following the saved-video.json pointers
+// in each data dir. The pointer's `dir` is absolute, so per-channel store
+// overrides resolve correctly without consulting channel config here.
+export async function listSavedVideos({
+ paths,
+ channelSlug,
+}: {
+ paths: Paths;
+ channelSlug?: string;
+}): Promise<SavedVideoEntry[]> {
+ const slugs = channelSlug ? [channelSlug] : await listChannelSlugs(paths);
+ const out: SavedVideoEntry[] = [];
+ for (const slug of slugs) {
+ const dataDir = path.join(paths.channelsDir, slug, "data");
+ if (!(await pathExists(dataDir))) continue;
+ const ids = await readdir(dataDir).catch(() => [] as string[]);
+ for (const videoId of ids) {
+ const videoDir = path.join(dataDir, videoId);
+ const pointer = await loadSavedVideo(videoDir);
+ if (!pointer) continue;
+ out.push({
+ slug,
+ videoId,
+ videoDir,
+ storedPath: savedVideoPath(pointer),
+ pointer,
+ });
+ }
+ }
+ out.sort(
+ (a, b) => a.slug.localeCompare(b.slug) || a.videoId.localeCompare(b.videoId),
+ );
+ return out;
+}
+
+export type SavedVideoTotals = {
+ count: number;
+ bytes: number;
+};
+
+// Aggregate counts/sizes for the saved-video store (or one channel). Cheap: it
+// reads only the pointers, not the containers themselves.
+export async function savedVideoTotals(opts: {
+ paths: Paths;
+ channelSlug?: string;
+}): Promise<SavedVideoTotals> {
+ const entries = await listSavedVideos(opts);
+ let bytes = 0;
+ for (const e of entries) bytes += e.pointer.bytes;
+ return { count: entries.length, bytes };
+}
diff --git a/common/controller/transcribeOne.ts b/common/controller/transcribeOne.ts
@@ -10,7 +10,8 @@ import type { TaskTracker } from "../jobs/taskHooks";
import { normalizeTranscript } from "./normalizeTranscript";
import { pingRemoteHealth, transcribeViaRemote } from "./remoteTranscribe";
import { TranscribeError } from "./transcribeError";
-import { isRealAudioFile } from "../lib/videoStatus";
+import { findSourceMedia, isRealAudioFile } from "../lib/videoStatus";
+import { resolveSavedVideo } from "../lib/savedVideo-server";
import { writeTranscribeOutcome } from "../lib/transcribeOutcome-server";
const { pathExists, readdir, rename, writeFile } = fs;
@@ -26,11 +27,25 @@ async function resolveAudioFile(
if (strict) return null;
const entries = await readdir(videoDir);
const candidates = entries.filter(isRealAudioFile);
- if (candidates.length === 0) return null;
- for (const preferred of AUDIO_PREFERENCE) {
- if (candidates.includes(preferred)) return preferred;
+ if (candidates.length > 0) {
+ for (const preferred of AUDIO_PREFERENCE) {
+ if (candidates.includes(preferred)) return preferred;
+ }
+ return [...candidates].sort()[0];
}
- return [...candidates].sort()[0];
+ // No extracted audio on disk — fall back to a persisted source video
+ // container. Transcribers that ffmpeg-slice their input (parakeet) read a
+ // container directly; this is what makes a kept-but-cleaned video, or a
+ // redownload-to-archive that only fetched the container, still transcribable.
+ const container = findSourceMedia(entries);
+ if (container) return container;
+ // The container may have been moved into the saved-video store (Phase 3),
+ // leaving a saved-video.json pointer. Resolve it and return a path relative to
+ // videoDir so the engine (cwd = videoDir) and the remote uploader
+ // (path.join(videoDir, …)) both read the stored file correctly.
+ const saved = await resolveSavedVideo(videoDir);
+ if (saved) return path.relative(videoDir, saved);
+ return null;
}
export type TranscribeOneOptions = {
diff --git a/common/jobs/syncSchedulerState.ts b/common/jobs/syncSchedulerState.ts
@@ -24,6 +24,10 @@ export type ChannelSyncState = {
// Outcome of the last reconciled scheduled sync, for the observability panel.
lastOutcome: "ok" | "failed" | null;
lastOutcomeAt: number | null;
+ // Epoch ms the scheduler last queued a keep-latest deletion check for this
+ // channel. Drives the keepLatestCheckIntervalMinutes cadence independently of
+ // the sync cadence. null = never run.
+ lastKeptCheckAt: number | null;
};
export type SchedulerSkip = { slug: string; reason: string };
@@ -38,6 +42,10 @@ export type SchedulerState = {
channels: Record<string, ChannelSyncState>;
// Newest-first, bounded to SCHEDULER_RUN_LOG_LIMIT entries.
runs: SchedulerRun[];
+ // Epoch ms the scheduler last queued a saved-video backup (a global, not
+ // per-channel, job). Drives the savedVideoBackup.intervalMinutes cadence.
+ // null = never run. See editor/app/scheduler/runTick.ts.
+ lastSavedVideoBackupAt: number | null;
};
export const SCHEDULER_RUN_LOG_LIMIT = 50;
@@ -50,11 +58,12 @@ export function emptyChannelSyncState(): ChannelSyncState {
lastQueuedAt: null,
lastOutcome: null,
lastOutcomeAt: null,
+ lastKeptCheckAt: null,
};
}
export function emptySchedulerState(): SchedulerState {
- return { channels: {}, runs: [] };
+ return { channels: {}, runs: [], lastSavedVideoBackupAt: null };
}
// Read the state file, tolerating a missing/corrupt file by returning an empty
@@ -80,7 +89,7 @@ export async function readSchedulerState(paths: Paths): Promise<SchedulerState>
const runs: SchedulerRun[] = Array.isArray(r.runs)
? r.runs.map(coerceRun).slice(0, SCHEDULER_RUN_LOG_LIMIT)
: [];
- return { channels, runs };
+ return { channels, runs, lastSavedVideoBackupAt: num(r.lastSavedVideoBackupAt) };
}
// Write the state atomically (tmp file + rename), creating the .scheduler dir on
@@ -92,6 +101,7 @@ export async function writeSchedulerState(
const out: SchedulerState = {
channels: state.channels,
runs: state.runs.slice(0, SCHEDULER_RUN_LOG_LIMIT),
+ lastSavedVideoBackupAt: state.lastSavedVideoBackupAt ?? null,
};
await mkdir(path.dirname(paths.schedulerStateFile), { recursive: true });
const tmp = `${paths.schedulerStateFile}.tmp-${process.pid}`;
@@ -139,6 +149,7 @@ function coerceChannelState(value: unknown): ChannelSyncState {
? r.lastOutcome
: null,
lastOutcomeAt: num(r.lastOutcomeAt),
+ lastKeptCheckAt: num(r.lastKeptCheckAt),
};
}
diff --git a/common/lib/channelConfig.ts b/common/lib/channelConfig.ts
@@ -4,6 +4,20 @@ export type ChannelHandling = "youtube" | "transcribe";
export type AudioFormat = "m4a" | "mp3" | "opus";
+// Who owns audio extraction for a transcribe-handling download:
+// "ytdlp" (default) -> yt-dlp's own `-x --audio-format` postprocessor, exactly
+// as before. No source container is kept.
+// "app" -> yt-dlp downloads the source container (no `-x`) and the
+// app runs ffmpeg (transcodeAudio) to produce audio.<fmt>.
+// Required whenever the source video must be kept/inspected
+// (the keep-latest persistence rule forces this).
+export type ExtractionMode = "ytdlp" | "app";
+
+export const EXTRACTION_MODE_VALUES: ReadonlyArray<ExtractionMode> = [
+ "ytdlp",
+ "app",
+];
+
export type AudioCheckConfig = {
enabled: boolean;
intervalSeconds?: number;
@@ -24,6 +38,26 @@ export type ChannelConfig = {
url?: string;
audioFormat?: AudioFormat;
keepSourceVideo?: boolean;
+ // Keep-latest retention/persistence window. The newest N videos (by upload
+ // date) are protected from the Clean-audio sweep AND have their source video
+ // persisted to the saved-video store (see common/controller/keptVideos.ts and
+ // the per-download persistence rule in downloadOneManaged.ts). Semantics:
+ // undefined / 0 -> disabled
+ // > 0 -> keep newest N (clamped to KEEP_LATEST_MAX)
+ // A kept video later found deleted-from-source is pinned permanently via the
+ // do-not-clean marker so it survives even after it rolls out of the window.
+ keepLatest?: number;
+ // Audio-extraction strategy for transcribe-handling downloads (see
+ // ExtractionMode above). Omitted/unknown -> "ytdlp" (legacy behavior). The
+ // keep-latest persistence rule forces "app" for the videos it persists,
+ // regardless of this setting.
+ extractionMode?: ExtractionMode;
+ // Per-channel override for the saved-video store root (global default is
+ // paths.savedVideosDir / SAVED_VIDEOS_DIR). When set, this channel's persisted
+ // source videos live under <savedVideosDir>/<slug>/<videoId>/. Lets a single
+ // channel's large videos land on a different disk than the rest. Resolved by
+ // savedVideoDir() in common/lib/savedVideo.ts. Empty/whitespace = use global.
+ savedVideosDir?: string;
ytdlpExtraArgs?: string[];
subLangs?: string;
lastSyncedAt?: string;
@@ -61,6 +95,9 @@ export const CHANNEL_SLEEP_BETWEEN_DOWNLOADS_MAX_SECONDS = 600;
export const SYNC_INTERVAL_MIN_MINUTES = 1;
export const SYNC_INTERVAL_MAX_MINUTES = 44640;
+// Upper bound for a channel's keep-latest window. 0/undefined disables it.
+export const KEEP_LATEST_MAX = 100000;
+
export const AUDIO_CHECK_INTERVAL_DEFAULT_SECONDS = 60;
export const AUDIO_CHECK_INTERVAL_MIN_SECONDS = 10;
export const AUDIO_CHECK_INTERVAL_MAX_SECONDS = 600;
@@ -116,6 +153,21 @@ export function parseChannelConfig(raw: unknown): ChannelConfig | null {
config.keepSourceVideo = r.keepSourceVideo;
}
if (
+ typeof r.keepLatest === "number" &&
+ Number.isFinite(r.keepLatest) &&
+ r.keepLatest >= 0
+ ) {
+ // 0 is the "disabled" sentinel preserved as-is; positives clamp to the cap.
+ config.keepLatest =
+ r.keepLatest === 0 ? 0 : clampInt(r.keepLatest, 1, KEEP_LATEST_MAX);
+ }
+ if (r.extractionMode === "ytdlp" || r.extractionMode === "app") {
+ config.extractionMode = r.extractionMode;
+ }
+ if (typeof r.savedVideosDir === "string" && r.savedVideosDir.trim() !== "") {
+ config.savedVideosDir = r.savedVideosDir.trim();
+ }
+ if (
Array.isArray(r.ytdlpExtraArgs) &&
r.ytdlpExtraArgs.every((x) => typeof x === "string")
) {
diff --git a/common/lib/diskSpace.ts b/common/lib/diskSpace.ts
@@ -29,26 +29,33 @@ export type DiskSpaceStatus = {
ok: boolean;
};
-// Measure free space on the transcripts data directory (where all downloads
-// land) and compare it against the configured floor. When the gate is disabled
-// (minFreeDiskGB === 0) this always reports ok.
-export async function checkDiskSpace(
- paths: Paths,
+// Compare free space on the filesystem holding `dir` against the configured
+// floor. When the gate is disabled (minFreeDiskGB === 0) this always reports ok
+// and skips the statfs syscall entirely (it runs before every video in a batch
+// and on every active-jobs poll, and a disabled gate hides the indicator anyway).
+export async function checkDiskSpaceFor(
+ dir: string,
settings: SiteSettings,
): Promise<DiskSpaceStatus> {
const enabled = settings.minFreeDiskGB > 0;
const thresholdBytes = settings.minFreeDiskGB * BYTES_PER_GB;
- // Skip the statfs syscall entirely when the gate is off — this runs before
- // every video in a batch and on every active-jobs poll, and a disabled gate
- // hides the indicator anyway.
if (!enabled) {
- return { enabled, freeBytes: Number.POSITIVE_INFINITY, thresholdBytes, ok: true };
+ return {
+ enabled,
+ freeBytes: Number.POSITIVE_INFINITY,
+ thresholdBytes,
+ ok: true,
+ };
}
- const freeBytes = await getFreeBytes(paths.transcriptsDir);
- return {
- enabled,
- freeBytes,
- thresholdBytes,
- ok: freeBytes >= thresholdBytes,
- };
+ const freeBytes = await getFreeBytes(dir);
+ return { enabled, freeBytes, thresholdBytes, ok: freeBytes >= thresholdBytes };
+}
+
+// Measure free space on the transcripts data directory (where all downloads
+// land) and compare it against the configured floor.
+export async function checkDiskSpace(
+ paths: Paths,
+ settings: SiteSettings,
+): Promise<DiskSpaceStatus> {
+ return checkDiskSpaceFor(paths.transcriptsDir, settings);
}
diff --git a/common/lib/paths.ts b/common/lib/paths.ts
@@ -6,6 +6,14 @@ export type Paths = {
monorepoRoot: string;
transcriptsDir: string;
channelsDir: string;
+ // Root of the saved-video store: persisted source-video containers (the
+ // keep-latest persistence rule) are MOVED out of the per-video data dir into
+ // savedVideosDir/<slug>/<videoId>/, leaving only a small saved-video.json
+ // pointer in the data dir. Defaults under transcriptsDir but is overridable
+ // via SAVED_VIDEOS_DIR so the (large) source videos can live on a separate
+ // disk. A channel may further override the root via ChannelConfig.savedVideosDir.
+ // See common/lib/savedVideo.ts.
+ savedVideosDir: string;
// Per-site config lives under sitesDir/<siteId>/site.json (+ chart-templates.json).
// See common/lib/site.ts. A "site" is a selection + presentation layer over the
// single global channel pool; channel downloads are never duplicated per site.
@@ -67,6 +75,10 @@ export type Paths = {
whisperBin: string;
whisperModel: string;
ffmpegBin: string;
+ // rsync binary used to mirror the saved-video store to a backup destination
+ // (Phase 4 of the video-persistence feature). See
+ // common/controller/backupSavedVideos.ts.
+ rsyncBin: string;
// parakeet (overlapping-segment stitching) app. parakeetBin is the standalone
// wrapper script invoked as the app binary; parakeetCliBin is the underlying
// parakeet-cli it drives; parakeetModel is the default .gguf model.
@@ -98,6 +110,8 @@ export function getPaths(): Paths {
monorepoRoot,
transcriptsDir,
channelsDir: path.join(transcriptsDir, "channels"),
+ savedVideosDir:
+ process.env.SAVED_VIDEOS_DIR ?? path.join(transcriptsDir, "saved-videos"),
sitesDir,
homepageDir,
homepageConfigFile: path.join(homepageDir, "homepage.json"),
@@ -138,6 +152,7 @@ export function getPaths(): Paths {
"ggml-base.en.bin",
),
ffmpegBin: process.env.FFMPEG_BIN ?? "ffmpeg",
+ rsyncBin: process.env.RSYNC_BIN ?? "rsync",
parakeetBin:
process.env.PARAKEET_STITCH_BIN ??
path.join(monorepoRoot, "scripts", "parakeet-stitch.mjs"),
diff --git a/common/lib/savedVideo-server.ts b/common/lib/savedVideo-server.ts
@@ -0,0 +1,154 @@
+import path from "node:path";
+import {
+ copyFile,
+ mkdir,
+ readFile,
+ rename,
+ rm,
+ stat,
+ writeFile,
+} from "node:fs/promises";
+import {
+ SAVED_VIDEO_POINTER_FILENAME,
+ parseSavedVideoPointer,
+ savedVideoPath,
+ type SavedVideoKeepReason,
+ type SavedVideoPointer,
+} from "./savedVideo";
+
+// Filesystem side of the saved-video store. See savedVideo.ts for the layout.
+
+export function savedVideoPointerPath(videoDir: string): string {
+ return path.join(videoDir, SAVED_VIDEO_POINTER_FILENAME);
+}
+
+export async function loadSavedVideo(
+ videoDir: string,
+): Promise<SavedVideoPointer | null> {
+ try {
+ const raw = await readFile(savedVideoPointerPath(videoDir), "utf8");
+ return parseSavedVideoPointer(JSON.parse(raw));
+ } catch {
+ return null;
+ }
+}
+
+export async function isSavedVideo(videoDir: string): Promise<boolean> {
+ return (await loadSavedVideo(videoDir)) !== null;
+}
+
+async function writePointer(
+ videoDir: string,
+ pointer: SavedVideoPointer,
+): Promise<void> {
+ const file = savedVideoPointerPath(videoDir);
+ const tmp = `${file}.tmp-${process.pid}`;
+ await writeFile(tmp, JSON.stringify(pointer, null, 2) + "\n");
+ await rename(tmp, file);
+}
+
+// Move a file, crossing device boundaries safely. A plain rename() works within
+// one filesystem; EXDEV (the store is on a different disk) falls back to a
+// copy-to-temp + atomic rename + unlink so a crash mid-copy never leaves a
+// partial file under the final name.
+async function moveFileCrossDevice(src: string, dest: string): Promise<void> {
+ await mkdir(path.dirname(dest), { recursive: true });
+ try {
+ await rename(src, dest);
+ return;
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code !== "EXDEV") throw err;
+ }
+ const tmp = `${dest}.tmp-${process.pid}`;
+ await copyFile(src, tmp);
+ await rename(tmp, dest);
+ await rm(src, { force: true });
+}
+
+// Resolve a usable absolute path to a video's persisted source container, or
+// null when there's no pointer or the stored file has gone missing.
+export async function resolveSavedVideo(
+ videoDir: string,
+): Promise<string | null> {
+ const pointer = await loadSavedVideo(videoDir);
+ if (!pointer) return null;
+ const file = savedVideoPath(pointer);
+ try {
+ await stat(file);
+ return file;
+ } catch {
+ return null;
+ }
+}
+
+// Move a downloaded source container OUT of the main data dir into the saved
+// store and write a pointer sidecar back into the data dir. Returns the pointer.
+// Overwrites any existing stored container for this video (the redownload case).
+export async function persistSourceVideo(opts: {
+ videoDir: string;
+ sourceFilename: string;
+ storeDir: string;
+ keepReason?: SavedVideoKeepReason;
+}): Promise<SavedVideoPointer> {
+ const src = path.join(opts.videoDir, opts.sourceFilename);
+ const dest = path.join(opts.storeDir, opts.sourceFilename);
+ const st = await stat(src);
+ await moveFileCrossDevice(src, dest);
+ const pointer: SavedVideoPointer = {
+ storedAt: new Date().toISOString(),
+ dir: opts.storeDir,
+ file: opts.sourceFilename,
+ bytes: st.size,
+ ...(opts.keepReason ? { keepReason: opts.keepReason } : {}),
+ };
+ await writePointer(opts.videoDir, pointer);
+ return pointer;
+}
+
+// Record a content hash on a video's pointer (populated by the backup step so
+// the store + UI can show a verified checksum). Best-effort: a missing pointer
+// is a no-op.
+export async function updateSavedVideoChecksum(
+ videoDir: string,
+ sha256: string,
+): Promise<void> {
+ const pointer = await loadSavedVideo(videoDir);
+ if (!pointer) return;
+ await writePointer(videoDir, { ...pointer, sha256 });
+}
+
+// Best-effort removal of a now-empty store dir (and its empty <slug> parent).
+async function pruneEmptyStoreDirs(storeDir: string): Promise<void> {
+ await rm(storeDir, { recursive: false, force: true }).catch(() => {});
+ await rm(path.dirname(storeDir), { recursive: false, force: true }).catch(
+ () => {},
+ );
+}
+
+// Reverse persistSourceVideo: move the stored container back into the data dir
+// and remove the pointer. Returns false when there was no pointer to reverse.
+export async function unpersistSavedVideo(videoDir: string): Promise<boolean> {
+ const pointer = await loadSavedVideo(videoDir);
+ if (!pointer) return false;
+ const src = savedVideoPath(pointer);
+ const dest = path.join(videoDir, pointer.file);
+ try {
+ await moveFileCrossDevice(src, dest);
+ } catch {
+ // The stored container is already gone; just drop the dangling pointer.
+ }
+ await rm(savedVideoPointerPath(videoDir), { force: true });
+ await pruneEmptyStoreDirs(pointer.dir);
+ return true;
+}
+
+// Permanently delete a video's persisted container + pointer (retention prune of
+// a video that rolled out of the keep-latest window). Returns bytes freed.
+export async function dropSavedVideo(videoDir: string): Promise<number> {
+ const pointer = await loadSavedVideo(videoDir);
+ if (!pointer) return 0;
+ await rm(savedVideoPath(pointer), { force: true });
+ await rm(savedVideoPointerPath(videoDir), { force: true });
+ await pruneEmptyStoreDirs(pointer.dir);
+ return pointer.bytes;
+}
diff --git a/common/lib/savedVideo.test.ts b/common/lib/savedVideo.test.ts
@@ -0,0 +1,155 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, stat, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "./paths";
+import {
+ parseSavedVideoPointer,
+ savedVideoDir,
+ savedVideoRoot,
+} from "./savedVideo";
+import {
+ dropSavedVideo,
+ loadSavedVideo,
+ persistSourceVideo,
+ resolveSavedVideo,
+ unpersistSavedVideo,
+} from "./savedVideo-server";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test lib/savedVideo.test.ts
+
+function fakePaths(savedVideosDir: string): Paths {
+ return { savedVideosDir } as Paths;
+}
+
+async function withTmp(fn: (root: string) => Promise<void>): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-saved-"));
+ try {
+ await fn(dir);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+test("savedVideoRoot prefers the per-channel override, else the global", () => {
+ const paths = fakePaths("/global/store");
+ assert.equal(savedVideoRoot(paths, undefined), "/global/store");
+ assert.equal(savedVideoRoot(paths, { savedVideosDir: " " }), "/global/store");
+ assert.equal(
+ savedVideoRoot(paths, { savedVideosDir: "/chan/store" }),
+ "/chan/store",
+ );
+});
+
+test("savedVideoDir composes <root>/<slug>/<videoId>", () => {
+ const paths = fakePaths("/global/store");
+ assert.equal(
+ savedVideoDir(paths, undefined, "chan", "vid1"),
+ path.join("/global/store", "chan", "vid1"),
+ );
+});
+
+test("parseSavedVideoPointer rejects malformed and reads keepReason", () => {
+ assert.equal(parseSavedVideoPointer(null), null);
+ assert.equal(parseSavedVideoPointer({ dir: "/a" }), null); // no file/bytes
+ const ok = parseSavedVideoPointer({
+ storedAt: "t",
+ dir: "/a",
+ file: "source-media.mp4",
+ bytes: 10,
+ keepReason: "keep-latest",
+ sha256: "abc",
+ });
+ assert.deepEqual(ok, {
+ storedAt: "t",
+ dir: "/a",
+ file: "source-media.mp4",
+ bytes: 10,
+ keepReason: "keep-latest",
+ sha256: "abc",
+ });
+ // An unknown keepReason is dropped rather than carried through.
+ const noReason = parseSavedVideoPointer({
+ dir: "/a",
+ file: "f.mp4",
+ bytes: 1,
+ keepReason: "bogus",
+ });
+ assert.equal(noReason?.keepReason, undefined);
+});
+
+test("persist moves the container into the store and writes a pointer", async () => {
+ await withTmp(async (root) => {
+ const videoDir = path.join(root, "data", "vid1");
+ const storeDir = path.join(root, "store", "chan", "vid1");
+ await mkdir(videoDir, { recursive: true });
+ await writeFile(path.join(videoDir, "source-media.mp4"), "video-bytes");
+
+ const pointer = await persistSourceVideo({
+ videoDir,
+ sourceFilename: "source-media.mp4",
+ storeDir,
+ keepReason: "keep-latest",
+ });
+
+ assert.equal(pointer.file, "source-media.mp4");
+ assert.equal(pointer.dir, storeDir);
+ assert.equal(pointer.keepReason, "keep-latest");
+ assert.equal(pointer.bytes, "video-bytes".length);
+ // Container moved OUT of the data dir, into the store.
+ await assert.rejects(stat(path.join(videoDir, "source-media.mp4")));
+ await stat(path.join(storeDir, "source-media.mp4"));
+ // Pointer readable + resolvable.
+ const loaded = await loadSavedVideo(videoDir);
+ assert.equal(loaded?.keepReason, "keep-latest");
+ assert.equal(
+ await resolveSavedVideo(videoDir),
+ path.join(storeDir, "source-media.mp4"),
+ );
+ });
+});
+
+test("unpersist returns the container to the data dir and drops the pointer", async () => {
+ await withTmp(async (root) => {
+ const videoDir = path.join(root, "data", "vid1");
+ const storeDir = path.join(root, "store", "chan", "vid1");
+ await mkdir(videoDir, { recursive: true });
+ await writeFile(path.join(videoDir, "source-media.webm"), "abc");
+ await persistSourceVideo({
+ videoDir,
+ sourceFilename: "source-media.webm",
+ storeDir,
+ keepReason: "override",
+ });
+
+ assert.equal(await unpersistSavedVideo(videoDir), true);
+ await stat(path.join(videoDir, "source-media.webm"));
+ assert.equal(await loadSavedVideo(videoDir), null);
+ assert.equal(await resolveSavedVideo(videoDir), null);
+ // Reversing again is a no-op.
+ assert.equal(await unpersistSavedVideo(videoDir), false);
+ });
+});
+
+test("drop deletes the stored container and reports bytes freed", async () => {
+ await withTmp(async (root) => {
+ const videoDir = path.join(root, "data", "vid1");
+ const storeDir = path.join(root, "store", "chan", "vid1");
+ await mkdir(videoDir, { recursive: true });
+ await writeFile(path.join(videoDir, "source-media.mkv"), "0123456789");
+ await persistSourceVideo({
+ videoDir,
+ sourceFilename: "source-media.mkv",
+ storeDir,
+ keepReason: "keep-latest",
+ });
+
+ const freed = await dropSavedVideo(videoDir);
+ assert.equal(freed, 10);
+ assert.equal(await loadSavedVideo(videoDir), null);
+ await assert.rejects(stat(path.join(storeDir, "source-media.mkv")));
+ assert.equal(await dropSavedVideo(videoDir), 0); // nothing left
+ });
+});
diff --git a/common/lib/savedVideo.ts b/common/lib/savedVideo.ts
@@ -0,0 +1,92 @@
+import path from "node:path";
+import type { Paths } from "./paths";
+import type { ChannelConfig } from "./channelConfig";
+
+// The saved-video store (Phase 3 of the video-persistence feature).
+//
+// When the per-download persistence rule decides to keep a source video, the
+// downloaded container (data/<id>/source-media.<ext> from Phase 2) is MOVED out
+// of the per-video data dir into a separate store and a small pointer sidecar is
+// left behind in the data dir. This keeps the main data volume holding only
+// audio + transcripts while the (large) source videos can live on another disk.
+//
+// Layout:
+// <root>/<slug>/<videoId>/source-media.<ext> (the moved container)
+// channels/<slug>/data/<videoId>/saved-video.json (the pointer back to it)
+//
+// <root> = ChannelConfig.savedVideosDir (per-channel override) || Paths.savedVideosDir.
+//
+// This module is pure (path math + pointer parse) so it's safe to import from
+// both client and server; the filesystem operations live in savedVideo-server.ts.
+
+export const SAVED_VIDEO_POINTER_FILENAME = "saved-video.json";
+
+// Why this source was persisted (mirrors persistencePlan's PersistCategory, minus
+// "none"). The retention prune only evicts "keep-latest" containers once they
+// roll out of the window; "pin" (irreplaceable) and "override" (manually
+// archived) containers are kept until explicitly unpersisted.
+export type SavedVideoKeepReason = "keep-latest" | "pin" | "override";
+
+export type SavedVideoPointer = {
+ // ISO timestamp the container was persisted into the store.
+ storedAt: string;
+ // Absolute store dir that holds the container (savedVideoDir() result).
+ dir: string;
+ // Container basename within `dir` (e.g. "source-media.mp4").
+ file: string;
+ // Size of the stored container in bytes (for backup manifests + disk reports).
+ bytes: number;
+ // Why it was persisted (governs retention pruning). Absent on legacy pointers,
+ // which the prune treats conservatively as non-evictable.
+ keepReason?: SavedVideoKeepReason;
+ // Optional content hash, populated by the backup/verify step (Phase 4).
+ sha256?: string;
+};
+
+function isKeepReason(v: unknown): v is SavedVideoKeepReason {
+ return v === "keep-latest" || v === "pin" || v === "override";
+}
+
+// The store root for a channel: the per-channel override when set, else the
+// global default. Whitespace-only overrides fall back to the global default.
+export function savedVideoRoot(
+ paths: Paths,
+ config: Pick<ChannelConfig, "savedVideosDir"> | null | undefined,
+): string {
+ const override = config?.savedVideosDir?.trim();
+ return override ? override : paths.savedVideosDir;
+}
+
+// The store dir for a single video: <root>/<slug>/<videoId>/.
+export function savedVideoDir(
+ paths: Paths,
+ config: Pick<ChannelConfig, "savedVideosDir"> | null | undefined,
+ channelSlug: string,
+ videoId: string,
+): string {
+ return path.join(savedVideoRoot(paths, config), channelSlug, videoId);
+}
+
+// Absolute path to the stored container a pointer references.
+export function savedVideoPath(pointer: SavedVideoPointer): string {
+ return path.join(pointer.dir, pointer.file);
+}
+
+export function parseSavedVideoPointer(raw: unknown): SavedVideoPointer | null {
+ if (!raw || typeof raw !== "object") return null;
+ const r = raw as Record<string, unknown>;
+ if (typeof r.dir !== "string" || r.dir === "") return null;
+ if (typeof r.file !== "string" || r.file === "") return null;
+ if (typeof r.bytes !== "number" || !Number.isFinite(r.bytes)) return null;
+ const pointer: SavedVideoPointer = {
+ storedAt: typeof r.storedAt === "string" ? r.storedAt : "",
+ dir: r.dir,
+ file: r.file,
+ bytes: r.bytes,
+ };
+ if (isKeepReason(r.keepReason)) pointer.keepReason = r.keepReason;
+ if (typeof r.sha256 === "string" && r.sha256 !== "") {
+ pointer.sha256 = r.sha256;
+ }
+ return pointer;
+}
diff --git a/common/lib/savedVideoBackup.ts b/common/lib/savedVideoBackup.ts
@@ -0,0 +1,80 @@
+import path from "node:path";
+
+// Backup manifest for the saved-video store (Phase 4 of the video-persistence
+// feature). When the store is mirrored to a backup destination, a manifest is
+// written at the destination root recording, for each saved container, its
+// canonical <slug>/<videoId>/<file> location, its size, and a streamed sha256.
+// The verify step compares this manifest against what's actually on the backup
+// destination (and the live store) to report drift.
+//
+// This module is pure (types + parse + path math) so it's importable from both
+// client and server; the filesystem/rsync work lives in
+// common/controller/backupSavedVideos.ts.
+
+export const BACKUP_MANIFEST_FILENAME = "backup-manifest.json";
+
+export type BackupEntry = {
+ // The channel slug the container belongs to.
+ slug: string;
+ // The video id within the channel.
+ videoId: string;
+ // Container basename (e.g. "source-media.mp4").
+ file: string;
+ // Size in bytes at backup time.
+ bytes: number;
+ // sha256 of the container contents, streamed at backup time.
+ sha256: string;
+};
+
+export type BackupManifest = {
+ // ISO timestamp the backup that produced this manifest completed.
+ backedUpAt: string;
+ // One entry per saved container mirrored to the destination.
+ entries: BackupEntry[];
+};
+
+// Canonical relative location of a backed-up container under the destination
+// root: <slug>/<videoId>/<file>. The store mirrors this layout regardless of
+// which physical root (global or per-channel override) the live copy lives on,
+// so every saved video lands in one predictable place in the backup.
+export function backupEntryRelDir(entry: Pick<BackupEntry, "slug" | "videoId">): string {
+ return path.join(entry.slug, entry.videoId);
+}
+
+export function backupEntryRelPath(
+ entry: Pick<BackupEntry, "slug" | "videoId" | "file">,
+): string {
+ return path.join(entry.slug, entry.videoId, entry.file);
+}
+
+function parseEntry(raw: unknown): BackupEntry | null {
+ if (!raw || typeof raw !== "object") return null;
+ const r = raw as Record<string, unknown>;
+ if (typeof r.slug !== "string" || r.slug === "") return null;
+ if (typeof r.videoId !== "string" || r.videoId === "") return null;
+ if (typeof r.file !== "string" || r.file === "") return null;
+ if (typeof r.bytes !== "number" || !Number.isFinite(r.bytes)) return null;
+ if (typeof r.sha256 !== "string" || r.sha256 === "") return null;
+ return {
+ slug: r.slug,
+ videoId: r.videoId,
+ file: r.file,
+ bytes: r.bytes,
+ sha256: r.sha256,
+ };
+}
+
+export function parseBackupManifest(raw: unknown): BackupManifest | null {
+ if (!raw || typeof raw !== "object") return null;
+ const r = raw as Record<string, unknown>;
+ if (!Array.isArray(r.entries)) return null;
+ const entries: BackupEntry[] = [];
+ for (const e of r.entries) {
+ const parsed = parseEntry(e);
+ if (parsed) entries.push(parsed);
+ }
+ return {
+ backedUpAt: typeof r.backedUpAt === "string" ? r.backedUpAt : "",
+ entries,
+ };
+}
diff --git a/common/lib/settings.ts b/common/lib/settings.ts
@@ -114,6 +114,24 @@ export type SiteSettings = {
// common/lib/site.ts. The one presentation field that lives globally so a
// shared footer doesn't have to be repeated per site.
socialLinks: SocialLink[];
+ // Backup configuration for the saved-video store (Phase 4 of the
+ // video-persistence feature). When enabled with a destination, the store is
+ // mirrored there (additively, no deletes) with a per-backup manifest, and the
+ // sync scheduler runs the backup on the configured cadence. See
+ // common/controller/backupSavedVideos.ts.
+ savedVideoBackup: SavedVideoBackupSettings;
+};
+
+export type SavedVideoBackupSettings = {
+ // Master switch for the scheduled backup. A backup can still be run manually
+ // when this is false, as long as a destination is set.
+ enabled: boolean;
+ // Destination root the store is mirrored into (a local path or any rsync
+ // target). Empty disables both scheduled and manual backups.
+ dest: string;
+ // Cadence (minutes) for the scheduled backup when enabled. Clamped into the
+ // sync-interval window; default daily.
+ intervalMinutes: number;
};
export type SyncSchedulerSettings = {
@@ -142,6 +160,13 @@ export type SyncSchedulerSettings = {
// [SYNC_HEARTBEAT_MIN_SECONDS, SYNC_HEARTBEAT_MAX_SECONDS]. The env var
// SYNC_HEARTBEAT_SECONDS overrides this at runtime. See SCHEDULED_SYNC.md.
heartbeatSeconds: number;
+ // Cadence (minutes) for the scheduled keep-latest deletion check. For each
+ // channel with ChannelConfig.keepLatest > 0, the tick re-probes the kept
+ // window for source deletion (checkKeptDeletedAction) at most this often and
+ // pins any gone videos as do-not-clean. Clamped into the sync-interval window;
+ // default daily. The check shares the same concurrency cap and quiet-hours
+ // window as scheduled syncs. See editor/app/scheduler/runTick.ts.
+ keepLatestCheckIntervalMinutes: number;
};
export type SocialLink = {
@@ -200,6 +225,8 @@ export const SYNC_SCHEDULER_MAX_CONCURRENT_DEFAULT = 2;
export const SYNC_SCHEDULER_MAX_CONCURRENT_MAX = 16;
export const SYNC_SCHEDULER_BACKOFF_BASE_DEFAULT_MINUTES = 30;
export const SYNC_SCHEDULER_BACKOFF_MAX_DEFAULT_MINUTES = 1440;
+export const KEEP_LATEST_CHECK_DEFAULT_INTERVAL_MINUTES = 1440;
+export const SAVED_VIDEO_BACKUP_DEFAULT_INTERVAL_MINUTES = 1440;
// Internal-heartbeat cadence bounds. 0 means "off" (use an external cron
// heartbeat); any other value is clamped into [MIN, MAX] seconds. The floor
@@ -218,6 +245,7 @@ export function defaultSyncScheduler(): SyncSchedulerSettings {
backoffBaseMinutes: SYNC_SCHEDULER_BACKOFF_BASE_DEFAULT_MINUTES,
backoffMaxMinutes: SYNC_SCHEDULER_BACKOFF_MAX_DEFAULT_MINUTES,
heartbeatSeconds: SYNC_HEARTBEAT_DEFAULT_SECONDS,
+ keepLatestCheckIntervalMinutes: KEEP_LATEST_CHECK_DEFAULT_INTERVAL_MINUTES,
};
}
@@ -290,6 +318,40 @@ export function sanitizeSyncScheduler(value: unknown): SyncSchedulerSettings {
),
),
heartbeatSeconds: clampHeartbeatSeconds(r.heartbeatSeconds),
+ keepLatestCheckIntervalMinutes: clampPositiveInt(
+ r.keepLatestCheckIntervalMinutes,
+ d.keepLatestCheckIntervalMinutes,
+ SYNC_INTERVAL_MAX_MINUTES,
+ ),
+ };
+}
+
+export function defaultSavedVideoBackup(): SavedVideoBackupSettings {
+ return {
+ enabled: false,
+ dest: "",
+ intervalMinutes: SAVED_VIDEO_BACKUP_DEFAULT_INTERVAL_MINUTES,
+ };
+}
+
+// Coerce a raw settings.savedVideoBackup value into a clean
+// SavedVideoBackupSettings. A missing destination forces enabled off, since a
+// backup with nowhere to go is meaningless.
+export function sanitizeSavedVideoBackup(
+ value: unknown,
+): SavedVideoBackupSettings {
+ const d = defaultSavedVideoBackup();
+ if (!value || typeof value !== "object") return d;
+ const r = value as Record<string, unknown>;
+ const dest = typeof r.dest === "string" ? r.dest.trim() : "";
+ return {
+ enabled: dest !== "" && r.enabled === true,
+ dest,
+ intervalMinutes: clampPositiveInt(
+ r.intervalMinutes,
+ d.intervalMinutes,
+ SYNC_INTERVAL_MAX_MINUTES,
+ ),
};
}
@@ -311,6 +373,7 @@ function defaults(): SiteSettings {
syncScheduler: defaultSyncScheduler(),
autoQueue: defaultAutoQueue(),
socialLinks: [],
+ savedVideoBackup: defaultSavedVideoBackup(),
};
}
@@ -473,6 +536,7 @@ export function getSettings(): SiteSettings {
merged.syncScheduler = sanitizeSyncScheduler(merged.syncScheduler);
merged.autoQueue = sanitizeAutoQueue(merged.autoQueue);
merged.socialLinks = parseSocialLinks(merged.socialLinks);
+ merged.savedVideoBackup = sanitizeSavedVideoBackup(merged.savedVideoBackup);
// Workers. When the file predates the worker model (no `workers` key),
// synthesize a default list from the (now-settled) active app + per-app
// configs so existing installs behave identically. Otherwise sanitize the
@@ -643,6 +707,7 @@ export async function writeSettings(next: SiteSettings): Promise<void> {
syncScheduler: sanitizeSyncScheduler(next.syncScheduler),
autoQueue: sanitizeAutoQueue(next.autoQueue),
socialLinks,
+ savedVideoBackup: sanitizeSavedVideoBackup(next.savedVideoBackup),
};
const tmp = `${file}.tmp-${process.pid}`;
await fs.promises.writeFile(tmp, JSON.stringify(merged, null, 2) + "\n");
diff --git a/common/lib/videoStatus.ts b/common/lib/videoStatus.ts
@@ -69,6 +69,48 @@ export function audioFilesToRemove(
return files;
}
+// Video container extensions the app may keep as a persisted source video and
+// transcribe directly from (parakeet's stitcher ffmpeg-slices any container).
+// Consolidated here so VideoPanel and the transcribe fallback share one list.
+export const VIDEO_CONTAINER_EXTS: ReadonlyArray<string> = [
+ "mp4",
+ "webm",
+ "mkv",
+ "mov",
+ "m4v",
+ "ogv",
+ "avi",
+];
+
+export function isVideoContainer(name: string): boolean {
+ const dot = name.lastIndexOf(".");
+ if (dot < 0) return false;
+ return VIDEO_CONTAINER_EXTS.includes(name.slice(dot + 1).toLowerCase());
+}
+
+// A persisted source video container is written under data/<id>/source-media.<ext>
+// (a deliberately distinct base name from audio.<ext> so it's never mistaken for
+// an extractable/cleanable audio file by isRealAudioFile). The keep-latest
+// persistence rule downloads the full video here and the app extracts audio from
+// it; the container is then kept (Phase 3 moves it to the saved-video store).
+export const SOURCE_MEDIA_BASENAME = "source-media";
+
+export function isSourceMediaFile(name: string): boolean {
+ if (!name.startsWith(`${SOURCE_MEDIA_BASENAME}.`)) return false;
+ if (name.includes(".tmp-")) return false;
+ if (name.endsWith(".info.json")) return false;
+ if (name.endsWith(".part")) return false;
+ if (name.endsWith(".part.good")) return false;
+ if (name.endsWith(".part.testing")) return false;
+ return true;
+}
+
+// The persisted source container in a video dir, if any (the keep-latest source
+// media). Null when no finalized source-media.<ext> exists.
+export function findSourceMedia(entries: string[]): string | null {
+ return entries.find(isSourceMediaFile) ?? null;
+}
+
// A genuine resumable partial: an interrupted audio download, NOT a live-chat
// sidecar that merely ends in .part.
export function isPartAudioFile(name: string): boolean {
diff --git a/common/ytdlp/downloadOneManaged.ts b/common/ytdlp/downloadOneManaged.ts
@@ -1,5 +1,5 @@
import path from "node:path";
-import { appendFile, mkdir, readdir, readFile } from "node:fs/promises";
+import { appendFile, mkdir, readdir, readFile, rm } from "node:fs/promises";
import { createWriteStream, type WriteStream } from "node:fs";
import { execa } from "execa";
import {
@@ -12,6 +12,20 @@ import {
type ChannelConfig,
} from "../lib/channelConfig";
import {
+ resolvePersistenceDecision,
+ type PersistenceDecision,
+} from "./persistencePlan";
+import {
+ isInKeepWindow,
+ uploadKeyFor,
+ type KeepWindow,
+} from "../controller/keptVideos";
+import { isDoNotClean } from "../lib/doNotClean-server";
+import { transcodeAudio } from "../controller/transcode";
+import { findSourceMedia } from "../lib/videoStatus";
+import { savedVideoDir } from "../lib/savedVideo";
+import { persistSourceVideo } from "../lib/savedVideo-server";
+import {
type AudioCheckAttemptStats,
type DownloadAttempt,
type DownloadOutcomeRecord,
@@ -69,6 +83,18 @@ export type ManagedDownloadOpts = {
// download filters. Plumbed from the playlist-level caller alongside the
// other resolved settings.
globalSkipLiveDownloads?: boolean;
+ // ---- Per-download persistence rule (Phase 2) ----
+ // The channel's keep-latest window as a cutoff, computed once per run by the
+ // caller (computeKeepWindow). This video's membership is decided against its
+ // own upload date — newer-than-the-cutoff videos persist their source.
+ keepWindow?: KeepWindow;
+ // Per-run override: force-keep (true) / force-discard (false) the source video.
+ keepSourceVideoOverride?: boolean;
+ // Per-run override: extract audio now and discard the container even for a
+ // video the keep-latest rule would otherwise persist (the save-disk backfill).
+ extractImmediately?: boolean;
+ // Per-run override of channelConfig.audioFormat for the extracted audio.
+ audioFormatOverride?: AudioFormat;
};
// When `reuseInfoJson` is true, the real download reuses the metadata the
@@ -90,22 +116,119 @@ function youtubeHandlingArgs(
return args;
}
-function transcribeHandlingArgs(
+// Format/extract args for a transcribe-handling download under a resolved
+// persistence plan. "ytdlp" mode is the legacy path: yt-dlp extracts the audio
+// itself (-x) and optionally keeps its bestaudio source via -k. "app" mode omits
+// -x so yt-dlp leaves the source container for the app to extract from
+// (finalizeAppExtraction) — pulling a full video when persisting, bestaudio
+// otherwise.
+function audioFormatSelectionArgs(
+ plan: PersistenceDecision,
+ fmt: AudioFormat,
config: ChannelConfig,
- reuseInfoJson = false,
): string[] {
- const fmt: AudioFormat = config.audioFormat ?? "mp3";
- const args: string[] = [];
- if (!reuseInfoJson) args.push("--write-info-json");
- args.push("-f", "bestaudio/worst", "-x", "--audio-format", fmt);
- if (config.keepSourceVideo) args.push("-k");
- return args;
+ if (plan.extractionMode === "ytdlp") {
+ const args = ["-f", "bestaudio/worst", "-x", "--audio-format", fmt];
+ if (config.keepSourceVideo) args.push("-k");
+ return args;
+ }
+ return plan.persist
+ ? ["-f", "bestvideo*+bestaudio/best"]
+ : ["-f", "bestaudio/worst"];
}
-function handlingArgs(config: ChannelConfig, reuseInfoJson: boolean): string[] {
- return config.handling === "youtube"
- ? youtubeHandlingArgs(config, reuseInfoJson)
- : transcribeHandlingArgs(config, reuseInfoJson);
+// The media output + info-json + format args for a transcribe-handling download.
+// In app mode the main output is source-media.<ext> (distinct from audio.<ext>
+// so it's never treated as cleanable audio); otherwise the historical audio.<ext>.
+// reuseInfoJson drops --write-info-json (the prefetch already wrote it).
+function transcribeMediaArgs(
+ url: string,
+ config: ChannelConfig,
+ plan: PersistenceDecision,
+ fmt: AudioFormat,
+ reuseInfoJson: boolean,
+): string[] {
+ const mediaName = plan.extractionMode === "app" ? "source-media" : "audio";
+ return [
+ ...outputArgsForUrl(url, { mediaName }),
+ ...(reuseInfoJson ? [] : ["--write-info-json"]),
+ ...audioFormatSelectionArgs(plan, fmt, config),
+ ];
+}
+
+// After an app-mode transcribe download, produce audio.<fmt> from the downloaded
+// source-media.<ext> container via ffmpeg. When the video is persisted the
+// container is MOVED into the saved-video store (Phase 3) and a pointer is left
+// in the data dir; otherwise the container is removed (the extract-now /
+// save-disk path). Best-effort throughout: a failed extraction leaves the
+// container in place (so a later pass or the transcribe-from-container fallback
+// can still recover), and a failed store-move leaves the container in the data
+// dir as source-media.<ext> (still persisted, just not relocated). Neither fails
+// the whole download.
+async function finalizeAppExtraction(opts: {
+ paths: Paths;
+ channelSlug: string;
+ channelConfig: ChannelConfig;
+ videoDir: string;
+ videoId: string;
+ fmt: AudioFormat;
+ persist: boolean;
+ // The persistence cause, recorded on the saved-video pointer (governs the
+ // retention prune). Only meaningful when persist is true.
+ category: PersistenceDecision["category"];
+ onLog: (s: string) => void;
+ signal: AbortSignal;
+}): Promise<void> {
+ const entries = await readdir(opts.videoDir).catch(() => [] as string[]);
+ const source = findSourceMedia(entries);
+ if (!source) {
+ opts.onLog(
+ `App extraction: no source-media container in ${opts.videoDir}; skipping.\n`,
+ );
+ return;
+ }
+ try {
+ await transcodeAudio({
+ paths: opts.paths,
+ videoDir: opts.videoDir,
+ sourceFilename: source,
+ targetFormat: opts.fmt,
+ onLog: opts.onLog,
+ signal: opts.signal,
+ });
+ } catch (err) {
+ opts.onLog(
+ `App extraction failed (${source} -> audio.${opts.fmt}): ${(err as Error).message}. Keeping the source container.\n`,
+ );
+ return;
+ }
+ if (!opts.persist) {
+ await rm(path.join(opts.videoDir, source), { force: true });
+ opts.onLog(`Discarded source container ${source} (audio-only).\n`);
+ return;
+ }
+ // Persist: move the container into the saved-video store + write a pointer.
+ const storeDir = savedVideoDir(
+ opts.paths,
+ opts.channelConfig,
+ opts.channelSlug,
+ opts.videoId,
+ );
+ try {
+ const pointer = await persistSourceVideo({
+ videoDir: opts.videoDir,
+ sourceFilename: source,
+ storeDir,
+ keepReason: opts.category === "none" ? undefined : opts.category,
+ });
+ opts.onLog(
+ `Persisted source video to ${path.join(pointer.dir, pointer.file)} (${pointer.bytes} bytes).\n`,
+ );
+ } catch (err) {
+ opts.onLog(
+ `Failed to move source container into the saved-video store (${(err as Error).message}); leaving ${source} in the data dir.\n`,
+ );
+ }
}
// The source a download attempt reads from: either the prefetched info json
@@ -335,6 +458,9 @@ async function runManagedDownload(
// yt-dlp's pinned (expirable) format URLs — so it re-extracts even though the
// prefetch ran.
let infoJsonPath: string | null = null;
+ // This video's recency key (upload_date), captured from the prefetched
+ // metadata so the keep-latest cutoff can classify it even before it's on disk.
+ let videoUploadKey = canonicalId ? uploadKeyFor(undefined, canonicalId) : "";
if (canonicalId && !opts.signal.aborted) {
const videoDir = path.join(channelDir, "data", canonicalId);
const prefetchArgs = [
@@ -372,6 +498,7 @@ async function runManagedDownload(
// the metadata file; a failed prefetch falls through to the legacy path so
// the existing auth-retry logic still gets a chance.
if (metadata) infoJsonPath = metaPath;
+ videoUploadKey = uploadKeyFor(metadata?.upload_date, canonicalId);
const decision = evaluateDownloadFilters({
metadata,
@@ -406,11 +533,59 @@ async function runManagedDownload(
// Audio-check re-extracts fresh format URLs, so it never reuses the prefetch.
const reuseInfoJson = infoJsonPath !== null;
+ // ---------- Per-download persistence decision ----------
+ // Resolve the channel keep-latest rule (plus per-run overrides) into a concrete
+ // plan: whether to keep the source video and who extracts the audio. Only
+ // affects transcribe-handling downloads (and the youtube no-subs fallback,
+ // which switches to transcribe). Cheap: a set/cutoff compare + one stat.
+ const fmt: AudioFormat =
+ opts.audioFormatOverride ?? opts.channelConfig.audioFormat ?? "mp3";
+ const pinned = canonicalId
+ ? await isDoNotClean(path.join(channelDir, "data", canonicalId))
+ : false;
+ const plan = resolvePersistenceDecision({
+ inWindow: isInKeepWindow(videoUploadKey, opts.keepWindow),
+ pinned,
+ channelExtractionMode: opts.channelConfig.extractionMode,
+ overrides: {
+ keepSourceVideoOverride: opts.keepSourceVideoOverride,
+ extractImmediately: opts.extractImmediately,
+ },
+ });
+
// ---------- Attempt 1: primary ----------
const audioCheckEnabled =
opts.channelConfig.handling === "transcribe" &&
opts.channelConfig.audioCheck?.enabled === true;
+ // The shared media args for the primary + auth-retry attempts. Audio-check owns
+ // its own args; youtube handling downloads subtitles (the persistence plan only
+ // applies to the no-subs fallback below). For transcribe handling the plan
+ // chooses audio-only vs. keep-source-video and yt-dlp vs. app extraction.
+ const mediaArgs =
+ opts.channelConfig.handling === "youtube"
+ ? [
+ ...outputArgsForUrl(opts.videoUrl),
+ ...youtubeHandlingArgs(opts.channelConfig, reuseInfoJson),
+ ]
+ : transcribeMediaArgs(
+ opts.videoUrl,
+ opts.channelConfig,
+ plan,
+ fmt,
+ reuseInfoJson,
+ );
+ if (opts.channelConfig.handling === "transcribe") {
+ opts.onLog(
+ `Persistence: ${plan.persist ? "keep source video" : "audio-only"} via ${plan.extractionMode} extraction (${plan.reason}).\n`,
+ );
+ if (audioCheckEnabled && plan.persist) {
+ opts.onLog(
+ `Note: this video qualifies for source-video persistence, but the channel uses audio-check; persistence is skipped for audio-checked downloads.\n`,
+ );
+ }
+ }
+
let primaryRes: AttemptOutcome;
let audioCheckStats: AudioCheckAttemptStats | undefined;
let audioCheckCorruptSource = false;
@@ -468,8 +643,7 @@ async function runManagedDownload(
"--ignore-config",
"--restrict-filenames",
...FULL_LOG_PROGRESS_ARGS,
- ...outputArgsForUrl(opts.videoUrl),
- ...handlingArgs(opts.channelConfig, reuseInfoJson),
+ ...mediaArgs,
"--print",
`after_video:${ARCHIVE_MARKER} %(extractor)s %(id)s`,
...channelConfigArgs(opts.channelConfig),
@@ -519,8 +693,7 @@ async function runManagedDownload(
"--ignore-config",
"--restrict-filenames",
...FULL_LOG_PROGRESS_ARGS,
- ...outputArgsForUrl(opts.videoUrl),
- ...handlingArgs(opts.channelConfig, reuseInfoJson),
+ ...mediaArgs,
"--print",
`after_video:${ARCHIVE_MARKER} %(extractor)s %(id)s`,
...channelConfigArgs(opts.channelConfig, opts.globalCookiesFromBrowser),
@@ -549,12 +722,38 @@ async function runManagedDownload(
}
}
- // ---------- Attempt 3: no-subs fallback (youtube handling only) ----------
const videoDir =
audioCheckVideoDir ??
path.join(channelDir, "data", canonicalId ?? "unknown");
const videoId = path.basename(videoDir);
+ // ---------- App-side extraction (transcribe handling) ----------
+ // After a successful non-audio-check transcribe download in app mode, produce
+ // audio.<fmt> from the downloaded source-media container and keep or discard it
+ // per the persistence plan. (Audio-check owns its own extraction; youtube
+ // handling extracts inside the no-subs fallback below.)
+ if (
+ lastSucceeded &&
+ !audioCheckEnabled &&
+ opts.channelConfig.handling === "transcribe" &&
+ plan.extractionMode === "app" &&
+ !opts.signal.aborted
+ ) {
+ await finalizeAppExtraction({
+ paths: opts.paths,
+ channelSlug: opts.channelSlug,
+ channelConfig: opts.channelConfig,
+ videoDir,
+ videoId,
+ fmt,
+ persist: plan.persist,
+ category: plan.category,
+ onLog: opts.onLog,
+ signal: opts.signal,
+ });
+ }
+
+ // ---------- Attempt 3: no-subs fallback (youtube handling only) ----------
if (
lastSucceeded &&
opts.channelConfig.handling === "youtube" &&
@@ -564,9 +763,8 @@ async function runManagedDownload(
const noCaptions = await metadataReportsNoCaptions(videoDir);
if (!hasTranscript && noCaptions) {
opts.onLog(
- `No subs available for ${videoId}; falling back to audio download + whisper.\n`,
+ `No subs available for ${videoId}; falling back to audio download + whisper${plan.persist ? " (keeping source video)" : ""}.\n`,
);
- const audioFmt: AudioFormat = opts.channelConfig.audioFormat ?? "mp3";
const fallbackConfig: ChannelConfig = {
...opts.channelConfig,
handling: "transcribe",
@@ -579,13 +777,14 @@ async function runManagedDownload(
// Feed yt-dlp the metadata it already wrote during the primary
// attempt instead of re-querying the extractor — saves a network
// round-trip per video, which adds up across batch runs and helps
- // dodge rate limits we'd otherwise burn on info we already have.
+ // dodge rate limits we'd otherwise burn on info we already have. The
+ // persistence plan applies here too, so a kept youtube video that lacks
+ // captions still archives its source container.
const fallbackArgs = [
"--ignore-config",
"--restrict-filenames",
...FULL_LOG_PROGRESS_ARGS,
- ...outputArgsForUrl(opts.videoUrl),
- ...transcribeHandlingArgs(fallbackConfig, true),
+ ...transcribeMediaArgs(opts.videoUrl, fallbackConfig, plan, fmt, true),
"--print",
`after_video:${ARCHIVE_MARKER} %(extractor)s %(id)s`,
...channelConfigArgs(fallbackConfig, fallbackCookieOverride),
@@ -614,6 +813,22 @@ async function runManagedDownload(
if (attemptSucceeded(fallbackRes.exitCode)) {
fellBackToTranscribe = true;
+ // In app mode, extract audio.<fmt> from the downloaded source container
+ // (and keep/discard it) before any inline whisper can read the audio.
+ if (plan.extractionMode === "app") {
+ await finalizeAppExtraction({
+ paths: opts.paths,
+ channelSlug: opts.channelSlug,
+ channelConfig: opts.channelConfig,
+ videoDir,
+ videoId,
+ fmt,
+ persist: plan.persist,
+ category: plan.category,
+ onLog: opts.onLog,
+ signal: opts.signal,
+ });
+ }
if (opts.inlineTranscribeOnFallback) {
// Inline whisper: matches whisperVideoAction's shape. Routes through
// the worker pool so the inline transcription respects worker config
@@ -623,7 +838,7 @@ async function runManagedDownload(
paths: opts.paths,
videoDir,
videoId,
- audioFilename: `audio.${audioFmt}`,
+ audioFilename: `audio.${fmt}`,
onLog: opts.onLog,
signal: opts.signal,
});
diff --git a/common/ytdlp/persistencePlan.test.ts b/common/ytdlp/persistencePlan.test.ts
@@ -0,0 +1,70 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { resolvePersistenceDecision } from "./persistencePlan";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test ytdlp/persistencePlan.test.ts
+
+test("outside the window with no overrides: audio-only via channel mode", () => {
+ const d = resolvePersistenceDecision({ inWindow: false, pinned: false });
+ assert.equal(d.persist, false);
+ assert.equal(d.extractionMode, "ytdlp");
+});
+
+test("channel app extraction is honored for audio-only downloads", () => {
+ const d = resolvePersistenceDecision({
+ inWindow: false,
+ pinned: false,
+ channelExtractionMode: "app",
+ });
+ assert.equal(d.persist, false);
+ assert.equal(d.extractionMode, "app");
+});
+
+test("inside the keep-latest window persists via app extraction", () => {
+ const d = resolvePersistenceDecision({ inWindow: true, pinned: false });
+ assert.equal(d.persist, true);
+ assert.equal(d.extractionMode, "app");
+ assert.equal(d.category, "keep-latest");
+});
+
+test("a do-not-clean pin persists even outside the window", () => {
+ const d = resolvePersistenceDecision({ inWindow: false, pinned: true });
+ assert.equal(d.persist, true);
+ assert.equal(d.extractionMode, "app");
+ assert.equal(d.category, "pin");
+});
+
+test("category is 'none' for an audio-only outcome", () => {
+ const d = resolvePersistenceDecision({ inWindow: false, pinned: false });
+ assert.equal(d.category, "none");
+});
+
+test("extractImmediately overrides the keep-latest window", () => {
+ const d = resolvePersistenceDecision({
+ inWindow: true,
+ pinned: true,
+ overrides: { extractImmediately: true },
+ });
+ assert.equal(d.persist, false);
+});
+
+test("keepSourceVideoOverride=true persists an out-of-window video", () => {
+ const d = resolvePersistenceDecision({
+ inWindow: false,
+ pinned: false,
+ overrides: { keepSourceVideoOverride: true },
+ });
+ assert.equal(d.persist, true);
+ assert.equal(d.extractionMode, "app");
+ assert.equal(d.category, "override");
+});
+
+test("keepSourceVideoOverride=false beats the window and extractImmediately", () => {
+ const d = resolvePersistenceDecision({
+ inWindow: true,
+ pinned: true,
+ overrides: { keepSourceVideoOverride: false, extractImmediately: true },
+ });
+ assert.equal(d.persist, false);
+});
diff --git a/common/ytdlp/persistencePlan.ts b/common/ytdlp/persistencePlan.ts
@@ -0,0 +1,96 @@
+import type { ExtractionMode } from "../lib/channelConfig";
+
+// The per-download persistence rule engine. Each individual download consults
+// the channel's keep-latest rule (plus any per-run overrides) to decide TWO
+// things:
+// 1. persist — keep the source video container (vs. audio-only)
+// 2. extractionMode — who extracts audio: yt-dlp's `-x` or the app's ffmpeg
+//
+// This is the heart of Phase 2: persistence is a per-channel rule evaluated per
+// individual download, NOT a separate runner/queue. Kept as a pure function so
+// the precedence is unit-testable in isolation from yt-dlp and the filesystem.
+
+export type PersistRunOverrides = {
+ // Force-keep (true) or force-discard (false) the source video for this run,
+ // overriding the channel's keep-latest rule. undefined = no override.
+ keepSourceVideoOverride?: boolean;
+ // Force "extract audio now and discard the container" even for a video the
+ // keep-latest rule would otherwise persist (the backfill / save-disk case).
+ extractImmediately?: boolean;
+};
+
+export type PersistenceDecisionInput = {
+ // This video falls inside the channel's keep-latest window (by upload date).
+ inWindow: boolean;
+ // This video carries a do-not-clean pin (manually protected / deleted-from-
+ // source). Pinned videos are irreplaceable, so we persist their source too.
+ pinned: boolean;
+ // The channel's configured extraction strategy for audio-only downloads.
+ channelExtractionMode?: ExtractionMode;
+ overrides?: PersistRunOverrides;
+};
+
+// Why a video's source is persisted — recorded on the saved-video pointer so the
+// retention prune (Phase 3) can tell a rolling keep-latest container (evictable
+// once it leaves the window) apart from a manually-archived one ("override") or a
+// pinned/irreplaceable one ("pin"), which must never be auto-pruned. "none" when
+// the video is not persisted.
+export type PersistCategory = "keep-latest" | "pin" | "override" | "none";
+
+export type PersistenceDecision = {
+ // Keep the downloaded source video container (Phase 3 moves it to the store;
+ // Phase 2 leaves it in the data dir as source-media.<ext>).
+ persist: boolean;
+ extractionMode: ExtractionMode;
+ // Coarse cause of the decision, used by the saved-store retention prune.
+ category: PersistCategory;
+ // Human-readable justification, surfaced in the download log.
+ reason: string;
+};
+
+// Precedence (highest first):
+// 1. per-run keepSourceVideoOverride === true -> persist
+// 2. per-run keepSourceVideoOverride === false -> don't persist
+// 3. per-run extractImmediately -> don't persist (extract now)
+// 4. do-not-clean pin -> persist
+// 5. inside the keep-latest window -> persist
+// 6. otherwise -> audio-only
+// Persisting implies app-side extraction (we need the container in hand); an
+// audio-only outcome uses the channel's configured mode (default "ytdlp").
+export function resolvePersistenceDecision(
+ input: PersistenceDecisionInput,
+): PersistenceDecision {
+ const o = input.overrides ?? {};
+ let persist: boolean;
+ let category: PersistCategory;
+ let reason: string;
+ if (o.keepSourceVideoOverride === true) {
+ persist = true;
+ category = "override";
+ reason = "per-run keep-source-video override";
+ } else if (o.keepSourceVideoOverride === false) {
+ persist = false;
+ category = "none";
+ reason = "per-run no-keep override";
+ } else if (o.extractImmediately) {
+ persist = false;
+ category = "none";
+ reason = "per-run extract-immediately override";
+ } else if (input.pinned) {
+ persist = true;
+ category = "pin";
+ reason = "do-not-clean pin";
+ } else if (input.inWindow) {
+ persist = true;
+ category = "keep-latest";
+ reason = "keep-latest window";
+ } else {
+ persist = false;
+ category = "none";
+ reason = "outside keep-latest window";
+ }
+ const extractionMode: ExtractionMode = persist
+ ? "app"
+ : (input.channelExtractionMode ?? "ytdlp");
+ return { persist, extractionMode, category, reason };
+}
diff --git a/common/ytdlp/runYtdlp.ts b/common/ytdlp/runYtdlp.ts
@@ -23,6 +23,7 @@ import {
import { resolveEffectiveAvailability } from "../lib/availability-server";
import { backfillAvailabilityFromMetadata } from "../controller/backfillAvailability";
import { resolveShardItems } from "../controller/shard";
+import { computeKeepWindow, type KeepWindow } from "../controller/keptVideos";
import { downloadOneManaged } from "./downloadOneManaged";
import type { TaskTracker } from "../jobs/taskHooks";
import type { JobProgress } from "../jobs/registry";
@@ -70,8 +71,15 @@ export type RunYtdlpOpts = {
progressBaseline?: number;
// download-one-audio only: full webpage URL of the single video to fetch.
singleVideoUrl?: string;
- // download-one-audio only: override channelConfig.audioFormat for this run.
+ // Override channelConfig.audioFormat for this run. Honored by download-one-audio
+ // and by the managed download paths (passed through to downloadOneManaged).
audioFormatOverride?: AudioFormat;
+ // Per-run persistence overrides (Phase 2), threaded to downloadOneManaged for
+ // every managed download in this run. keepSourceVideoOverride forces keep
+ // (true) / discard (false); extractImmediately forces extract-now + discard
+ // even for a video the keep-latest rule would persist.
+ keepSourceVideoOverride?: boolean;
+ extractImmediately?: boolean;
// download-one-audio only: appended after configArgs, before the URL.
extraYtdlpArgs?: string[];
// retry-bucket only: video IDs to limit the run to (matched against the
@@ -177,12 +185,20 @@ const OUTPUT_ARGS: string[] = [
// path literally to the canonical id and stop depending on the extractor id.
// Falls back to %(id)s for URLs whose canonical id is missing or not a safe
// directory name (the post-download reconcile pass cleans those up).
-export function outputArgsForUrl(url: string): string[] {
+// `mediaName` overrides the base name of the main media output (default
+// "audio"). App-side extraction downloads the source container as
+// `source-media.<ext>` so it's never confused with an extractable audio file;
+// every other caller keeps the historical `audio.<ext>`.
+export function outputArgsForUrl(
+ url: string,
+ opts: { mediaName?: string } = {},
+): string[] {
+ const mediaName = opts.mediaName ?? "audio";
const id = extractVideoId(url);
if (id && /^[\w.-]+$/.test(id) && id !== "." && id !== "..") {
return [
"-o",
- `data/${id}/audio.%(ext)s`,
+ `data/${id}/${mediaName}.%(ext)s`,
"-o",
`subtitle:data/${id}/transcript`,
"-o",
@@ -190,6 +206,17 @@ export function outputArgsForUrl(url: string): string[] {
"--no-write-playlist-metafiles",
];
}
+ if (mediaName !== "audio") {
+ return [
+ "-o",
+ `data/%(id)s/${mediaName}.%(ext)s`,
+ "-o",
+ "subtitle:data/%(id)s/transcript",
+ "-o",
+ "infojson:data/%(id)s/metadata",
+ "--no-write-playlist-metafiles",
+ ];
+ }
return OUTPUT_ARGS;
}
@@ -547,6 +574,16 @@ async function runManagedDownloads(
opts.channelConfig.sleepBetweenDownloadsSeconds ??
settings.sleepBetweenDownloadsSeconds;
+ // The channel's keep-latest window as a cutoff, computed ONCE per run from the
+ // current on-disk catalog. Each video downloaded below is classified against it
+ // by its own upload date — so the newest videos (not yet on disk) still get
+ // persisted. A keepLatest of 0/undefined yields an inert window.
+ const keepWindow: KeepWindow = await computeKeepWindow({
+ paths: opts.paths,
+ channelSlug: opts.channelSlug,
+ keepLatest: effectiveChannelConfig.keepLatest ?? 0,
+ });
+
const limit = pLimit(1);
let failedCount = 0;
let skippedCount = 0;
@@ -593,6 +630,10 @@ async function runManagedDownloads(
appendArchive: !opts.ignoreArchive,
inlineTranscribeOnFallback,
globalSkipLiveDownloads,
+ keepWindow,
+ keepSourceVideoOverride: opts.keepSourceVideoOverride,
+ extractImmediately: opts.extractImmediately,
+ audioFormatOverride: opts.audioFormatOverride,
});
} finally {
task?.end();
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,11 @@
# Changelog
## [Unreleased]
+- **First-class video-persistence UI (phase 5, the final phase): a Saved Videos area, per-channel retention controls, and per-video persist/unpersist.** The video-persistence subsystem built up over phases 1–4 is now driveable end to end from the editor. A new top-level **Saved videos** page (`/saved-videos`, in the Pool nav) summarizes the whole saved-video store — total count and size, per-channel breakdown (count, size, how many carry a backup checksum), the default store location, and the last backup time — and hosts the **backup configuration** (destination, scheduled on/off, interval) plus **Back up now** / **Verify backup** buttons. Each channel's **Cleanup stage** gains a **Retention & persistence** section (shown whenever keep-latest is on or the channel has saved videos) with live counts and three buttons: **Check kept videos** (re-probe the window for deleted-from-source videos and pin them), **Persist kept now** (a new bulk catch-up pass that re-fetches the source container for any in-window video whose source isn't saved yet — `persistKeptAction` / `common/controller/persistKept.ts`), and **Back up saved videos**. The **channel settings form** adds a Retention & persistence section: **keep latest** (window size), **extraction mode** (yt-dlp vs app-side ffmpeg), and a per-channel **saved-video store dir** override. Each **video page** gains a **Source video** card showing persisted status (file, size, stored time, keep reason, sha256, location) with an **Unpersist** control that moves the container back into the data dir, or a **Persist source video** button (re-fetch + archive) when it isn't saved yet. New job-kind label for `persist-kept`; `persist-kept` is re-runnable from bookmarks. The saved-video actions moved from `editor/app/savedVideos/` to `editor/app/saved-videos/` to match the route. Covered by `editor/e2e/saved-videos.spec.ts` and `common/controller/persistKept.test.ts`. (Deferred: surfacing kept-check/persist as Actionable-page rows, and streaming the player directly from the store — unpersist brings the container back to the data dir to play it.)
+- **Backups for the saved-video store: rsync mirror + per-backup manifest + drift verification (phase 4 of the video-persistence subsystem; backend + scheduler, UI lands later).** The (large, often irreplaceable) saved source videos can now be **backed up to a configured destination**. A backup walks every saved-video pointer across all channels (so per-channel store overrides are covered automatically) and **`rsync`-mirrors each container** into `<dest>/<slug>/<videoId>/` — incremental and resumable (`-a --partial`), additive (no deletes), so re-running only transfers changed or new files. It writes a **`backup-manifest.json`** at the destination root recording each container's canonical location, byte size, and a **streamed sha256**, and caches that hash back onto the live pointer. A **verify** step reads the manifest back and reports drift in four buckets — `missing`, `sizeMismatch`, `checksumMismatch` (re-hashing each present file), and `extra` (containers at the destination the manifest doesn't know about). New global settings block **`savedVideoBackup`** (`{ enabled, dest, intervalMinutes }`; a blank `dest` forces `enabled` off) plus a **`RSYNC_BIN`** env override. When enabled with a destination, the **sync scheduler** runs the backup automatically on its own cadence (a global, not per-channel, job — suppressed during quiet hours, tracked via `lastSavedVideoBackupAt`). Backups can also be run/verified manually via `backupSavedVideosAction` / `verifySavedVideoBackupAction` (managed jobs on a dedicated `saved-videos` queue). The destination is treated as a local filesystem path (a mounted backup disk). See the new `common/lib/savedVideoBackup.ts` (manifest types/parse), `common/controller/{backupSavedVideos,savedVideoInventory}.ts` (+ tests), `common/lib/paths.ts` (`rsyncBin`), `common/lib/settings.ts` (`savedVideoBackup`), `common/jobs/syncSchedulerState.ts`, `editor/app/savedVideos/backupActions.ts`, and `editor/app/scheduler/runTick.ts`.
+- **Saved-video store: persisted source videos move to a separate dir/disk, with retention pruning (phase 3 of the video-persistence subsystem; backend, UI lands later).** When the per-download persistence rule (phase 2) keeps a source video, the downloaded container is now **moved out of the per-video data dir into a separate saved-video store** — leaving only a small `saved-video.json` pointer behind — so the main data volume holds just audio + transcripts while the (large) source videos can live on another disk. The store root defaults to `<transcripts>/saved-videos`, is overridable globally via the **`SAVED_VIDEOS_DIR`** env var, and can be further overridden **per channel** (`savedVideosDir` in `config.json`); a video's container lands under `<root>/<slug>/<videoId>/`. The move is **cross-device-safe** (rename within a disk, copy-to-temp + atomic rename + unlink across disks) and **best-effort** — a failed move leaves the container in the data dir as `source-media.<ext>` (still persisted, just not relocated) rather than failing the download. **Transcription resolves from the store**: when no extracted `audio.*` exists, the transcribe fallback follows the pointer to the stored container (returned as a path relative to the video dir so both the local engine and the remote uploader read it correctly), so a kept-but-cleaned or archive-only video still transcribes. **Retention pruning** (the phase-2 follow-up) now bounds the store: the Clean-audio sweep also evicts any *keep-latest* container that has rolled out of the window — but **never** a manually-archived (`override`) or pinned/irreplaceable (`pin`/do-not-clean) one, distinguished by a `keepReason` recorded on each pointer. Reversible helpers ship for the upcoming UI: `unpersistSavedVideo` (move the container back) and `dropSavedVideo` (delete it). A reusable `checkDiskSpaceFor(dir, …)` lands so disk gating can target the store filesystem (used by the UI/backup phases). Note: source-video persistence is still skipped for audio-check channels (deferred), and saved-store counts aren't yet surfaced in the channel snapshot (lands with the phase-5 UI). See the new `common/lib/savedVideo.ts` (+ `savedVideo-server.ts` + tests), `common/controller/pruneSavedVideos.ts` (+ tests), `common/lib/paths.ts` (`savedVideosDir`), `common/lib/channelConfig.ts` (`savedVideosDir`), `common/lib/diskSpace.ts`, `common/ytdlp/{persistencePlan,downloadOneManaged}.ts`, `common/controller/{transcribeOne,cleanAudioFromTranscribed}.ts`.
+- **Per-download persistence rule + app-side audio extraction (phase 2 of the video-persistence subsystem; backend, UI lands later).** Each individual download now consults the channel's keep-latest rule (plus any per-run overrides) to decide *what to keep*: a video inside the keep-latest window — or one carrying a `do-not-clean` pin — downloads its **full source video** (`bestvideo*+bestaudio/best`) and the app extracts `audio.<fmt>` from it with ffmpeg, keeping the container as `source-media.<ext>` (a deliberately distinct name from `audio.<ext>` so it's never mistaken for cleanable audio); everything else stays audio-only as before. Crucially the keep decision is made per video by its **upload date against the channel's Nth-newest cutoff** (computed once per run via the new `computeKeepWindow`/`isInKeepWindow`), so the newest videos — which aren't on disk yet at download time — are correctly persisted. A new **`extractionMode`** channel setting (`"ytdlp"` default | `"app"`) selects who extracts audio for audio-only downloads; persisting always forces app-side extraction. **Per-run overrides** thread through `download-missing` (and the shared managed-download path): `keepSourceVideoOverride` (force keep/discard), `extractImmediately` (extract now + discard the container even on a keep channel — the disk-saving backfill case), and `audioFormatOverride`. **Transcription falls back to the source container** when no extracted `audio.*` exists (parakeet ffmpeg-slices any container), so a kept-but-cleaned video or an archive-only download is still transcribable. New per-video **"Archive source video"** action (`redownloadToArchiveAction`) re-fetches an existing video purely to grab + keep its source container without disturbing the transcript. Legacy `"ytdlp"`-mode downloads produce byte-identical yt-dlp args to before (no behavior change for existing channels). Note: persisted source containers currently remain in the data dir and are not auto-pruned when they roll out of the window, and source-video persistence is skipped (with a log note) for audio-check channels — both addressed by the saved-video store in phase 3. See `common/ytdlp/persistencePlan.ts` (+ tests), `common/controller/keptVideos.ts` (keep-window), `common/ytdlp/downloadOneManaged.ts`, `common/ytdlp/runYtdlp.ts`, `common/controller/transcribeOne.ts`, `common/lib/videoStatus.ts` (`source-media`/`isVideoContainer`), and `editor/app/channels/[slug]/pipelineActions.ts` / `videos/[id]/videoActions.ts`.
+- **New per-channel "keep latest N" retention rule that protects recent videos from the Clean-audio sweep and pins any that get deleted from their source (backend; UI lands in a later change).** A channel can set `keepLatest` (in `config.json` for now) to shield its newest N videos — by upload date, a rolling window — from the **Clean audio** cleanup: those dirs are skipped just like a `do-not-clean` marker, and the snapshot's reclaim estimates/cleanup buckets exclude them (a new `keptCount` is recorded). Because a kept video can later be **deleted from its source** (YouTube etc.) and become irreplaceable, a new **kept-deletion check** re-probes just the kept window's availability (reusing `runAvailabilityCheck` with `onlyIds` + `recheck-non-deleted`) and **permanently pins** any video found `deleted`/`private`/`members_only` with a `do-not-clean` marker, so it survives even after it rolls out of the window. The check runs on demand via the new `check-kept-deleted` managed job (`checkKeptDeletedAction`, re-runnable/bookmarkable) and automatically from the sync scheduler on its own cadence (new `syncScheduler.keepLatestCheckIntervalMinutes`, default daily; per-channel `lastKeptCheckAt` state; suppressed during quiet hours, capped per tick, and skipped for a channel just synced this tick). This is phase 1 of a larger **video-persistence** subsystem (per-download persistence rules, a separate saved-video store, and backups follow). See `common/lib/channelConfig.ts` (`keepLatest`), the new `common/controller/keptVideos.ts` (`computeKeptVideoIds` + tests) and `common/controller/checkKeptDeleted.ts`, `common/controller/cleanAudioFromTranscribed.ts`, `common/controller/channelSnapshot.ts`, and the scheduler wiring in `common/lib/settings.ts`, `common/jobs/syncSchedulerState.ts`, and `editor/app/scheduler/runTick.ts`.
- **The monitor widget can now carry optional control buttons (Pause/Resume Transcriptions, Drain all).** The `/widget` view stays read-only by default, but a new **Show control buttons** option in the widget builder (URL flag `controls=1`) adds an interactive row at the top with the same **Pause Transcriptions** toggle (between-segment GPU release) and **Drain all** as the full app — so a pinned/iframe monitor can pause the GPU or wind work down without opening the editor. The controls stay visible even when the widget is otherwise idle (so you can pause preemptively), and the widget refetches its worker payload on a pause/resume so the toggle flips immediately instead of waiting for the next poll. Defaults keep the widget control-free, so existing links render unchanged. See `editor/app/widget/lib/config.ts`, the new `editor/app/widget/components/WidgetControls.tsx`, `editor/app/widget/components/MonitorWidget.tsx`, and `editor/app/widget/builder/components/WidgetBuilder.tsx`.
- **"Pause Transcriptions" now frees the GPU between parakeet segments instead of running the in-flight video to completion.** The global pause control (renamed from "Pause all" → **Pause Transcriptions** / **Resume Transcriptions**) used to only stop handing out new worker slots — any in-flight transcription kept running until its whole file was done, so the GPU stayed busy. Pausing now *also* sends a graceful between-segment stop to any partial-capable in-flight job: a **parakeet** worker finishes the current window, caches it (`win-NNNN.json`), and exits `paused` (a skip, not a failure), so the GPU frees within one segment and the video resumes from its cached windows on the next run — the same mechanism as the per-worker **Stop & keep progress** button, now wired into the global pause. Non-parakeet engines (whisper-cpp, chough) keep prior behavior: they stop taking new work but run their in-flight file to completion. The button is also surfaced on the **Active Jobs** screen (`/jobs/active`) next to **Drain all**, not just the Workers page. `resumeAll()` restores each worker's pre-pause state as before. See `common/jobs/workerPool.ts` (`pauseAll`), the new `editor/app/jobs/components/PauseTranscriptionsButton.tsx` (shared by both screens), `editor/app/workers/components/WorkersView.tsx`, and `editor/app/jobs/active/page.tsx`.
- **"Drain all" (and the per-runner Drain) no longer hangs the auto-transcribe runner.** Draining could leave the runner stuck "draining" forever — appearing to freeze the app — whenever one of its per-video transcription units was *parked* in the worker pool waiting for a free slot at the moment drain fired (much more likely now that auto units yield slots to manual transcriptions). The runner forwarded only its hard-cancel signal — never the soft `drainSignal` — to a parked unit's `pool.acquire()`, so a soft drain could never unblock it: the unit's promise never settled, the runner's in-flight count never reached zero, and its drain loop spun indefinitely (hard **Cancel**/**Stop** always worked, since that signal *was* forwarded). The runner now threads `ctx.drainSignal` into each transcription unit, so a parked (not-yet-started) unit unblocks and is skipped on drain while a unit already transcribing finishes normally — correct drain semantics, and the runner finalizes promptly. See `common/controller/autoRunner.ts` (the `launchUnit` drainSignal wiring) and the new parked-unit drain regression test in `editor/e2e/auto-queue.spec.ts`.
diff --git a/editor/app/channels/[slug]/components/stages/CleanupStage.tsx b/editor/app/channels/[slug]/components/stages/CleanupStage.tsx
@@ -6,10 +6,13 @@ import { formatBytes } from "yt-dlp-transcript-common/lib/format";
import { QueueControl } from "../../../../components/QueueControl";
import { cancelJobAction } from "../../../../jobs/actions";
import {
+ checkKeptDeletedAction,
cleanAudioAction,
cleanExtraAudioFormatsAction,
removeWrongFormatAudioAction,
} from "../../whisperActions";
+import { persistKeptAction } from "../../persistActions";
+import { backupSavedVideosAction } from "../../../../saved-videos/backupActions";
import { VideoIdList } from "../VideoIdList";
type Props = {
@@ -22,6 +25,15 @@ type Props = {
transcribedAudioBytes: number;
extraFormatsBytes: number;
foreignAudioBytes: number;
+ // Retention & persistence (Phase 5). keepLatest is the configured window size
+ // (0 = off); keptCount is how many videos are currently in the window on disk;
+ // savedCount/savedBytes summarize this channel's persisted source videos.
+ // backupConfigured gates the "Back up saved videos" button.
+ keepLatest: number;
+ keptCount: number;
+ savedCount: number;
+ savedBytes: number;
+ backupConfigured: boolean;
};
export function CleanupStage({
@@ -33,11 +45,28 @@ export function CleanupStage({
transcribedAudioBytes,
extraFormatsBytes,
foreignAudioBytes,
+ keepLatest,
+ keptCount,
+ savedCount,
+ savedBytes,
+ backupConfigured,
}: Props) {
const defaultQueueKey = `channel:${slug}`;
const [cleanQueue, setCleanQueue] = useState(defaultQueueKey);
return (
<div className="flex flex-col gap-6">
+ {(keepLatest > 0 || savedCount > 0) && (
+ <RetentionSection
+ slug={slug}
+ keepLatest={keepLatest}
+ keptCount={keptCount}
+ savedCount={savedCount}
+ savedBytes={savedBytes}
+ backupConfigured={backupConfigured}
+ existingQueues={existingQueues}
+ defaultQueueKey={defaultQueueKey}
+ />
+ )}
<div className="flex flex-col gap-2">
<Heading
title="Clean audio for transcribed videos"
@@ -245,6 +274,130 @@ function ExtraAudioFormatsSection({
);
}
+function RetentionSection({
+ slug,
+ keepLatest,
+ keptCount,
+ savedCount,
+ savedBytes,
+ backupConfigured,
+ existingQueues,
+ defaultQueueKey,
+}: {
+ slug: string;
+ keepLatest: number;
+ keptCount: number;
+ savedCount: number;
+ savedBytes: number;
+ backupConfigured: boolean;
+ existingQueues: string[];
+ defaultQueueKey: string;
+}) {
+ const [queue, setQueue] = useState(defaultQueueKey);
+ return (
+ <div
+ aria-label="retention and persistence section"
+ className="flex flex-col gap-3 rounded border border-zinc-200 dark:border-zinc-800 p-3"
+ >
+ <div>
+ <h3 className="text-base font-semibold">Retention & persistence</h3>
+ <p className="text-sm text-zinc-500">
+ The keep-latest window protects the newest videos from cleanup and
+ persists their source containers to the saved-video store. Configure
+ the window size under{" "}
+ <a href={`/channels/${slug}/edit`} className="underline">
+ channel settings
+ </a>
+ .
+ </p>
+ </div>
+ <dl className="flex flex-wrap gap-x-6 gap-y-1 text-sm">
+ <Stat label="Keep latest" value={keepLatest > 0 ? String(keepLatest) : "off"} />
+ <Stat label="In window (on disk)" value={String(keptCount)} />
+ <Stat label="Saved source videos" value={String(savedCount)} />
+ <Stat
+ label="Saved store size"
+ value={savedBytes > 0 ? formatBytes(savedBytes) : "—"}
+ />
+ </dl>
+ <div className="flex flex-col gap-2 sm:flex-row sm:flex-wrap sm:gap-4">
+ <StreamActionLog
+ trigger={() => checkKeptDeletedAction(slug, queue)}
+ cancelAction={cancelJobAction}
+ buttonLabel="Check kept videos"
+ runningLabel="Checking…"
+ label="Check kept videos"
+ extraControls={
+ <QueueControl
+ value={queue}
+ onChange={setQueue}
+ defaultQueueKey={defaultQueueKey}
+ existingQueues={existingQueues}
+ actionLabel="Check kept videos"
+ />
+ }
+ />
+ <StreamActionLog
+ trigger={() => persistKeptAction(slug, queue)}
+ cancelAction={cancelJobAction}
+ buttonLabel="Persist kept now"
+ runningLabel="Persisting…"
+ label="Persist kept videos"
+ extraControls={
+ <QueueControl
+ value={queue}
+ onChange={setQueue}
+ defaultQueueKey={defaultQueueKey}
+ existingQueues={existingQueues}
+ actionLabel="Persist kept videos"
+ />
+ }
+ />
+ {backupConfigured && (
+ <StreamActionLog
+ trigger={() => backupSavedVideosAction()}
+ cancelAction={cancelJobAction}
+ buttonLabel="Back up saved videos"
+ runningLabel="Backing up…"
+ label="Back up saved videos"
+ />
+ )}
+ </div>
+ <p className="text-xs text-zinc-500">
+ <strong>Check kept videos</strong> re-probes the window for
+ deleted-from-source videos and pins them.{" "}
+ <strong>Persist kept now</strong> re-fetches the source container for any
+ in-window video whose source isn't saved yet.
+ {backupConfigured ? (
+ <>
+ {" "}
+ <strong>Back up saved videos</strong> mirrors the whole saved-video
+ store (all channels) to the configured destination.
+ </>
+ ) : (
+ <>
+ {" "}
+ Configure a backup destination under{" "}
+ <a href="/saved-videos" className="underline">
+ Saved videos
+ </a>{" "}
+ to enable backups.
+ </>
+ )}
+ </p>
+ </div>
+ );
+}
+
+function Stat({ label, value }: { label: string; value: string }) {
+ return (
+ <div className="flex flex-col">
+ <dt className="text-xs uppercase tracking-wide text-zinc-500">{label}</dt>
+ <dd className="font-medium tabular-nums">{value}</dd>
+ </div>
+ );
+}
+
function ReclaimEstimate({ bytes }: { bytes: number }) {
return (
<p className="text-sm text-zinc-600 dark:text-zinc-400">
diff --git a/editor/app/channels/[slug]/page.tsx b/editor/app/channels/[slug]/page.tsx
@@ -15,6 +15,8 @@ import {
} from "yt-dlp-transcript-common/controller/channelSnapshot";
import { loadFailedTranscriptions } from "yt-dlp-transcript-common/controller/failedTranscriptions";
import { loadFailedTranscodings } from "yt-dlp-transcript-common/controller/failedTranscodings";
+import { savedVideoTotals } from "yt-dlp-transcript-common/controller/savedVideoInventory";
+import { getSettings } from "yt-dlp-transcript-common/lib/settings";
import {
loadShardConfig,
type ShardConfig,
@@ -144,6 +146,10 @@ export default async function ChannelDetailPage({
: queueKeyForUrl(config.url);
const existing = await readChannelSnapshot(paths, slug);
const snapshot = existing ?? (await generateChannelSnapshot(paths, slug));
+ // Saved-video store summary for this channel + whether backups are configured,
+ // for the Cleanup stage's Retention & persistence section (Phase 5).
+ const savedTotals = await savedVideoTotals({ paths, channelSlug: slug });
+ const backupConfigured = getSettings().savedVideoBackup.dest.trim() !== "";
// The retry-failures panel is interactive (a click mutates the file), so
// read it fresh on every render — the snapshot bucket only reflects state
// at refresh time.
@@ -266,6 +272,11 @@ export default async function ChannelDetailPage({
transcribedAudioBytes={snapshot.cleanupBytes?.transcribedWithAudio ?? 0}
extraFormatsBytes={snapshot.cleanupBytes?.multipleAudioFormats ?? 0}
foreignAudioBytes={snapshot.cleanupBytes?.foreignAudio ?? 0}
+ keepLatest={config.keepLatest ?? 0}
+ keptCount={snapshot.keptCount ?? 0}
+ savedCount={savedTotals.count}
+ savedBytes={savedTotals.bytes}
+ backupConfigured={backupConfigured}
/>
),
diagnostics: (
diff --git a/editor/app/channels/[slug]/persistActions.ts b/editor/app/channels/[slug]/persistActions.ts
@@ -0,0 +1,57 @@
+"use server";
+
+import { revalidatePath } from "next/cache";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { getSettings } from "yt-dlp-transcript-common/lib/settings";
+import { checkDiskSpace } from "yt-dlp-transcript-common/lib/diskSpace";
+import { formatBytes } from "yt-dlp-transcript-common/lib/format";
+import {
+ downloadQueueKey,
+ resolveQueueKey,
+} from "yt-dlp-transcript-common/lib/queueKeys";
+import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels";
+import { persistKept } from "yt-dlp-transcript-common/controller/persistKept";
+import {
+ runManagedFunction,
+ type StreamActionResult,
+} from "yt-dlp-transcript-common/jobs/streamCommand";
+
+// Bulk catch-up: ensure every video in the channel's keep-latest window has its
+// source container saved to the store. Re-downloads only those not already
+// saved. Runs on the channel's download queue since it issues real downloads.
+export async function persistKeptAction(
+ slug: string,
+ queueKey?: string,
+): Promise<StreamActionResult> {
+ const paths = getPaths();
+ const config = await readChannelConfig(paths, slug);
+ if (!config) return { ok: false, error: `Channel "${slug}" not found` };
+ const disk = await checkDiskSpace(paths, getSettings());
+ if (!disk.ok) {
+ return {
+ ok: false,
+ error:
+ `Low disk space: ${formatBytes(disk.freeBytes)} free, ` +
+ `${formatBytes(disk.thresholdBytes)} required. Free up space or ` +
+ `lower the floor in Settings.`,
+ };
+ }
+ return runManagedFunction({
+ kind: "persist-kept",
+ queueKey: resolveQueueKey(downloadQueueKey(config), queueKey),
+ paths,
+ channelSlug: slug,
+ spec: { kind: "persist-kept", slug, params: { queueKey } },
+ fn: async (onLog, signal) => {
+ await persistKept({
+ paths,
+ channelSlug: slug,
+ channelConfig: config,
+ onLog,
+ signal,
+ });
+ revalidatePath(`/channels/${slug}`);
+ revalidatePath("/saved-videos");
+ },
+ });
+}
diff --git a/editor/app/channels/[slug]/pipelineActions.ts b/editor/app/channels/[slug]/pipelineActions.ts
@@ -3,6 +3,7 @@
import { revalidatePath } from "next/cache";
import {
HANDLING_VALUES,
+ type AudioFormat,
type ChannelHandling,
} from "yt-dlp-transcript-common/lib/channelConfig";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
@@ -69,6 +70,11 @@ async function runPipelineAction(
shardIndex?: number;
bucketIds?: ReadonlyArray<string>;
handlingOverride?: ChannelHandling;
+ // Per-run persistence overrides (Phase 2), forwarded to runYtdlp →
+ // downloadOneManaged for every managed download in this run.
+ keepSourceVideoOverride?: boolean;
+ extractImmediately?: boolean;
+ audioFormatOverride?: AudioFormat;
// Replay descriptor, forwarded onto the job record so it can be bookmarked.
spec?: JobSpec;
},
@@ -160,6 +166,9 @@ async function runPipelineAction(
shardIndex: options?.shardIndex,
bucketIds: options?.bucketIds,
handlingOverride: options?.handlingOverride,
+ keepSourceVideoOverride: options?.keepSourceVideoOverride,
+ extractImmediately: options?.extractImmediately,
+ audioFormatOverride: options?.audioFormatOverride,
setProgress,
progressBaseline,
// On a 429/network failure, record the shared per-platform cooldown so
@@ -214,6 +223,12 @@ export async function downloadMissingAction(
abortOnError?: boolean,
shardTotal?: number,
shardIndex?: number,
+ // Per-run persistence overrides (Phase 2). The canonical use is a backfill that
+ // extracts audio immediately to save disk (extractImmediately), but a forced
+ // keep / audio-format override is supported too.
+ keepSourceVideoOverride?: boolean,
+ extractImmediately?: boolean,
+ audioFormatOverride?: AudioFormat,
): Promise<StreamActionResult> {
return runPipelineAction(
slug,
@@ -225,10 +240,22 @@ export async function downloadMissingAction(
abortOnError,
shardTotal,
shardIndex,
+ keepSourceVideoOverride,
+ extractImmediately,
+ audioFormatOverride,
spec: {
kind: "download-missing",
slug,
- params: { queueKey, ignoreArchive, abortOnError, shardTotal, shardIndex },
+ params: {
+ queueKey,
+ ignoreArchive,
+ abortOnError,
+ shardTotal,
+ shardIndex,
+ keepSourceVideoOverride,
+ extractImmediately,
+ audioFormatOverride,
+ },
},
},
);
diff --git a/editor/app/channels/[slug]/videos/[id]/components/VideoPanel.tsx b/editor/app/channels/[slug]/videos/[id]/components/VideoPanel.tsx
@@ -9,6 +9,7 @@ import {
type ChannelHandling,
} from "yt-dlp-transcript-common/lib/channelConfig";
import type { DownloadOutcomeRecord } from "yt-dlp-transcript-common/lib/downloadOutcome";
+import type { SavedVideoPointer } from "yt-dlp-transcript-common/lib/savedVideo";
import type { AvailabilityHistoryEntry } from "yt-dlp-transcript-common/lib/availability";
import { formatBytes } from "yt-dlp-transcript-common/lib/format";
import { QueueControl } from "../../../../../components/QueueControl";
@@ -22,9 +23,11 @@ import {
downloadVideoPipelineAction,
markVideoUntranscribableAction,
setPrimaryTranscriptAction,
+ redownloadToArchiveAction,
toggleDoNotCleanAction,
transcodeAudioAction,
transcribeOneAction,
+ unpersistVideoAction,
whisperVideoAction,
type DeleteDirActionResult,
} from "../videoActions";
@@ -56,6 +59,9 @@ type Props = {
availabilityHistory: AvailabilityHistoryEntry[];
channelAudioFormat?: AudioFormat;
doNotClean?: boolean;
+ // This video's saved-video pointer when its source container is persisted to
+ // the store, else null. Drives the Source-video persistence card (Phase 5).
+ savedVideo?: SavedVideoPointer | null;
prevHref?: string;
nextHref?: string;
position?: { index: number; total: number };
@@ -132,6 +138,7 @@ export function VideoPanel({
availabilityHistory,
channelAudioFormat,
doNotClean = false,
+ savedVideo = null,
prevHref,
nextHref,
position,
@@ -306,6 +313,27 @@ export function VideoPanel({
)}
<PipelineStageCard
+ id="source-video-persistence"
+ title="Source video"
+ summary={
+ savedVideo
+ ? `Persisted to the saved-video store (${formatBytes(savedVideo.bytes)}).`
+ : "Persist the source video to the saved-video store."
+ }
+ defaultOpen={!!savedVideo}
+ tone={savedVideo ? "ok" : "neutral"}
+ >
+ <SourceVideoSection
+ slug={slug}
+ videoId={videoId}
+ handling={handling}
+ defaultQueueKey={defaultQueueKey}
+ existingQueues={existingQueues}
+ savedVideo={savedVideo}
+ />
+ </PipelineStageCard>
+
+ <PipelineStageCard
id="archive-media"
title="Archive media"
summary={
@@ -848,6 +876,126 @@ function MarkUntranscribableSection({
);
}
+function SourceVideoSection({
+ slug,
+ videoId,
+ handling,
+ defaultQueueKey,
+ existingQueues,
+ savedVideo,
+}: {
+ slug: string;
+ videoId: string;
+ handling: ChannelHandling;
+ defaultQueueKey: string;
+ existingQueues: string[];
+ savedVideo: SavedVideoPointer | null;
+}) {
+ const [queueKey, setQueueKey] = useState(defaultQueueKey);
+ const [pending, startTransition] = useTransition();
+ const [error, setError] = useState<string | null>(null);
+
+ if (handling !== "transcribe") {
+ return (
+ <p className="text-sm text-zinc-500">
+ Source-video persistence applies to transcribe-handling channels only.
+ </p>
+ );
+ }
+
+ if (savedVideo) {
+ return (
+ <div className="flex flex-col gap-3">
+ <p className="text-sm text-zinc-500">
+ The source container for this video is persisted to the saved-video
+ store, so the Clean-audio sweep can reclaim its <code>audio.*</code>{" "}
+ without losing the original. Transcription falls back to it when no
+ audio is on disk.
+ </p>
+ <dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-sm">
+ <dt className="text-zinc-500">File</dt>
+ <dd className="font-mono break-all">{savedVideo.file}</dd>
+ <dt className="text-zinc-500">Size</dt>
+ <dd className="tabular-nums">{formatBytes(savedVideo.bytes)}</dd>
+ <dt className="text-zinc-500">Stored</dt>
+ <dd>{new Date(savedVideo.storedAt).toLocaleString()}</dd>
+ {savedVideo.keepReason && (
+ <>
+ <dt className="text-zinc-500">Reason</dt>
+ <dd>{savedVideo.keepReason}</dd>
+ </>
+ )}
+ {savedVideo.sha256 && (
+ <>
+ <dt className="text-zinc-500">sha256</dt>
+ <dd className="font-mono text-xs break-all">
+ {savedVideo.sha256}
+ </dd>
+ </>
+ )}
+ <dt className="text-zinc-500">Location</dt>
+ <dd className="font-mono text-xs break-all">{savedVideo.dir}</dd>
+ </dl>
+ <p className="text-xs text-zinc-500">
+ Unpersist moves the container back into this video's data dir
+ (where it appears under Files and plays inline) and removes the
+ pointer.
+ </p>
+ <button
+ type="button"
+ disabled={pending}
+ aria-label={`unpersist source video ${videoId}`}
+ onClick={() => {
+ setError(null);
+ startTransition(async () => {
+ const res = await unpersistVideoAction(slug, videoId);
+ if (!res.ok) setError(res.error);
+ });
+ }}
+ className="self-start px-3 py-2 rounded-md border border-zinc-300 dark:border-zinc-700 text-sm font-medium hover:bg-zinc-100 dark:hover:bg-zinc-800 disabled:opacity-50"
+ >
+ {pending ? "Restoring…" : "Unpersist (move back to data dir)"}
+ </button>
+ {error && (
+ <span
+ className="text-sm text-red-600 dark:text-red-400"
+ aria-label="unpersist error"
+ >
+ {error}
+ </span>
+ )}
+ </div>
+ );
+ }
+
+ return (
+ <div className="flex flex-col gap-3">
+ <p className="text-sm text-zinc-500">
+ Re-fetch this video's full source container and move it into the
+ saved-video store, without disturbing the existing transcript. Use this
+ to archive a video that was downloaded audio-only. (Videos in a
+ channel's keep-latest window persist automatically on download.)
+ </p>
+ <StreamActionLog
+ trigger={() => redownloadToArchiveAction(slug, videoId, queueKey)}
+ cancelAction={cancelJobAction}
+ buttonLabel="Persist source video"
+ runningLabel="Archiving…"
+ label={`Persist source video for ${videoId}`}
+ extraControls={
+ <QueueControl
+ value={queueKey}
+ onChange={setQueueKey}
+ defaultQueueKey={defaultQueueKey}
+ existingQueues={existingQueues}
+ actionLabel={`Persist source video for ${videoId}`}
+ />
+ }
+ />
+ </div>
+ );
+}
+
function DoNotCleanSection({
slug,
videoId,
diff --git a/editor/app/channels/[slug]/videos/[id]/page.tsx b/editor/app/channels/[slug]/videos/[id]/page.tsx
@@ -8,6 +8,7 @@ import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels"
import { loadDownloadOutcome } from "yt-dlp-transcript-common/lib/downloadOutcome-server";
import { loadAvailability } from "yt-dlp-transcript-common/lib/availability-server";
import { isDoNotClean } from "yt-dlp-transcript-common/lib/doNotClean-server";
+import { loadSavedVideo } from "yt-dlp-transcript-common/lib/savedVideo-server";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import { resolvePrimaryVtt } from "yt-dlp-transcript-common/lib/videoStatus";
import {
@@ -101,6 +102,7 @@ export default async function VideoDetailPage({
const availabilityRecord = await loadAvailability(videoDir);
const availabilityHistory = availabilityRecord?.history ?? [];
const doNotClean = await isDoNotClean(videoDir);
+ const savedVideo = await loadSavedVideo(videoDir);
const registry = getRegistry();
const existingQueues = registry.activeQueueNames();
@@ -179,6 +181,7 @@ export default async function VideoDetailPage({
availabilityHistory={availabilityHistory}
channelAudioFormat={config.audioFormat}
doNotClean={doNotClean}
+ savedVideo={savedVideo}
/>
</div>
);
diff --git a/editor/app/channels/[slug]/videos/[id]/videoActions.ts b/editor/app/channels/[slug]/videos/[id]/videoActions.ts
@@ -9,7 +9,9 @@ import type {
ChannelConfig,
} from "yt-dlp-transcript-common/lib/channelConfig";
import { AUDIO_FORMAT_VALUES } from "yt-dlp-transcript-common/lib/channelConfig";
-import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { getPaths, type Paths } from "yt-dlp-transcript-common/lib/paths";
+import { checkDiskSpace } from "yt-dlp-transcript-common/lib/diskSpace";
+import { formatBytes } from "yt-dlp-transcript-common/lib/format";
import {
audioFilesToRemove,
isRealAudioFile,
@@ -26,6 +28,7 @@ import { pruneFailedTranscriptions } from "yt-dlp-transcript-common/controller/f
import { transcodeAudio } from "yt-dlp-transcript-common/controller/transcode";
import { transcribeWithWorker } from "yt-dlp-transcript-common/controller/transcribeOne";
import { findVideoSourceUrl } from "yt-dlp-transcript-common/controller/undownloadedVideos";
+import { unpersistSavedVideo } from "yt-dlp-transcript-common/lib/savedVideo-server";
import { getSettings } from "yt-dlp-transcript-common/lib/settings";
import { downloadOneManaged } from "yt-dlp-transcript-common/ytdlp/downloadOneManaged";
import { runYtdlp } from "yt-dlp-transcript-common/ytdlp/runYtdlp";
@@ -171,6 +174,82 @@ export async function downloadVideoPipelineAction(
});
}
+// Re-fetch an already-downloaded video purely to archive its SOURCE container,
+// without disturbing the existing transcript. Forces the persistence rule on
+// (keepSourceVideoOverride=true) so downloadOneManaged downloads the full video,
+// app-extracts audio, and keeps the container (Phase 3 moves it to the saved
+// store). Works on a video already in the archive — downloadOneManaged has no
+// archive prefilter, so it always re-downloads.
+export async function redownloadToArchiveAction(
+ slug: string,
+ videoId: string,
+ queueKey?: string,
+): Promise<StreamActionResult> {
+ const r = await loadConfigOrError(slug);
+ if (!r.ok) return r;
+ const paths = getPaths();
+ const url = await findVideoSourceUrl(paths, slug, videoId, r.config);
+ if (!url) {
+ return {
+ ok: false,
+ error:
+ "Could not determine the video URL: no metadata.info.json and the playlist does not contain a matching entry.",
+ };
+ }
+ const err = await lowDiskError(paths);
+ if (err) return err;
+ const settings = getSettings();
+ return runManagedFunction({
+ kind: "redownload-archive",
+ queueKey: videoQueueKey(r.config, queueKey),
+ paths,
+ channelSlug: slug,
+ videoId,
+ fn: async (onLog, signal, _setProgress, ctx) => {
+ const task = makeTaskTracker(ctx, onLog).start({
+ id: videoId,
+ label: videoId,
+ kind: "download",
+ });
+ try {
+ onLog(`Re-downloading ${videoId} to archive its source video…\n`);
+ await downloadOneManaged({
+ channelSlug: slug,
+ channelConfig: r.config,
+ paths,
+ videoUrl: url,
+ onLog: task.onLog,
+ signal,
+ globalCookiesFromBrowser: settings.cookiesFromBrowser || undefined,
+ inlineTranscribeOnFallback: settings.inlineTranscribeOnFallback,
+ globalSkipLiveDownloads: settings.skipLiveDownloads,
+ appendArchive: true,
+ keepSourceVideoOverride: true,
+ });
+ revalidatePath(`/channels/${slug}/videos/${videoId}`);
+ revalidatePath(`/channels/${slug}`);
+ } finally {
+ task.end();
+ }
+ },
+ });
+}
+
+// Preflight disk-space gate shared with the per-video download actions.
+async function lowDiskError(
+ paths: Paths,
+): Promise<{ ok: false; error: string } | null> {
+ const disk = await checkDiskSpace(paths, getSettings());
+ if (disk.ok) return null;
+ return {
+ ok: false,
+ error:
+ `Low disk space: ${formatBytes(disk.freeBytes)} free, ` +
+ `${formatBytes(disk.thresholdBytes)} required. Free up space or ` +
+ `lower the floor in Settings.`,
+ };
+}
+
export async function whisperVideoAction(
slug: string,
videoId: string,
@@ -441,3 +520,22 @@ export async function toggleDoNotCleanAction(
requestChannelSnapshot(getPaths(), slug);
return { ok: true };
}
+
+// Reverse persistence: move this video's stored source container back into its
+// data dir and drop the pointer (Phase 5). The companion "persist" direction is
+// redownloadToArchiveAction, which re-fetches the container when it's not on
+// disk. Returns false-shaped error if there was nothing persisted to reverse.
+export async function unpersistVideoAction(
+ slug: string,
+ videoId: string,
+): Promise<{ ok: true } | { ok: false; error: string }> {
+ const videoDir = videoDirOf(slug, videoId);
+ const reversed = await unpersistSavedVideo(videoDir);
+ if (!reversed) {
+ return { ok: false, error: "This video has no persisted source to restore." };
+ }
+ revalidatePath(`/channels/${slug}/videos/${videoId}`);
+ revalidatePath(`/channels/${slug}`);
+ revalidatePath("/saved-videos");
+ return { ok: true };
+}
diff --git a/editor/app/channels/[slug]/whisperActions.ts b/editor/app/channels/[slug]/whisperActions.ts
@@ -25,6 +25,7 @@ import {
} from "yt-dlp-transcript-common/controller/failedTranscodings";
import { clearFailedTranscriptions } from "yt-dlp-transcript-common/controller/failedTranscriptions";
import { cleanAudioFromTranscribed } from "yt-dlp-transcript-common/controller/cleanAudioFromTranscribed";
+import { checkKeptDeleted } from "yt-dlp-transcript-common/controller/checkKeptDeleted";
import { cleanExtraAudioFormats } from "yt-dlp-transcript-common/controller/cleanExtraAudioFormats";
import { removeWrongFormatAudio } from "yt-dlp-transcript-common/controller/removeWrongFormatAudio";
import { verifyTranscripts } from "yt-dlp-transcript-common/controller/verifyTranscripts";
@@ -367,6 +368,36 @@ export async function cleanAudioAction(
});
}
+// Re-probe the channel's keep-latest window for source deletion and pin any
+// gone videos (do-not-clean) so they survive even after rolling out of the
+// window. Mirrors cleanAudioAction's managed-job shape. Also driven by the sync
+// scheduler (editor/app/scheduler/runTick.ts).
+export async function checkKeptDeletedAction(
+ slug: string,
+ queueKey?: string,
+): Promise<StreamActionResult> {
+ const paths = getPaths();
+ return runManagedFunction({
+ kind: "check-kept-deleted",
+ queueKey: resolveQueueKey(channelQueueKey(slug), queueKey),
+ paths,
+ channelSlug: slug,
+ spec: { kind: "check-kept-deleted", slug, params: { queueKey } },
+ fn: async (onLog, signal) => {
+ const result = await checkKeptDeleted({
+ channelSlug: slug,
+ paths,
+ onLog,
+ signal,
+ });
+ onLog(
+ `Kept-deletion check: inspected ${result.kept}, ${result.deleted} gone from source, pinned ${result.pinned}.`,
+ );
+ revalidatePath(`/channels/${slug}`);
+ },
+ });
+}
+
export type VerifyResult =
| { ok: true; duplicates: string[]; missing: string[] }
| { ok: false; error: string };
diff --git a/editor/app/channels/components/ChannelForm.tsx b/editor/app/channels/components/ChannelForm.tsx
@@ -162,6 +162,51 @@ export function ChannelForm({
</label>
<AudioCheckFields config={c} />
</Section>
+ <Section title="Retention & persistence">
+ <label className="flex flex-col gap-1 text-sm">
+ <span className="font-medium">Keep latest (source videos)</span>
+ <input
+ type="number"
+ name="keepLatest"
+ defaultValue={c?.keepLatest != null ? String(c.keepLatest) : ""}
+ min={0}
+ max={100000}
+ placeholder="0 (off)"
+ aria-label="keep latest"
+ className="rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 px-2 py-1 text-sm w-32"
+ />
+ <span className="text-xs text-zinc-500">
+ Protect the newest N videos from the Clean-audio sweep and persist
+ their source containers to the saved-video store. A kept video later
+ found deleted-from-source is pinned permanently. Blank or 0 disables.
+ </span>
+ </label>
+ <label className="flex flex-col gap-1 text-sm">
+ <span className="font-medium">Extraction mode</span>
+ <select
+ name="extractionMode"
+ defaultValue={c?.extractionMode ?? ""}
+ aria-label="extraction mode"
+ className="rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 px-2 py-1 text-sm"
+ >
+ <option value="">Default (yt-dlp extracts)</option>
+ <option value="ytdlp">yt-dlp (-x postprocessor)</option>
+ <option value="app">App (download container, ffmpeg extracts)</option>
+ </select>
+ <span className="text-xs text-zinc-500">
+ Who extracts audio for transcribe-handling downloads. App mode keeps
+ the source container in hand so it can be persisted; the keep-latest
+ rule forces app mode for the videos it persists regardless.
+ </span>
+ </label>
+ <Field
+ label="Saved-video store dir"
+ name="savedVideosDir"
+ defaultValue={c?.savedVideosDir ?? ""}
+ placeholder="(global default)"
+ hint="Per-channel override for where this channel's persisted source videos live (e.g. a larger disk). Blank uses the global SAVED_VIDEOS_DIR default."
+ />
+ </Section>
<CollapsibleSection title="Advanced">
<label className="flex flex-col gap-1 text-sm">
<span className="font-medium">
diff --git a/editor/app/channels/components/parseChannelForm.ts b/editor/app/channels/components/parseChannelForm.ts
@@ -6,9 +6,11 @@ import {
AUDIO_CHECK_INTERVAL_MIN_SECONDS,
AUDIO_CHECK_MAX_ROLLBACKS_MAX,
AUDIO_CHECK_MAX_ROLLBACKS_MIN,
+ KEEP_LATEST_MAX,
SYNC_INTERVAL_MAX_MINUTES,
SYNC_INTERVAL_MIN_MINUTES,
type AudioCheckConfig,
+ type ExtractionMode,
} from "yt-dlp-transcript-common/lib/channelConfig";
import {
PLATFORM_VALUES,
@@ -31,6 +33,9 @@ export const CHANNEL_FORM_FIELDS = [
"url",
"audioFormat",
"keepSourceVideo",
+ "keepLatest",
+ "extractionMode",
+ "savedVideosDir",
"ytdlpExtraArgs",
"syncIntervalMinutes",
"sleepBetweenDownloadsSeconds",
@@ -65,6 +70,31 @@ export function parseChannelForm(formData: FormData): ParsedChannelForm {
? audioFormatRaw
: undefined;
const keepSourceVideo = formData.get("keepSourceVideo") != null;
+
+ // Keep-latest retention/persistence window. Blank = inherit-off (omit), so a
+ // cleared input clears the stored value. "0" is the explicit "disabled"
+ // sentinel; any positive value is the window size.
+ const keepLatestRaw = String(formData.get("keepLatest") ?? "").trim();
+ let keepLatest: number | undefined;
+ if (keepLatestRaw) {
+ const n = Number.parseInt(keepLatestRaw, 10);
+ if (!Number.isFinite(n) || n < 0 || n > KEEP_LATEST_MAX) {
+ throw new Error(`Keep latest must be 0 (off) or 1–${KEEP_LATEST_MAX}`);
+ }
+ keepLatest = n;
+ }
+
+ // Audio-extraction owner. "" = inherit the default ("ytdlp"); otherwise an
+ // explicit mode. Persisting a source video forces "app" at download time
+ // regardless of this setting.
+ const extractionModeRaw = String(formData.get("extractionMode") ?? "").trim();
+ const extractionMode: ExtractionMode | undefined =
+ extractionModeRaw === "ytdlp" || extractionModeRaw === "app"
+ ? extractionModeRaw
+ : undefined;
+
+ const savedVideosDir = stringOrUndef(formData, "savedVideosDir");
+
const argsBlob = String(formData.get("ytdlpExtraArgs") ?? "").trim();
const ytdlpExtraArgs = argsBlob
? argsBlob
@@ -175,6 +205,9 @@ export function parseChannelForm(formData: FormData): ParsedChannelForm {
if (url) config.url = url;
if (audioFormat) config.audioFormat = audioFormat;
if (keepSourceVideo) config.keepSourceVideo = true;
+ if (keepLatest != null) config.keepLatest = keepLatest;
+ if (extractionMode) config.extractionMode = extractionMode;
+ if (savedVideosDir) config.savedVideosDir = savedVideosDir;
if (ytdlpExtraArgs?.length) config.ytdlpExtraArgs = ytdlpExtraArgs;
if (syncIntervalMinutes != null) {
config.syncIntervalMinutes = syncIntervalMinutes;
diff --git a/editor/app/jobs/jobKindLabels.ts b/editor/app/jobs/jobKindLabels.ts
@@ -13,7 +13,13 @@ const JOB_KIND_LABELS: Record<string, string> = {
"download-missing": "Download missing",
"download-missing-subs": "Download missing subs",
"import-one": "Import video",
+ "redownload-archive": "Archive source video",
"retry-bucket": "Retry",
+ "clean-audio-transcribed": "Clean audio",
+ "check-kept-deleted": "Check kept videos",
+ "persist-kept": "Persist kept videos",
+ "backup-saved-videos": "Back up saved videos",
+ "verify-saved-video-backup": "Verify saved-video backup",
sync: "Sync",
};
diff --git a/editor/app/jobs/runJobSpec.ts b/editor/app/jobs/runJobSpec.ts
@@ -14,6 +14,7 @@ import {
syncAction,
} from "../channels/[slug]/pipelineActions";
import {
+ checkKeptDeletedAction,
cleanAudioAction,
cleanExtraAudioFormatsAction,
clearFailedTranscodingsAction,
@@ -25,6 +26,7 @@ import {
transcribeBucketAction,
transcribeMissingAction,
} from "../channels/[slug]/whisperActions";
+import { persistKeptAction } from "../channels/[slug]/persistActions";
// The single place that maps a stored JobSpec back to the server action that
// runs it. Bucket jobs re-derive their work from the channel's CURRENT snapshot
@@ -101,6 +103,9 @@ export async function runJobSpec(spec: JobSpec): Promise<StreamActionResult> {
bool(p.abortOnError),
num(p.shardTotal),
num(p.shardIndex),
+ bool(p.keepSourceVideoOverride),
+ bool(p.extractImmediately),
+ str(p.audioFormatOverride) as AudioFormat | undefined,
);
case "sync":
return syncAction(spec.slug, queueKey);
@@ -128,6 +133,10 @@ export async function runJobSpec(spec: JobSpec): Promise<StreamActionResult> {
return removeWrongFormatAudioAction(spec.slug, queueKey);
case "clean-audio-transcribed":
return cleanAudioAction(spec.slug, queueKey);
+ case "check-kept-deleted":
+ return checkKeptDeletedAction(spec.slug, queueKey);
+ case "persist-kept":
+ return persistKeptAction(spec.slug, queueKey);
default:
return { ok: false, error: `Cannot re-run job kind: ${spec.kind}` };
}
diff --git a/editor/app/layout.tsx b/editor/app/layout.tsx
@@ -59,6 +59,7 @@ const NAV_GROUPS: NavGroup[] = [
{ href: "/auto-queue", label: "Auto-queue" },
{ href: "/build", label: "Build" },
{ href: "/actionable", label: "Actionable" },
+ { href: "/saved-videos", label: "Saved videos" },
],
},
{
diff --git a/editor/app/saved-videos/backupActions.ts b/editor/app/saved-videos/backupActions.ts
@@ -0,0 +1,119 @@
+"use server";
+
+import { revalidatePath } from "next/cache";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { getSettings, writeSettings } from "yt-dlp-transcript-common/lib/settings";
+import { SYNC_INTERVAL_MAX_MINUTES } from "yt-dlp-transcript-common/lib/channelConfig";
+import {
+ backupSavedVideos,
+ verifySavedVideoBackup,
+} from "yt-dlp-transcript-common/controller/backupSavedVideos";
+import {
+ runManagedFunction,
+ type StreamActionResult,
+} from "yt-dlp-transcript-common/jobs/streamCommand";
+
+// Saved-video backups are a global (not per-channel) concern, so they share a
+// dedicated queue rather than a channel queue.
+const SAVED_VIDEOS_QUEUE = "saved-videos";
+
+// Mirror the saved-video store to the configured backup destination and write a
+// fresh manifest. The destination comes from settings (savedVideoBackup.dest);
+// an explicit `dest` overrides it for an ad-hoc run.
+export async function backupSavedVideosAction(
+ dest?: string,
+ queueKey?: string,
+): Promise<StreamActionResult> {
+ const paths = getPaths();
+ const settings = getSettings();
+ const target = (dest ?? settings.savedVideoBackup.dest).trim();
+ if (!target) {
+ return { ok: false, error: "No backup destination configured" };
+ }
+ return runManagedFunction({
+ kind: "backup-saved-videos",
+ queueKey: queueKey === undefined ? SAVED_VIDEOS_QUEUE : queueKey.trim(),
+ paths,
+ fn: async (onLog, signal) => {
+ const result = await backupSavedVideos({
+ paths,
+ dest: target,
+ onLog,
+ signal,
+ });
+ onLog(
+ `Saved-video backup: ${result.backedUp}/${result.entries} container(s), ${result.bytes} bytes.`,
+ );
+ revalidatePath("/saved-videos");
+ },
+ });
+}
+
+// Verify the backup destination against its manifest and report drift.
+export async function verifySavedVideoBackupAction(
+ dest?: string,
+ queueKey?: string,
+): Promise<StreamActionResult> {
+ const paths = getPaths();
+ const settings = getSettings();
+ const target = (dest ?? settings.savedVideoBackup.dest).trim();
+ if (!target) {
+ return { ok: false, error: "No backup destination configured" };
+ }
+ return runManagedFunction({
+ kind: "verify-saved-video-backup",
+ queueKey: queueKey === undefined ? SAVED_VIDEOS_QUEUE : queueKey.trim(),
+ paths,
+ fn: async (onLog, signal) => {
+ const drift = await verifySavedVideoBackup({
+ paths,
+ dest: target,
+ onLog,
+ signal,
+ });
+ onLog(
+ drift.ok
+ ? `Backup verified clean (checked ${drift.checked}).`
+ : `Backup drift: ${drift.missing.length} missing, ${drift.sizeMismatch.length} size-mismatch, ${drift.checksumMismatch.length} checksum-mismatch, ${drift.extra.length} extra.`,
+ );
+ },
+ });
+}
+
+export type SaveBackupConfigResult = { ok: true } | { ok: false; error: string };
+
+// Persist just the saved-video backup block (dest / enabled / interval) from the
+// Saved Videos page, leaving the rest of SiteSettings untouched. writeSettings
+// re-sanitizes (blank dest forces enabled off; interval clamped).
+export async function saveSavedVideoBackupAction(
+ _prev: SaveBackupConfigResult | undefined,
+ formData: FormData,
+): Promise<SaveBackupConfigResult> {
+ const dest = String(formData.get("backupDest") ?? "").trim();
+ const enabled = formData.get("backupEnabled") === "on";
+ const intervalRaw = String(formData.get("backupIntervalMinutes") ?? "").trim();
+ const intervalMinutes = Number.parseInt(intervalRaw, 10);
+ if (intervalRaw && (!Number.isFinite(intervalMinutes) || intervalMinutes < 1)) {
+ return {
+ ok: false,
+ error: `Backup interval must be 1–${SYNC_INTERVAL_MAX_MINUTES} minutes`,
+ };
+ }
+ const settings = getSettings();
+ try {
+ await writeSettings({
+ ...settings,
+ savedVideoBackup: {
+ enabled,
+ dest,
+ intervalMinutes: intervalRaw
+ ? intervalMinutes
+ : settings.savedVideoBackup.intervalMinutes,
+ },
+ });
+ } catch (e) {
+ return { ok: false, error: (e as Error).message };
+ }
+ revalidatePath("/saved-videos");
+ return { ok: true };
+}
diff --git a/editor/app/saved-videos/components/SavedVideosControls.tsx b/editor/app/saved-videos/components/SavedVideosControls.tsx
@@ -0,0 +1,134 @@
+"use client";
+
+import { useActionState, useState } from "react";
+import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog";
+import { cancelJobAction } from "../../jobs/actions";
+import {
+ backupSavedVideosAction,
+ saveSavedVideoBackupAction,
+ verifySavedVideoBackupAction,
+ type SaveBackupConfigResult,
+} from "../backupActions";
+
+type Props = {
+ enabled: boolean;
+ dest: string;
+ intervalMinutes: number;
+};
+
+// Backup config form + run/verify buttons for the Saved Videos page. The
+// destination drives all three: with no dest, scheduled and manual backups are
+// both disabled.
+export function SavedVideosControls({ enabled, dest, intervalMinutes }: Props) {
+ const [state, formAction] = useActionState<
+ SaveBackupConfigResult | undefined,
+ FormData
+ >(saveSavedVideoBackupAction, undefined);
+ // Track the dest input live so the run/verify buttons enable/disable without a
+ // round-trip. Seeded from the saved value.
+ const [destInput, setDestInput] = useState(dest);
+ const hasDest = destInput.trim() !== "";
+
+ return (
+ <div className="flex flex-col gap-4">
+ <form
+ action={formAction}
+ className="flex flex-col gap-3 rounded border border-zinc-200 dark:border-zinc-800 p-4 max-w-xl"
+ aria-label="backup config"
+ >
+ <h2 className="text-base font-semibold">Backup configuration</h2>
+ {state?.ok === false && (
+ <div
+ role="alert"
+ className="rounded border border-red-300 bg-red-50 dark:border-red-800 dark:bg-red-950 px-3 py-2 text-sm text-red-700 dark:text-red-300"
+ >
+ {state.error}
+ </div>
+ )}
+ {state?.ok === true && (
+ <div
+ aria-label="backup config saved"
+ className="rounded border border-green-300 bg-green-50 dark:border-green-800 dark:bg-green-950 px-3 py-2 text-sm text-green-700 dark:text-green-300"
+ >
+ Saved.
+ </div>
+ )}
+ <label className="flex flex-col gap-1 text-sm">
+ <span className="font-medium">Backup destination</span>
+ <input
+ type="text"
+ name="backupDest"
+ value={destInput}
+ onChange={(e) => setDestInput(e.target.value)}
+ placeholder="/mnt/backup/saved-videos"
+ aria-label="backup destination"
+ className="rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 px-2 py-1 text-sm font-mono"
+ />
+ <span className="text-xs text-zinc-500">
+ Local filesystem path (a mounted backup disk) the store is mirrored
+ into. The mirror is additive — it never deletes. Blank disables both
+ scheduled and manual backups.
+ </span>
+ </label>
+ <label className="flex items-start gap-2 text-sm">
+ <input
+ type="checkbox"
+ name="backupEnabled"
+ defaultChecked={enabled}
+ aria-label="scheduled backup enabled"
+ className="mt-1"
+ />
+ <span className="flex flex-col gap-0.5">
+ <span className="font-medium">Scheduled backup</span>
+ <span className="text-xs text-zinc-500">
+ When on (and a destination is set), the sync scheduler runs the
+ backup on the cadence below. Manual backups work regardless.
+ </span>
+ </span>
+ </label>
+ <label className="flex flex-col gap-1 text-sm">
+ <span className="font-medium">Backup interval (minutes)</span>
+ <input
+ type="number"
+ name="backupIntervalMinutes"
+ defaultValue={String(intervalMinutes)}
+ min={1}
+ aria-label="backup interval minutes"
+ className="rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 px-2 py-1 text-sm w-40"
+ />
+ </label>
+ <div>
+ <button
+ type="submit"
+ className="px-3 py-2 rounded-md bg-zinc-900 dark:bg-zinc-100 text-zinc-100 dark:text-zinc-900 text-sm font-medium hover:opacity-90"
+ >
+ Save backup config
+ </button>
+ </div>
+ </form>
+ <div className="flex flex-col gap-2 sm:flex-row sm:flex-wrap sm:gap-4">
+ <StreamActionLog
+ trigger={() => backupSavedVideosAction()}
+ cancelAction={cancelJobAction}
+ buttonLabel="Back up now"
+ runningLabel="Backing up…"
+ label="Back up saved videos"
+ disabled={!hasDest}
+ />
+ <StreamActionLog
+ trigger={() => verifySavedVideoBackupAction()}
+ cancelAction={cancelJobAction}
+ buttonLabel="Verify backup"
+ runningLabel="Verifying…"
+ label="Verify saved-video backup"
+ disabled={!hasDest}
+ />
+ </div>
+ {!hasDest && (
+ <p className="text-xs text-zinc-500">
+ Set a backup destination above and save to enable backups.
+ </p>
+ )}
+ </div>
+ );
+}
diff --git a/editor/app/saved-videos/page.tsx b/editor/app/saved-videos/page.tsx
@@ -0,0 +1,163 @@
+import type { Metadata } from "next";
+import Link from "next/link";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { getSettings } from "yt-dlp-transcript-common/lib/settings";
+import { formatBytes } from "yt-dlp-transcript-common/lib/format";
+import { listSavedVideos } from "yt-dlp-transcript-common/controller/savedVideoInventory";
+import { readSchedulerState } from "yt-dlp-transcript-common/jobs/syncSchedulerState";
+import { SavedVideosControls } from "./components/SavedVideosControls";
+
+export const dynamic = "force-dynamic";
+
+export const metadata: Metadata = { title: "Saved videos" };
+
+type ChannelSummary = {
+ slug: string;
+ count: number;
+ bytes: number;
+ verified: number;
+};
+
+export default async function SavedVideosPage() {
+ const paths = getPaths();
+ const settings = getSettings();
+ const backup = settings.savedVideoBackup;
+ const entries = await listSavedVideos({ paths });
+ const scheduler = await readSchedulerState(paths);
+
+ const byChannel = new Map<string, ChannelSummary>();
+ let totalBytes = 0;
+ let totalVerified = 0;
+ for (const e of entries) {
+ const row =
+ byChannel.get(e.slug) ??
+ { slug: e.slug, count: 0, bytes: 0, verified: 0 };
+ row.count += 1;
+ row.bytes += e.pointer.bytes;
+ if (e.pointer.sha256) {
+ row.verified += 1;
+ totalVerified += 1;
+ }
+ byChannel.set(e.slug, row);
+ totalBytes += e.pointer.bytes;
+ }
+ const channels = [...byChannel.values()].sort((a, b) =>
+ a.slug.localeCompare(b.slug),
+ );
+ const lastBackup = scheduler.lastSavedVideoBackupAt
+ ? new Date(scheduler.lastSavedVideoBackupAt).toLocaleString()
+ : null;
+
+ return (
+ <main className="flex flex-col gap-8 p-6 max-w-4xl">
+ <header className="flex flex-col gap-1">
+ <h1 className="text-xl font-semibold">Saved videos</h1>
+ <p className="text-sm text-zinc-500">
+ Source video containers persisted by the keep-latest retention rule
+ live in a separate store, leaving the main data volume holding only
+ audio + transcripts. Configure a channel's keep-latest window and
+ per-channel store dir under that channel's settings.
+ </p>
+ </header>
+
+ <section
+ aria-label="store summary"
+ className="grid grid-cols-2 sm:grid-cols-4 gap-3"
+ >
+ <SummaryCard label="Saved videos" value={String(entries.length)} />
+ <SummaryCard label="Total size" value={formatBytes(totalBytes)} />
+ <SummaryCard label="Channels" value={String(channels.length)} />
+ <SummaryCard
+ label="With checksum"
+ value={`${totalVerified}/${entries.length}`}
+ />
+ </section>
+
+ <section aria-label="store location" className="flex flex-col gap-1 text-sm">
+ <div>
+ <span className="text-zinc-500">Default store dir: </span>
+ <span className="font-mono">{paths.savedVideosDir}</span>
+ </div>
+ <div>
+ <span className="text-zinc-500">Last backup: </span>
+ {lastBackup ? (
+ <span>{lastBackup}</span>
+ ) : (
+ <span className="text-zinc-500">never</span>
+ )}
+ {backup.enabled && backup.dest && (
+ <span className="text-zinc-500">
+ {" "}
+ · scheduled every {backup.intervalMinutes} min
+ </span>
+ )}
+ </div>
+ </section>
+
+ <section aria-label="per-channel saved videos" className="flex flex-col gap-2">
+ <h2 className="text-base font-semibold">By channel</h2>
+ {channels.length === 0 ? (
+ <p className="text-sm text-zinc-600 dark:text-zinc-400">
+ No saved videos yet. Set a channel's keep-latest window, then run
+ a sync (or <em>Persist kept now</em> from its Cleanup stage).
+ </p>
+ ) : (
+ <table className="text-sm border-collapse">
+ <thead>
+ <tr className="text-left text-zinc-500">
+ <th className="py-1 pr-6 font-medium">Channel</th>
+ <th className="py-1 pr-6 font-medium tabular-nums">Saved</th>
+ <th className="py-1 pr-6 font-medium tabular-nums">Size</th>
+ <th className="py-1 font-medium tabular-nums">Checksummed</th>
+ </tr>
+ </thead>
+ <tbody>
+ {channels.map((c) => (
+ <tr
+ key={c.slug}
+ className="border-t border-zinc-200 dark:border-zinc-800"
+ >
+ <td className="py-1 pr-6">
+ <Link
+ href={`/channels/${c.slug}`}
+ className="underline hover:no-underline"
+ >
+ {c.slug}
+ </Link>
+ </td>
+ <td className="py-1 pr-6 tabular-nums">{c.count}</td>
+ <td className="py-1 pr-6 tabular-nums">
+ {formatBytes(c.bytes)}
+ </td>
+ <td className="py-1 tabular-nums">
+ {c.verified}/{c.count}
+ </td>
+ </tr>
+ ))}
+ </tbody>
+ </table>
+ )}
+ </section>
+
+ <section aria-label="backups" className="flex flex-col gap-2">
+ <h2 className="text-base font-semibold">Backups</h2>
+ <SavedVideosControls
+ enabled={backup.enabled}
+ dest={backup.dest}
+ intervalMinutes={backup.intervalMinutes}
+ />
+ </section>
+ </main>
+ );
+}
+
+function SummaryCard({ label, value }: { label: string; value: string }) {
+ return (
+ <div className="rounded border border-zinc-200 dark:border-zinc-800 p-3">
+ <div className="text-xs uppercase tracking-wide text-zinc-500">
+ {label}
+ </div>
+ <div className="text-lg font-semibold tabular-nums mt-0.5">{value}</div>
+ </div>
+ );
+}
diff --git a/editor/app/scheduler/runTick.ts b/editor/app/scheduler/runTick.ts
@@ -19,6 +19,8 @@ import {
type SchedulerState,
} from "yt-dlp-transcript-common/jobs/syncSchedulerState";
import { syncAction } from "../channels/[slug]/pipelineActions";
+import { checkKeptDeletedAction } from "../channels/[slug]/whisperActions";
+import { backupSavedVideosAction } from "../saved-videos/backupActions";
export type SchedulerTickResult = {
ok: boolean;
@@ -30,6 +32,10 @@ export type SchedulerTickResult = {
running: number;
// Channels for which a sync was queued this tick.
queued: string[];
+ // Channels for which a keep-latest deletion check was queued this tick.
+ keptChecksQueued: string[];
+ // Whether a saved-video backup was queued this tick.
+ savedVideoBackupQueued: boolean;
// Channels deliberately held back, with a reason.
skipped: SchedulerSkip[];
};
@@ -58,6 +64,8 @@ export async function runSchedulerTick(): Promise<SchedulerTickResult> {
reason: "tick already running",
running: 0,
queued: [],
+ keptChecksQueued: [],
+ savedVideoBackupQueued: false,
skipped: [],
};
}
@@ -85,6 +93,8 @@ export async function runSchedulerTick(): Promise<SchedulerTickResult> {
reason: "scheduler disabled",
running,
queued: [],
+ keptChecksQueued: [],
+ savedVideoBackupQueued: false,
skipped: [],
};
}
@@ -131,6 +141,54 @@ export async function runSchedulerTick(): Promise<SchedulerTickResult> {
scheduler.quietHoursStart,
scheduler.quietHoursEnd,
);
+
+ // Keep-latest deletion checks run on their own cadence, on the per-channel
+ // local queue (channel:<slug>) — separate from the platform download queue a
+ // sync uses, so the two don't serialize against each other. Suppressed during
+ // quiet hours, capped per tick like syncs, and skipped for any channel we
+ // just queued a sync for (avoid hitting the source twice in one tick).
+ const keptChecksQueued: string[] = [];
+ if (!quiet) {
+ const queuedThisTick = new Set(queued);
+ const intervalMs = scheduler.keepLatestCheckIntervalMinutes * 60_000;
+ const dueForKeptCheck = channels.filter((c) => {
+ if (!c.config.keepLatest || c.config.keepLatest <= 0) return false;
+ if (queuedThisTick.has(c.slug)) return false;
+ const last = channelState(state, c.slug).lastKeptCheckAt ?? 0;
+ return now - last >= intervalMs;
+ });
+ for (const c of dueForKeptCheck.slice(0, scheduler.maxConcurrentSyncs)) {
+ const result = await checkKeptDeletedAction(c.slug);
+ if (!result.ok) {
+ skipped.push({ slug: c.slug, reason: `kept-check: ${result.error}` });
+ continue;
+ }
+ channelState(state, c.slug).lastKeptCheckAt = now;
+ keptChecksQueued.push(c.slug);
+ void result.stream.cancel();
+ }
+ }
+
+ // Saved-video backup: a single global job on its own cadence. Suppressed
+ // during quiet hours; runs at most once per intervalMinutes. Independent of
+ // the per-channel concurrency cap (it touches no source provider).
+ let savedVideoBackupQueued = false;
+ const backup = settings.savedVideoBackup;
+ if (!quiet && backup.enabled && backup.dest.trim()) {
+ const intervalMs = backup.intervalMinutes * 60_000;
+ const last = state.lastSavedVideoBackupAt ?? 0;
+ if (now - last >= intervalMs) {
+ const result = await backupSavedVideosAction();
+ if (result.ok) {
+ state.lastSavedVideoBackupAt = now;
+ savedVideoBackupQueued = true;
+ void result.stream.cancel();
+ } else {
+ skipped.push({ slug: "(saved-video backup)", reason: result.error });
+ }
+ }
+ }
+
recordRun(state, { at: now, queued, skipped });
await writeSchedulerState(paths, state);
@@ -140,6 +198,8 @@ export async function runSchedulerTick(): Promise<SchedulerTickResult> {
reason: quiet ? "quiet hours" : undefined,
running,
queued,
+ keptChecksQueued,
+ savedVideoBackupQueued,
skipped,
};
} finally {
diff --git a/editor/app/settings/actions.ts b/editor/app/settings/actions.ts
@@ -144,6 +144,9 @@ export async function saveSettingsAction(
backoffBaseMinutes: intOrNaN("syncSchedulerBackoffBaseMinutes"),
backoffMaxMinutes: intOrNaN("syncSchedulerBackoffMaxMinutes"),
heartbeatSeconds: intOrNaN("syncSchedulerHeartbeatSeconds"),
+ keepLatestCheckIntervalMinutes: intOrNaN(
+ "syncSchedulerKeepLatestCheckIntervalMinutes",
+ ),
};
let socialInput: unknown;
@@ -195,6 +198,9 @@ export async function saveSettingsAction(
// re-sanitizes it regardless.
autoQueue: getSettings().autoQueue,
socialLinks,
+ // Preserve the saved-video backup config on an unrelated settings save (the
+ // Saved Videos page edits it). writeSettings re-sanitizes it regardless.
+ savedVideoBackup: getSettings().savedVideoBackup,
};
try {
await writeSettings(next);
diff --git a/editor/e2e/saved-videos.spec.ts b/editor/e2e/saved-videos.spec.ts
@@ -0,0 +1,110 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { test, expect } from "@playwright/test";
+import { readJson, resetData, resolvePath } from "./helpers";
+import { baseUrl } from "./baseUrl";
+
+// The transcribe fixture's channel slug.
+const SLUG = "test-transcribe";
+
+// Persist a video's source container into the saved-video store: write the
+// container under test-transcripts/saved-videos/<slug>/<id>/ and a pointer
+// sidecar back in the data dir. Mirrors persistSourceVideo's on-disk result.
+async function seedSavedVideo(
+ videoId: string,
+ bytes: number,
+ sha256?: string,
+): Promise<void> {
+ const storeDir = resolvePath(
+ `test-transcripts/saved-videos/${SLUG}/${videoId}`,
+ );
+ await mkdir(storeDir, { recursive: true });
+ await writeFile(`${storeDir}/source-media.mp4`, "x".repeat(bytes));
+ const pointer = {
+ storedAt: "2026-06-01T00:00:00.000Z",
+ dir: storeDir,
+ file: "source-media.mp4",
+ bytes,
+ keepReason: "keep-latest",
+ ...(sha256 ? { sha256 } : {}),
+ };
+ await writeFile(
+ resolvePath(`test-transcripts/channels/${SLUG}/data/${videoId}/saved-video.json`),
+ JSON.stringify(pointer, null, 2) + "\n",
+ );
+ await fetch(`${baseUrl}/api/test/invalidate-cache`).catch(() => {});
+}
+
+test("Saved Videos page lists persisted source videos per channel", async ({
+ page,
+}) => {
+ await resetData("one-transcribe-channel-with-audio");
+ await seedSavedVideo("vidA", 2048, "a".repeat(64));
+
+ await page.goto("/saved-videos");
+ await expect(
+ page.getByRole("heading", { name: "Saved videos", level: 1 }),
+ ).toBeVisible();
+
+ // Summary cards: one saved video, one channel, one with checksum.
+ const summary = page.getByLabel("store summary");
+ await expect(summary).toContainText("Saved videos");
+
+ // Per-channel table row links to the channel and reports the saved count.
+ const byChannel = page.getByLabel("per-channel saved videos");
+ await expect(byChannel.getByRole("link", { name: SLUG })).toBeVisible();
+ await expect(byChannel).toContainText("1/1"); // checksummed
+});
+
+test("Saved Videos page saves the backup configuration", async ({ page }) => {
+ await resetData("one-transcribe-channel-with-audio");
+
+ await page.goto("/saved-videos");
+ const dest = page.getByLabel("backup destination");
+ await dest.fill("/tmp/ttb-e2e-backup-dest");
+ await page.getByLabel("scheduled backup enabled").check();
+ await page.getByRole("button", { name: "Save backup config" }).click();
+ await expect(page.getByLabel("backup config saved")).toBeVisible();
+
+ // The setting persisted to test-settings.json.
+ const settings = await readJson<{
+ savedVideoBackup?: { dest?: string; enabled?: boolean };
+ }>("test-settings.json");
+ expect(settings.savedVideoBackup?.dest).toBe("/tmp/ttb-e2e-backup-dest");
+ expect(settings.savedVideoBackup?.enabled).toBe(true);
+
+ // The run/verify buttons become enabled once a dest is configured.
+ await expect(
+ page.getByRole("button", { name: "Back up now" }),
+ ).toBeEnabled();
+});
+
+test("channel Cleanup stage shows the retention & persistence section", async ({
+ page,
+}) => {
+ await resetData("one-transcribe-channel-with-audio");
+ await seedSavedVideo("vidA", 1024);
+
+ await page.goto(`/channels/${SLUG}`);
+ await page.getByRole("button", { name: "Cleanup stage summary" }).click();
+ const section = page.getByLabel("retention and persistence section");
+ await expect(section).toBeVisible();
+ await expect(section).toContainText("Saved source videos");
+ await expect(
+ section.getByRole("button", { name: "Check kept videos" }),
+ ).toBeVisible();
+ await expect(
+ section.getByRole("button", { name: "Persist kept now" }),
+ ).toBeVisible();
+});
+
+test("video page shows persisted source status and an unpersist control", async ({
+ page,
+}) => {
+ await resetData("one-transcribe-channel-with-audio");
+ await seedSavedVideo("vidA", 4096);
+
+ await page.goto(`/channels/${SLUG}/videos/vidA`);
+ await expect(
+ page.getByLabel("unpersist source video vidA"),
+ ).toBeVisible();
+});
