commit d4ed8093566f61e5d4e65e69cdde4ef402aa49d2
parent 78f836fda3c7175864e7c1ab7376984f010ac310
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 25 Jun 2026 00:33:47 -0400
Phase 2: per-download persistence rule engine + app-side extraction
Each individual download now consults the channel's keep-latest rule (plus
per-run overrides) to decide whether to keep the source video and who extracts
the audio — the per-channel-rule model the user asked for, not a separate runner.
Decision (common/ytdlp/persistencePlan.ts, pure + unit-tested):
- persist when the video is inside the keep-latest window OR carries a
do-not-clean pin OR a per-run keep override; precedence: keepSourceVideoOverride
(true/false) > extractImmediately > pin > window > audio-only.
- persisting implies app-side extraction (need the container in hand).
Keep-window cutoff (common/controller/keptVideos.ts): computeKeepWindow +
isInKeepWindow classify a video by its own upload date against the channel's
Nth-newest cutoff, computed once per run. This is what lets the NEWEST videos —
not yet on disk at download time — be persisted (a plain dir-listing can't).
App-side extraction (common/ytdlp/downloadOneManaged.ts):
- transcribe handling in "app" mode downloads the source container (no -x):
bestvideo*+bestaudio/best when persisting, bestaudio/worst otherwise, written
as source-media.<ext> (distinct from audio.<ext> so cleanup/transcription never
treat it as audio). finalizeAppExtraction then runs transcodeAudio to produce
audio.<fmt> and keeps or discards the container per the plan.
- "ytdlp" mode (default) produces byte-identical args to before — no behavior
change for existing channels. The youtube no-subs fallback honors the plan too.
- audio-check downloads log that source-video persistence is skipped (out of
scope this phase; the saved-video store in phase 3 addresses it).
Per-run overrides threaded runPipelineAction -> runYtdlp -> downloadOneManaged:
keepSourceVideoOverride, extractImmediately (the disk-saving backfill case),
audioFormatOverride. Exposed on downloadMissingAction (+ replay spec/dispatcher).
Transcribe-from-container (common/controller/transcribeOne.ts): resolveAudioFile
falls back to a source-media container when no extracted audio exists, so a
kept-but-cleaned video or an archive-only download is still transcribable
(parakeet ffmpeg-slices any container).
New per-video redownloadToArchiveAction (editor .../videos/[id]/videoActions.ts):
re-fetch an existing video purely to grab + keep its source container without
disturbing the transcript (forces keepSourceVideoOverride). Job label added.
Data model: ChannelConfig.extractionMode ("ytdlp" | "app"); videoStatus gains
isVideoContainer / VIDEO_CONTAINER_EXTS / source-media helpers; outputArgsForUrl
takes an optional mediaName.
Known follow-ups (phase 3): persisted containers stay in the data dir and aren't
auto-pruned when they roll out of the window; audio-check + persistence.
Tests: persistencePlan (7) + keptVideos keep-window (3) added; all green.
Typecheck clean for common and editor.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Diffstat:
14 files changed, 705 insertions(+), 47 deletions(-)
diff --git a/common/controller/keptVideos.test.ts b/common/controller/keptVideos.test.ts
@@ -4,7 +4,11 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import type { Paths } from "../lib/paths";
-import { computeKeptVideoIds } from "./keptVideos";
+import {
+ computeKeepWindow,
+ computeKeptVideoIds,
+ isInKeepWindow,
+} from "./keptVideos";
// Run with:
// pnpm --filter yt-dlp-transcript-common exec tsx --test common/controller/keptVideos.test.ts
@@ -111,3 +115,54 @@ test("missing channel data dir yields an empty set", async () => {
);
});
});
+
+test("keep window: an under-full channel admits any candidate", async () => {
+ await withPaths(async (paths) => {
+ await seedChannel(paths.channelsDir, "ch", [["a", "20240101"]]);
+ const w = await computeKeepWindow({
+ paths,
+ channelSlug: "ch",
+ keepLatest: 3,
+ });
+ assert.equal(w.full, false);
+ assert.equal(w.cutoffKey, null);
+ // A brand-new (not-yet-on-disk) video is admitted regardless of its date.
+ assert.equal(isInKeepWindow("20200101", w), true);
+ });
+});
+
+test("keep window: a full channel admits only candidates >= the cutoff", async () => {
+ await withPaths(async (paths) => {
+ await seedChannel(paths.channelsDir, "ch", [
+ ["a", "20240101"],
+ ["b", "20240301"],
+ ["c", "20240201"],
+ ]);
+ const w = await computeKeepWindow({
+ paths,
+ channelSlug: "ch",
+ keepLatest: 2,
+ });
+ assert.equal(w.full, true);
+ // Newest two are 20240301, 20240201 -> the 2nd-newest (cutoff) is 20240201.
+ assert.equal(w.cutoffKey, "20240201");
+ // A newer upload displaces the boundary video -> kept.
+ assert.equal(isInKeepWindow("20240401", w), true);
+ // A tie with the cutoff is kept.
+ assert.equal(isInKeepWindow("20240201", w), true);
+ // An older upload stays out of the window.
+ assert.equal(isInKeepWindow("20240101", w), false);
+ });
+});
+
+test("keep window: keepLatest <= 0 is inert", async () => {
+ await withPaths(async (paths) => {
+ await seedChannel(paths.channelsDir, "ch", [["a", "20240101"]]);
+ const w = await computeKeepWindow({
+ paths,
+ channelSlug: "ch",
+ keepLatest: 0,
+ });
+ assert.equal(isInKeepWindow("99999999", w), false);
+ });
+});
diff --git a/common/controller/keptVideos.ts b/common/controller/keptVideos.ts
@@ -33,25 +33,34 @@ export async function listChannelVideoIds(
}
}
+// Recency sort key for a video given its already-known upload_date: the
+// YYYYMMDD string, falling back to a YYYYMMDD_ prefix on the id, else "" (sorts
+// oldest). Shared so the live cutoff test for a not-yet-downloaded video keys it
+// the same way the on-disk window does.
+export function uploadKeyFor(
+ uploadDate: string | undefined,
+ id: string,
+): string {
+ if (uploadDate) return uploadDate;
+ return id.match(/^(\d{8})(?:_|$)/)?.[1] ?? "";
+}
+
// Recency sort key for a video: upload_date (YYYYMMDD) from metadata.info.json,
// falling back to a YYYYMMDD_ prefix on the dir name, else "" (sorts oldest).
async function uploadKey(videoDir: string, id: string): Promise<string> {
const meta = await loadRawMetadataFromDir(videoDir);
- if (meta?.upload_date) return meta.upload_date;
- return id.match(/^(\d{8})(?:_|$)/)?.[1] ?? "";
+ return uploadKeyFor(meta?.upload_date, id);
}
-// The set of the newest `keepLatest` video ids for a channel (by upload date,
-// newest first; ties broken by id descending for determinism). Empty when
-// keepLatest <= 0 or the channel has no data dir.
-export async function computeKeptVideoIds({
- paths,
- channelSlug,
- keepLatest,
-}: ComputeKeptOptions): Promise<Set<string>> {
- if (!Number.isFinite(keepLatest) || keepLatest <= 0) return new Set();
+// Sort a channel's data-dir videos newest-first by upload key (ties broken by id
+// descending for determinism). Shared by computeKeptVideoIds (the on-disk window)
+// and computeKeepWindow (the live cutoff used at download time).
+async function keyedVideosNewestFirst(
+ paths: Paths,
+ channelSlug: string,
+): Promise<Array<{ id: string; key: string }>> {
const ids = await listChannelVideoIds(paths, channelSlug);
- if (ids.length === 0) return new Set();
+ if (ids.length === 0) return [];
const dataDir = path.join(paths.channelsDir, channelSlug, "data");
const keyed = await Promise.all(
ids.map(async (id) => ({
@@ -62,5 +71,59 @@ export async function computeKeptVideoIds({
keyed.sort((a, b) =>
a.key === b.key ? b.id.localeCompare(a.id) : b.key.localeCompare(a.key),
);
+ return keyed;
+}
+
+// The set of the newest `keepLatest` video ids for a channel (by upload date,
+// newest first; ties broken by id descending for determinism). Empty when
+// keepLatest <= 0 or the channel has no data dir. Used by cleanup/snapshot,
+// which see every kept video already on disk.
+export async function computeKeptVideoIds({
+ paths,
+ channelSlug,
+ keepLatest,
+}: ComputeKeptOptions): Promise<Set<string>> {
+ if (!Number.isFinite(keepLatest) || keepLatest <= 0) return new Set();
+ const keyed = await keyedVideosNewestFirst(paths, channelSlug);
return new Set(keyed.slice(0, Math.floor(keepLatest)).map((k) => k.id));
}
+
+// The keep-latest window expressed as a CUTOFF rather than a fixed id set. This
+// is what the per-download rule needs: at download time the newest videos aren't
+// on disk yet, so membership can't come from listing dirs. Instead we capture
+// - cutoffKey: the upload key of the Nth-newest video currently on disk
+// - full: whether the channel already holds >= keepLatest videos
+// and test a candidate by its own upload key (see isInKeepWindow). A candidate
+// newer than the current Nth would displace it, so it belongs in the window.
+export type KeepWindow = {
+ keepLatest: number;
+ cutoffKey: string | null;
+ full: boolean;
+};
+
+export async function computeKeepWindow({
+ paths,
+ channelSlug,
+ keepLatest,
+}: ComputeKeptOptions): Promise<KeepWindow> {
+ const n = Number.isFinite(keepLatest) ? Math.floor(keepLatest) : 0;
+ if (n <= 0) return { keepLatest: 0, cutoffKey: null, full: false };
+ const keyed = await keyedVideosNewestFirst(paths, channelSlug);
+ const full = keyed.length >= n;
+ const cutoffKey = full ? keyed[n - 1].key : null;
+ return { keepLatest: n, cutoffKey, full };
+}
+
+// Whether a candidate video (identified by its own upload key) belongs in the
+// keep-latest window. With fewer than N videos on disk the window isn't full, so
+// any candidate is kept; once full, a candidate is kept iff it's at least as new
+// as the current Nth-newest (a tie or newer displaces the boundary video).
+export function isInKeepWindow(
+ uploadKey: string,
+ window: KeepWindow | undefined,
+): boolean {
+ if (!window || window.keepLatest <= 0) return false;
+ if (!window.full) return true;
+ if (window.cutoffKey == null) return true;
+ return uploadKey >= window.cutoffKey;
+}
diff --git a/common/controller/transcribeOne.ts b/common/controller/transcribeOne.ts
@@ -10,7 +10,7 @@ import type { TaskTracker } from "../jobs/taskHooks";
import { normalizeTranscript } from "./normalizeTranscript";
import { pingRemoteHealth, transcribeViaRemote } from "./remoteTranscribe";
import { TranscribeError } from "./transcribeError";
-import { isRealAudioFile } from "../lib/videoStatus";
+import { findSourceMedia, isRealAudioFile } from "../lib/videoStatus";
import { writeTranscribeOutcome } from "../lib/transcribeOutcome-server";
const { pathExists, readdir, rename, writeFile } = fs;
@@ -26,11 +26,19 @@ async function resolveAudioFile(
if (strict) return null;
const entries = await readdir(videoDir);
const candidates = entries.filter(isRealAudioFile);
- if (candidates.length === 0) return null;
- for (const preferred of AUDIO_PREFERENCE) {
- if (candidates.includes(preferred)) return preferred;
+ if (candidates.length > 0) {
+ for (const preferred of AUDIO_PREFERENCE) {
+ if (candidates.includes(preferred)) return preferred;
+ }
+ return [...candidates].sort()[0];
}
- return [...candidates].sort()[0];
+ // No extracted audio on disk — fall back to a persisted source video
+ // container. Transcribers that ffmpeg-slice their input (parakeet) read a
+ // container directly; this is what makes a kept-but-cleaned video, or a
+ // redownload-to-archive that only fetched the container, still transcribable.
+ const container = findSourceMedia(entries);
+ if (container) return container;
+ return null;
}
export type TranscribeOneOptions = {
diff --git a/common/lib/channelConfig.ts b/common/lib/channelConfig.ts
@@ -4,6 +4,20 @@ export type ChannelHandling = "youtube" | "transcribe";
export type AudioFormat = "m4a" | "mp3" | "opus";
+// Who owns audio extraction for a transcribe-handling download:
+// "ytdlp" (default) -> yt-dlp's own `-x --audio-format` postprocessor, exactly
+// as before. No source container is kept.
+// "app" -> yt-dlp downloads the source container (no `-x`) and the
+// app runs ffmpeg (transcodeAudio) to produce audio.<fmt>.
+// Required whenever the source video must be kept/inspected
+// (the keep-latest persistence rule forces this).
+export type ExtractionMode = "ytdlp" | "app";
+
+export const EXTRACTION_MODE_VALUES: ReadonlyArray<ExtractionMode> = [
+ "ytdlp",
+ "app",
+];
+
export type AudioCheckConfig = {
enabled: boolean;
intervalSeconds?: number;
@@ -33,6 +47,11 @@ export type ChannelConfig = {
// A kept video later found deleted-from-source is pinned permanently via the
// do-not-clean marker so it survives even after it rolls out of the window.
keepLatest?: number;
+ // Audio-extraction strategy for transcribe-handling downloads (see
+ // ExtractionMode above). Omitted/unknown -> "ytdlp" (legacy behavior). The
+ // keep-latest persistence rule forces "app" for the videos it persists,
+ // regardless of this setting.
+ extractionMode?: ExtractionMode;
ytdlpExtraArgs?: string[];
subLangs?: string;
lastSyncedAt?: string;
@@ -136,6 +155,9 @@ export function parseChannelConfig(raw: unknown): ChannelConfig | null {
config.keepLatest =
r.keepLatest === 0 ? 0 : clampInt(r.keepLatest, 1, KEEP_LATEST_MAX);
}
+ if (r.extractionMode === "ytdlp" || r.extractionMode === "app") {
+ config.extractionMode = r.extractionMode;
+ }
if (
Array.isArray(r.ytdlpExtraArgs) &&
r.ytdlpExtraArgs.every((x) => typeof x === "string")
diff --git a/common/lib/videoStatus.ts b/common/lib/videoStatus.ts
@@ -69,6 +69,48 @@ export function audioFilesToRemove(
return files;
}
+// Video container extensions the app may keep as a persisted source video and
+// transcribe directly from (parakeet's stitcher ffmpeg-slices any container).
+// Consolidated here so VideoPanel and the transcribe fallback share one list.
+export const VIDEO_CONTAINER_EXTS: ReadonlyArray<string> = [
+ "mp4",
+ "webm",
+ "mkv",
+ "mov",
+ "m4v",
+ "ogv",
+ "avi",
+];
+
+export function isVideoContainer(name: string): boolean {
+ const dot = name.lastIndexOf(".");
+ if (dot < 0) return false;
+ return VIDEO_CONTAINER_EXTS.includes(name.slice(dot + 1).toLowerCase());
+}
+
+// A persisted source video container is written under data/<id>/source-media.<ext>
+// (a deliberately distinct base name from audio.<ext> so it's never mistaken for
+// an extractable/cleanable audio file by isRealAudioFile). The keep-latest
+// persistence rule downloads the full video here and the app extracts audio from
+// it; the container is then kept (Phase 3 moves it to the saved-video store).
+export const SOURCE_MEDIA_BASENAME = "source-media";
+
+export function isSourceMediaFile(name: string): boolean {
+ if (!name.startsWith(`${SOURCE_MEDIA_BASENAME}.`)) return false;
+ if (name.includes(".tmp-")) return false;
+ if (name.endsWith(".info.json")) return false;
+ if (name.endsWith(".part")) return false;
+ if (name.endsWith(".part.good")) return false;
+ if (name.endsWith(".part.testing")) return false;
+ return true;
+}
+
+// The persisted source container in a video dir, if any (the keep-latest source
+// media). Null when no finalized source-media.<ext> exists.
+export function findSourceMedia(entries: string[]): string | null {
+ return entries.find(isSourceMediaFile) ?? null;
+}
+
// A genuine resumable partial: an interrupted audio download, NOT a live-chat
// sidecar that merely ends in .part.
export function isPartAudioFile(name: string): boolean {
diff --git a/common/ytdlp/downloadOneManaged.ts b/common/ytdlp/downloadOneManaged.ts
@@ -1,5 +1,5 @@
import path from "node:path";
-import { appendFile, mkdir, readdir, readFile } from "node:fs/promises";
+import { appendFile, mkdir, readdir, readFile, rm } from "node:fs/promises";
import { createWriteStream, type WriteStream } from "node:fs";
import { execa } from "execa";
import {
@@ -12,6 +12,18 @@ import {
type ChannelConfig,
} from "../lib/channelConfig";
import {
+ resolvePersistenceDecision,
+ type PersistenceDecision,
+} from "./persistencePlan";
+import {
+ isInKeepWindow,
+ uploadKeyFor,
+ type KeepWindow,
+} from "../controller/keptVideos";
+import { isDoNotClean } from "../lib/doNotClean-server";
+import { transcodeAudio } from "../controller/transcode";
+import { findSourceMedia } from "../lib/videoStatus";
+import {
type AudioCheckAttemptStats,
type DownloadAttempt,
type DownloadOutcomeRecord,
@@ -69,6 +81,18 @@ export type ManagedDownloadOpts = {
// download filters. Plumbed from the playlist-level caller alongside the
// other resolved settings.
globalSkipLiveDownloads?: boolean;
+ // ---- Per-download persistence rule (Phase 2) ----
+ // The channel's keep-latest window as a cutoff, computed once per run by the
+ // caller (computeKeepWindow). This video's membership is decided against its
+ // own upload date — newer-than-the-cutoff videos persist their source.
+ keepWindow?: KeepWindow;
+ // Per-run override: force-keep (true) / force-discard (false) the source video.
+ keepSourceVideoOverride?: boolean;
+ // Per-run override: extract audio now and discard the container even for a
+ // video the keep-latest rule would otherwise persist (the save-disk backfill).
+ extractImmediately?: boolean;
+ // Per-run override of channelConfig.audioFormat for the extracted audio.
+ audioFormatOverride?: AudioFormat;
};
// When `reuseInfoJson` is true, the real download reuses the metadata the
@@ -90,22 +114,89 @@ function youtubeHandlingArgs(
return args;
}
-function transcribeHandlingArgs(
+// Format/extract args for a transcribe-handling download under a resolved
+// persistence plan. "ytdlp" mode is the legacy path: yt-dlp extracts the audio
+// itself (-x) and optionally keeps its bestaudio source via -k. "app" mode omits
+// -x so yt-dlp leaves the source container for the app to extract from
+// (finalizeAppExtraction) — pulling a full video when persisting, bestaudio
+// otherwise.
+function audioFormatSelectionArgs(
+ plan: PersistenceDecision,
+ fmt: AudioFormat,
config: ChannelConfig,
- reuseInfoJson = false,
): string[] {
- const fmt: AudioFormat = config.audioFormat ?? "mp3";
- const args: string[] = [];
- if (!reuseInfoJson) args.push("--write-info-json");
- args.push("-f", "bestaudio/worst", "-x", "--audio-format", fmt);
- if (config.keepSourceVideo) args.push("-k");
- return args;
+ if (plan.extractionMode === "ytdlp") {
+ const args = ["-f", "bestaudio/worst", "-x", "--audio-format", fmt];
+ if (config.keepSourceVideo) args.push("-k");
+ return args;
+ }
+ return plan.persist
+ ? ["-f", "bestvideo*+bestaudio/best"]
+ : ["-f", "bestaudio/worst"];
}
-function handlingArgs(config: ChannelConfig, reuseInfoJson: boolean): string[] {
- return config.handling === "youtube"
- ? youtubeHandlingArgs(config, reuseInfoJson)
- : transcribeHandlingArgs(config, reuseInfoJson);
+// The media output + info-json + format args for a transcribe-handling download.
+// In app mode the main output is source-media.<ext> (distinct from audio.<ext>
+// so it's never treated as cleanable audio); otherwise the historical audio.<ext>.
+// reuseInfoJson drops --write-info-json (the prefetch already wrote it).
+function transcribeMediaArgs(
+ url: string,
+ config: ChannelConfig,
+ plan: PersistenceDecision,
+ fmt: AudioFormat,
+ reuseInfoJson: boolean,
+): string[] {
+ const mediaName = plan.extractionMode === "app" ? "source-media" : "audio";
+ return [
+ ...outputArgsForUrl(url, { mediaName }),
+ ...(reuseInfoJson ? [] : ["--write-info-json"]),
+ ...audioFormatSelectionArgs(plan, fmt, config),
+ ];
+}
+
+// After an app-mode transcribe download, produce audio.<fmt> from the downloaded
+// source-media.<ext> container via ffmpeg. When the video is not persisted the
+// container is then removed (the extract-now / save-disk path). Best-effort: a
+// failed extraction leaves the container in place (so a later pass or the
+// transcribe-from-container fallback can still recover) and is reported rather
+// than failing the whole download.
+async function finalizeAppExtraction(opts: {
+ paths: Paths;
+ videoDir: string;
+ fmt: AudioFormat;
+ persist: boolean;
+ onLog: (s: string) => void;
+ signal: AbortSignal;
+}): Promise<void> {
+ const entries = await readdir(opts.videoDir).catch(() => [] as string[]);
+ const source = findSourceMedia(entries);
+ if (!source) {
+ opts.onLog(
+ `App extraction: no source-media container in ${opts.videoDir}; skipping.\n`,
+ );
+ return;
+ }
+ try {
+ await transcodeAudio({
+ paths: opts.paths,
+ videoDir: opts.videoDir,
+ sourceFilename: source,
+ targetFormat: opts.fmt,
+ onLog: opts.onLog,
+ signal: opts.signal,
+ });
+ } catch (err) {
+ opts.onLog(
+ `App extraction failed (${source} -> audio.${opts.fmt}): ${(err as Error).message}. Keeping the source container.\n`,
+ );
+ return;
+ }
+ if (opts.persist) {
+ opts.onLog(`Kept source container ${source} (persisted).\n`);
+ } else {
+ await rm(path.join(opts.videoDir, source), { force: true });
+ opts.onLog(`Discarded source container ${source} (audio-only).\n`);
+ }
}
// The source a download attempt reads from: either the prefetched info json
@@ -335,6 +426,9 @@ async function runManagedDownload(
// yt-dlp's pinned (expirable) format URLs — so it re-extracts even though the
// prefetch ran.
let infoJsonPath: string | null = null;
+ // This video's recency key (upload_date), captured from the prefetched
+ // metadata so the keep-latest cutoff can classify it even before it's on disk.
+ let videoUploadKey = canonicalId ? uploadKeyFor(undefined, canonicalId) : "";
if (canonicalId && !opts.signal.aborted) {
const videoDir = path.join(channelDir, "data", canonicalId);
const prefetchArgs = [
@@ -372,6 +466,7 @@ async function runManagedDownload(
// the metadata file; a failed prefetch falls through to the legacy path so
// the existing auth-retry logic still gets a chance.
if (metadata) infoJsonPath = metaPath;
+ videoUploadKey = uploadKeyFor(metadata?.upload_date, canonicalId);
const decision = evaluateDownloadFilters({
metadata,
@@ -406,11 +501,59 @@ async function runManagedDownload(
// Audio-check re-extracts fresh format URLs, so it never reuses the prefetch.
const reuseInfoJson = infoJsonPath !== null;
+ // ---------- Per-download persistence decision ----------
+ // Resolve the channel keep-latest rule (plus per-run overrides) into a concrete
+ // plan: whether to keep the source video and who extracts the audio. Only
+ // affects transcribe-handling downloads (and the youtube no-subs fallback,
+ // which switches to transcribe). Cheap: a set/cutoff compare + one stat.
+ const fmt: AudioFormat =
+ opts.audioFormatOverride ?? opts.channelConfig.audioFormat ?? "mp3";
+ const pinned = canonicalId
+ ? await isDoNotClean(path.join(channelDir, "data", canonicalId))
+ : false;
+ const plan = resolvePersistenceDecision({
+ inWindow: isInKeepWindow(videoUploadKey, opts.keepWindow),
+ pinned,
+ channelExtractionMode: opts.channelConfig.extractionMode,
+ overrides: {
+ keepSourceVideoOverride: opts.keepSourceVideoOverride,
+ extractImmediately: opts.extractImmediately,
+ },
+ });
+
// ---------- Attempt 1: primary ----------
const audioCheckEnabled =
opts.channelConfig.handling === "transcribe" &&
opts.channelConfig.audioCheck?.enabled === true;
+ // The shared media args for the primary + auth-retry attempts. Audio-check owns
+ // its own args; youtube handling downloads subtitles (the persistence plan only
+ // applies to the no-subs fallback below). For transcribe handling the plan
+ // chooses audio-only vs. keep-source-video and yt-dlp vs. app extraction.
+ const mediaArgs =
+ opts.channelConfig.handling === "youtube"
+ ? [
+ ...outputArgsForUrl(opts.videoUrl),
+ ...youtubeHandlingArgs(opts.channelConfig, reuseInfoJson),
+ ]
+ : transcribeMediaArgs(
+ opts.videoUrl,
+ opts.channelConfig,
+ plan,
+ fmt,
+ reuseInfoJson,
+ );
+ if (opts.channelConfig.handling === "transcribe") {
+ opts.onLog(
+ `Persistence: ${plan.persist ? "keep source video" : "audio-only"} via ${plan.extractionMode} extraction (${plan.reason}).\n`,
+ );
+ if (audioCheckEnabled && plan.persist) {
+ opts.onLog(
+ `Note: this video qualifies for source-video persistence, but the channel uses audio-check; persistence is skipped for audio-checked downloads.\n`,
+ );
+ }
+ }
+
let primaryRes: AttemptOutcome;
let audioCheckStats: AudioCheckAttemptStats | undefined;
let audioCheckCorruptSource = false;
@@ -468,8 +611,7 @@ async function runManagedDownload(
"--ignore-config",
"--restrict-filenames",
...FULL_LOG_PROGRESS_ARGS,
- ...outputArgsForUrl(opts.videoUrl),
- ...handlingArgs(opts.channelConfig, reuseInfoJson),
+ ...mediaArgs,
"--print",
`after_video:${ARCHIVE_MARKER} %(extractor)s %(id)s`,
...channelConfigArgs(opts.channelConfig),
@@ -519,8 +661,7 @@ async function runManagedDownload(
"--ignore-config",
"--restrict-filenames",
...FULL_LOG_PROGRESS_ARGS,
- ...outputArgsForUrl(opts.videoUrl),
- ...handlingArgs(opts.channelConfig, reuseInfoJson),
+ ...mediaArgs,
"--print",
`after_video:${ARCHIVE_MARKER} %(extractor)s %(id)s`,
...channelConfigArgs(opts.channelConfig, opts.globalCookiesFromBrowser),
@@ -549,12 +690,34 @@ async function runManagedDownload(
}
}
- // ---------- Attempt 3: no-subs fallback (youtube handling only) ----------
const videoDir =
audioCheckVideoDir ??
path.join(channelDir, "data", canonicalId ?? "unknown");
const videoId = path.basename(videoDir);
+ // ---------- App-side extraction (transcribe handling) ----------
+ // After a successful non-audio-check transcribe download in app mode, produce
+ // audio.<fmt> from the downloaded source-media container and keep or discard it
+ // per the persistence plan. (Audio-check owns its own extraction; youtube
+ // handling extracts inside the no-subs fallback below.)
+ if (
+ lastSucceeded &&
+ !audioCheckEnabled &&
+ opts.channelConfig.handling === "transcribe" &&
+ plan.extractionMode === "app" &&
+ !opts.signal.aborted
+ ) {
+ await finalizeAppExtraction({
+ paths: opts.paths,
+ videoDir,
+ fmt,
+ persist: plan.persist,
+ onLog: opts.onLog,
+ signal: opts.signal,
+ });
+ }
+
+ // ---------- Attempt 3: no-subs fallback (youtube handling only) ----------
if (
lastSucceeded &&
opts.channelConfig.handling === "youtube" &&
@@ -564,9 +727,8 @@ async function runManagedDownload(
const noCaptions = await metadataReportsNoCaptions(videoDir);
if (!hasTranscript && noCaptions) {
opts.onLog(
- `No subs available for ${videoId}; falling back to audio download + whisper.\n`,
+ `No subs available for ${videoId}; falling back to audio download + whisper${plan.persist ? " (keeping source video)" : ""}.\n`,
);
- const audioFmt: AudioFormat = opts.channelConfig.audioFormat ?? "mp3";
const fallbackConfig: ChannelConfig = {
...opts.channelConfig,
handling: "transcribe",
@@ -579,13 +741,14 @@ async function runManagedDownload(
// Feed yt-dlp the metadata it already wrote during the primary
// attempt instead of re-querying the extractor — saves a network
// round-trip per video, which adds up across batch runs and helps
- // dodge rate limits we'd otherwise burn on info we already have.
+ // dodge rate limits we'd otherwise burn on info we already have. The
+ // persistence plan applies here too, so a kept youtube video that lacks
+ // captions still archives its source container.
const fallbackArgs = [
"--ignore-config",
"--restrict-filenames",
...FULL_LOG_PROGRESS_ARGS,
- ...outputArgsForUrl(opts.videoUrl),
- ...transcribeHandlingArgs(fallbackConfig, true),
+ ...transcribeMediaArgs(opts.videoUrl, fallbackConfig, plan, fmt, true),
"--print",
`after_video:${ARCHIVE_MARKER} %(extractor)s %(id)s`,
...channelConfigArgs(fallbackConfig, fallbackCookieOverride),
@@ -614,6 +777,18 @@ async function runManagedDownload(
if (attemptSucceeded(fallbackRes.exitCode)) {
fellBackToTranscribe = true;
+ // In app mode, extract audio.<fmt> from the downloaded source container
+ // (and keep/discard it) before any inline whisper can read the audio.
+ if (plan.extractionMode === "app") {
+ await finalizeAppExtraction({
+ paths: opts.paths,
+ videoDir,
+ fmt,
+ persist: plan.persist,
+ onLog: opts.onLog,
+ signal: opts.signal,
+ });
+ }
if (opts.inlineTranscribeOnFallback) {
// Inline whisper: matches whisperVideoAction's shape. Routes through
// the worker pool so the inline transcription respects worker config
@@ -623,7 +798,7 @@ async function runManagedDownload(
paths: opts.paths,
videoDir,
videoId,
- audioFilename: `audio.${audioFmt}`,
+ audioFilename: `audio.${fmt}`,
onLog: opts.onLog,
signal: opts.signal,
});
diff --git a/common/ytdlp/persistencePlan.test.ts b/common/ytdlp/persistencePlan.test.ts
@@ -0,0 +1,62 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { resolvePersistenceDecision } from "./persistencePlan";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test ytdlp/persistencePlan.test.ts
+
+test("outside the window with no overrides: audio-only via channel mode", () => {
+ const d = resolvePersistenceDecision({ inWindow: false, pinned: false });
+ assert.equal(d.persist, false);
+ assert.equal(d.extractionMode, "ytdlp");
+});
+
+test("channel app extraction is honored for audio-only downloads", () => {
+ const d = resolvePersistenceDecision({
+ inWindow: false,
+ pinned: false,
+ channelExtractionMode: "app",
+ });
+ assert.equal(d.persist, false);
+ assert.equal(d.extractionMode, "app");
+});
+
+test("inside the keep-latest window persists via app extraction", () => {
+ const d = resolvePersistenceDecision({ inWindow: true, pinned: false });
+ assert.equal(d.persist, true);
+ assert.equal(d.extractionMode, "app");
+});
+
+test("a do-not-clean pin persists even outside the window", () => {
+ const d = resolvePersistenceDecision({ inWindow: false, pinned: true });
+ assert.equal(d.persist, true);
+ assert.equal(d.extractionMode, "app");
+});
+
+test("extractImmediately overrides the keep-latest window", () => {
+ const d = resolvePersistenceDecision({
+ inWindow: true,
+ pinned: true,
+ overrides: { extractImmediately: true },
+ });
+ assert.equal(d.persist, false);
+});
+
+test("keepSourceVideoOverride=true persists an out-of-window video", () => {
+ const d = resolvePersistenceDecision({
+ inWindow: false,
+ pinned: false,
+ overrides: { keepSourceVideoOverride: true },
+ });
+ assert.equal(d.persist, true);
+ assert.equal(d.extractionMode, "app");
+});
+
+test("keepSourceVideoOverride=false beats the window and extractImmediately", () => {
+ const d = resolvePersistenceDecision({
+ inWindow: true,
+ pinned: true,
+ overrides: { keepSourceVideoOverride: false, extractImmediately: true },
+ });
+ assert.equal(d.persist, false);
+});
diff --git a/common/ytdlp/persistencePlan.ts b/common/ytdlp/persistencePlan.ts
@@ -0,0 +1,80 @@
+import type { ExtractionMode } from "../lib/channelConfig";
+
+// The per-download persistence rule engine. Each individual download consults
+// the channel's keep-latest rule (plus any per-run overrides) to decide TWO
+// things:
+// 1. persist — keep the source video container (vs. audio-only)
+// 2. extractionMode — who extracts audio: yt-dlp's `-x` or the app's ffmpeg
+//
+// This is the heart of Phase 2: persistence is a per-channel rule evaluated per
+// individual download, NOT a separate runner/queue. Kept as a pure function so
+// the precedence is unit-testable in isolation from yt-dlp and the filesystem.
+
+export type PersistRunOverrides = {
+ // Force-keep (true) or force-discard (false) the source video for this run,
+ // overriding the channel's keep-latest rule. undefined = no override.
+ keepSourceVideoOverride?: boolean;
+ // Force "extract audio now and discard the container" even for a video the
+ // keep-latest rule would otherwise persist (the backfill / save-disk case).
+ extractImmediately?: boolean;
+};
+
+export type PersistenceDecisionInput = {
+ // This video falls inside the channel's keep-latest window (by upload date).
+ inWindow: boolean;
+ // This video carries a do-not-clean pin (manually protected / deleted-from-
+ // source). Pinned videos are irreplaceable, so we persist their source too.
+ pinned: boolean;
+ // The channel's configured extraction strategy for audio-only downloads.
+ channelExtractionMode?: ExtractionMode;
+ overrides?: PersistRunOverrides;
+};
+
+export type PersistenceDecision = {
+ // Keep the downloaded source video container (Phase 3 moves it to the store;
+ // Phase 2 leaves it in the data dir as source-media.<ext>).
+ persist: boolean;
+ extractionMode: ExtractionMode;
+ // Human-readable justification, surfaced in the download log.
+ reason: string;
+};
+
+// Precedence (highest first):
+// 1. per-run keepSourceVideoOverride === true -> persist
+// 2. per-run keepSourceVideoOverride === false -> don't persist
+// 3. per-run extractImmediately -> don't persist (extract now)
+// 4. do-not-clean pin -> persist
+// 5. inside the keep-latest window -> persist
+// 6. otherwise -> audio-only
+// Persisting implies app-side extraction (we need the container in hand); an
+// audio-only outcome uses the channel's configured mode (default "ytdlp").
+export function resolvePersistenceDecision(
+ input: PersistenceDecisionInput,
+): PersistenceDecision {
+ const o = input.overrides ?? {};
+ let persist: boolean;
+ let reason: string;
+ if (o.keepSourceVideoOverride === true) {
+ persist = true;
+ reason = "per-run keep-source-video override";
+ } else if (o.keepSourceVideoOverride === false) {
+ persist = false;
+ reason = "per-run no-keep override";
+ } else if (o.extractImmediately) {
+ persist = false;
+ reason = "per-run extract-immediately override";
+ } else if (input.pinned) {
+ persist = true;
+ reason = "do-not-clean pin";
+ } else if (input.inWindow) {
+ persist = true;
+ reason = "keep-latest window";
+ } else {
+ persist = false;
+ reason = "outside keep-latest window";
+ }
+ const extractionMode: ExtractionMode = persist
+ ? "app"
+ : (input.channelExtractionMode ?? "ytdlp");
+ return { persist, extractionMode, reason };
+}
diff --git a/common/ytdlp/runYtdlp.ts b/common/ytdlp/runYtdlp.ts
@@ -23,6 +23,7 @@ import {
import { resolveEffectiveAvailability } from "../lib/availability-server";
import { backfillAvailabilityFromMetadata } from "../controller/backfillAvailability";
import { resolveShardItems } from "../controller/shard";
+import { computeKeepWindow, type KeepWindow } from "../controller/keptVideos";
import { downloadOneManaged } from "./downloadOneManaged";
import type { TaskTracker } from "../jobs/taskHooks";
import type { JobProgress } from "../jobs/registry";
@@ -70,8 +71,15 @@ export type RunYtdlpOpts = {
progressBaseline?: number;
// download-one-audio only: full webpage URL of the single video to fetch.
singleVideoUrl?: string;
- // download-one-audio only: override channelConfig.audioFormat for this run.
+ // Override channelConfig.audioFormat for this run. Honored by download-one-audio
+ // and by the managed download paths (passed through to downloadOneManaged).
audioFormatOverride?: AudioFormat;
+ // Per-run persistence overrides (Phase 2), threaded to downloadOneManaged for
+ // every managed download in this run. keepSourceVideoOverride forces keep
+ // (true) / discard (false); extractImmediately forces extract-now + discard
+ // even for a video the keep-latest rule would persist.
+ keepSourceVideoOverride?: boolean;
+ extractImmediately?: boolean;
// download-one-audio only: appended after configArgs, before the URL.
extraYtdlpArgs?: string[];
// retry-bucket only: video IDs to limit the run to (matched against the
@@ -177,12 +185,20 @@ const OUTPUT_ARGS: string[] = [
// path literally to the canonical id and stop depending on the extractor id.
// Falls back to %(id)s for URLs whose canonical id is missing or not a safe
// directory name (the post-download reconcile pass cleans those up).
-export function outputArgsForUrl(url: string): string[] {
+// `mediaName` overrides the base name of the main media output (default
+// "audio"). App-side extraction downloads the source container as
+// `source-media.<ext>` so it's never confused with an extractable audio file;
+// every other caller keeps the historical `audio.<ext>`.
+export function outputArgsForUrl(
+ url: string,
+ opts: { mediaName?: string } = {},
+): string[] {
+ const mediaName = opts.mediaName ?? "audio";
const id = extractVideoId(url);
if (id && /^[\w.-]+$/.test(id) && id !== "." && id !== "..") {
return [
"-o",
- `data/${id}/audio.%(ext)s`,
+ `data/${id}/${mediaName}.%(ext)s`,
"-o",
`subtitle:data/${id}/transcript`,
"-o",
@@ -190,6 +206,17 @@ export function outputArgsForUrl(url: string): string[] {
"--no-write-playlist-metafiles",
];
}
+ if (mediaName !== "audio") {
+ return [
+ "-o",
+ `data/%(id)s/${mediaName}.%(ext)s`,
+ "-o",
+ "subtitle:data/%(id)s/transcript",
+ "-o",
+ "infojson:data/%(id)s/metadata",
+ "--no-write-playlist-metafiles",
+ ];
+ }
return OUTPUT_ARGS;
}
@@ -547,6 +574,16 @@ async function runManagedDownloads(
opts.channelConfig.sleepBetweenDownloadsSeconds ??
settings.sleepBetweenDownloadsSeconds;
+ // The channel's keep-latest window as a cutoff, computed ONCE per run from the
+ // current on-disk catalog. Each video downloaded below is classified against it
+ // by its own upload date — so the newest videos (not yet on disk) still get
+ // persisted. A keepLatest of 0/undefined yields an inert window.
+ const keepWindow: KeepWindow = await computeKeepWindow({
+ paths: opts.paths,
+ channelSlug: opts.channelSlug,
+ keepLatest: effectiveChannelConfig.keepLatest ?? 0,
+ });
+
const limit = pLimit(1);
let failedCount = 0;
let skippedCount = 0;
@@ -593,6 +630,10 @@ async function runManagedDownloads(
appendArchive: !opts.ignoreArchive,
inlineTranscribeOnFallback,
globalSkipLiveDownloads,
+ keepWindow,
+ keepSourceVideoOverride: opts.keepSourceVideoOverride,
+ extractImmediately: opts.extractImmediately,
+ audioFormatOverride: opts.audioFormatOverride,
});
} finally {
task?.end();
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,7 @@
# Changelog
## [Unreleased]
+- **Per-download persistence rule + app-side audio extraction (phase 2 of the video-persistence subsystem; backend, UI lands later).** Each individual download now consults the channel's keep-latest rule (plus any per-run overrides) to decide *what to keep*: a video inside the keep-latest window — or one carrying a `do-not-clean` pin — downloads its **full source video** (`bestvideo*+bestaudio/best`) and the app extracts `audio.<fmt>` from it with ffmpeg, keeping the container as `source-media.<ext>` (a deliberately distinct name from `audio.<ext>` so it's never mistaken for cleanable audio); everything else stays audio-only as before. Crucially the keep decision is made per video by its **upload date against the channel's Nth-newest cutoff** (computed once per run via the new `computeKeepWindow`/`isInKeepWindow`), so the newest videos — which aren't on disk yet at download time — are correctly persisted. A new **`extractionMode`** channel setting (`"ytdlp"` default | `"app"`) selects who extracts audio for audio-only downloads; persisting always forces app-side extraction. **Per-run overrides** thread through `download-missing` (and the shared managed-download path): `keepSourceVideoOverride` (force keep/discard), `extractImmediately` (extract now + discard the container even on a keep channel — the disk-saving backfill case), and `audioFormatOverride`. **Transcription falls back to the source container** when no extracted `audio.*` exists (parakeet ffmpeg-slices any container), so a kept-but-cleaned video or an archive-only download is still transcribable. New per-video **"Archive source video"** action (`redownloadToArchiveAction`) re-fetches an existing video purely to grab + keep its source container without disturbing the transcript. Legacy `"ytdlp"`-mode downloads produce byte-identical yt-dlp args to before (no behavior change for existing channels). Note: persisted source containers currently remain in the data dir and are not auto-pruned when they roll out of the window, and source-video persistence is skipped (with a log note) for audio-check channels — both addressed by the saved-video store in phase 3. See `common/ytdlp/persistencePlan.ts` (+ tests), `common/controller/keptVideos.ts` (keep-window), `common/ytdlp/downloadOneManaged.ts`, `common/ytdlp/runYtdlp.ts`, `common/controller/transcribeOne.ts`, `common/lib/videoStatus.ts` (`source-media`/`isVideoContainer`), and `editor/app/channels/[slug]/pipelineActions.ts` / `videos/[id]/videoActions.ts`.
- **New per-channel "keep latest N" retention rule that protects recent videos from the Clean-audio sweep and pins any that get deleted from their source (backend; UI lands in a later change).** A channel can set `keepLatest` (in `config.json` for now) to shield its newest N videos — by upload date, a rolling window — from the **Clean audio** cleanup: those dirs are skipped just like a `do-not-clean` marker, and the snapshot's reclaim estimates/cleanup buckets exclude them (a new `keptCount` is recorded). Because a kept video can later be **deleted from its source** (YouTube etc.) and become irreplaceable, a new **kept-deletion check** re-probes just the kept window's availability (reusing `runAvailabilityCheck` with `onlyIds` + `recheck-non-deleted`) and **permanently pins** any video found `deleted`/`private`/`members_only` with a `do-not-clean` marker, so it survives even after it rolls out of the window. The check runs on demand via the new `check-kept-deleted` managed job (`checkKeptDeletedAction`, re-runnable/bookmarkable) and automatically from the sync scheduler on its own cadence (new `syncScheduler.keepLatestCheckIntervalMinutes`, default daily; per-channel `lastKeptCheckAt` state; suppressed during quiet hours, capped per tick, and skipped for a channel just synced this tick). This is phase 1 of a larger **video-persistence** subsystem (per-download persistence rules, a separate saved-video store, and backups follow). See `common/lib/channelConfig.ts` (`keepLatest`), the new `common/controller/keptVideos.ts` (`computeKeptVideoIds` + tests) and `common/controller/checkKeptDeleted.ts`, `common/controller/cleanAudioFromTranscribed.ts`, `common/controller/channelSnapshot.ts`, and the scheduler wiring in `common/lib/settings.ts`, `common/jobs/syncSchedulerState.ts`, and `editor/app/scheduler/runTick.ts`.
- **The monitor widget can now carry optional control buttons (Pause/Resume Transcriptions, Drain all).** The `/widget` view stays read-only by default, but a new **Show control buttons** option in the widget builder (URL flag `controls=1`) adds an interactive row at the top with the same **Pause Transcriptions** toggle (between-segment GPU release) and **Drain all** as the full app — so a pinned/iframe monitor can pause the GPU or wind work down without opening the editor. The controls stay visible even when the widget is otherwise idle (so you can pause preemptively), and the widget refetches its worker payload on a pause/resume so the toggle flips immediately instead of waiting for the next poll. Defaults keep the widget control-free, so existing links render unchanged. See `editor/app/widget/lib/config.ts`, the new `editor/app/widget/components/WidgetControls.tsx`, `editor/app/widget/components/MonitorWidget.tsx`, and `editor/app/widget/builder/components/WidgetBuilder.tsx`.
- **"Pause Transcriptions" now frees the GPU between parakeet segments instead of running the in-flight video to completion.** The global pause control (renamed from "Pause all" → **Pause Transcriptions** / **Resume Transcriptions**) used to only stop handing out new worker slots — any in-flight transcription kept running until its whole file was done, so the GPU stayed busy. Pausing now *also* sends a graceful between-segment stop to any partial-capable in-flight job: a **parakeet** worker finishes the current window, caches it (`win-NNNN.json`), and exits `paused` (a skip, not a failure), so the GPU frees within one segment and the video resumes from its cached windows on the next run — the same mechanism as the per-worker **Stop & keep progress** button, now wired into the global pause. Non-parakeet engines (whisper-cpp, chough) keep prior behavior: they stop taking new work but run their in-flight file to completion. The button is also surfaced on the **Active Jobs** screen (`/jobs/active`) next to **Drain all**, not just the Workers page. `resumeAll()` restores each worker's pre-pause state as before. See `common/jobs/workerPool.ts` (`pauseAll`), the new `editor/app/jobs/components/PauseTranscriptionsButton.tsx` (shared by both screens), `editor/app/workers/components/WorkersView.tsx`, and `editor/app/jobs/active/page.tsx`.
diff --git a/editor/app/channels/[slug]/pipelineActions.ts b/editor/app/channels/[slug]/pipelineActions.ts
@@ -3,6 +3,7 @@
import { revalidatePath } from "next/cache";
import {
HANDLING_VALUES,
+ type AudioFormat,
type ChannelHandling,
} from "yt-dlp-transcript-common/lib/channelConfig";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
@@ -69,6 +70,11 @@ async function runPipelineAction(
shardIndex?: number;
bucketIds?: ReadonlyArray<string>;
handlingOverride?: ChannelHandling;
+ // Per-run persistence overrides (Phase 2), forwarded to runYtdlp →
+ // downloadOneManaged for every managed download in this run.
+ keepSourceVideoOverride?: boolean;
+ extractImmediately?: boolean;
+ audioFormatOverride?: AudioFormat;
// Replay descriptor, forwarded onto the job record so it can be bookmarked.
spec?: JobSpec;
},
@@ -160,6 +166,9 @@ async function runPipelineAction(
shardIndex: options?.shardIndex,
bucketIds: options?.bucketIds,
handlingOverride: options?.handlingOverride,
+ keepSourceVideoOverride: options?.keepSourceVideoOverride,
+ extractImmediately: options?.extractImmediately,
+ audioFormatOverride: options?.audioFormatOverride,
setProgress,
progressBaseline,
// On a 429/network failure, record the shared per-platform cooldown so
@@ -214,6 +223,12 @@ export async function downloadMissingAction(
abortOnError?: boolean,
shardTotal?: number,
shardIndex?: number,
+ // Per-run persistence overrides (Phase 2). The canonical use is a backfill that
+ // extracts audio immediately to save disk (extractImmediately), but a forced
+ // keep / audio-format override is supported too.
+ keepSourceVideoOverride?: boolean,
+ extractImmediately?: boolean,
+ audioFormatOverride?: AudioFormat,
): Promise<StreamActionResult> {
return runPipelineAction(
slug,
@@ -225,10 +240,22 @@ export async function downloadMissingAction(
abortOnError,
shardTotal,
shardIndex,
+ keepSourceVideoOverride,
+ extractImmediately,
+ audioFormatOverride,
spec: {
kind: "download-missing",
slug,
- params: { queueKey, ignoreArchive, abortOnError, shardTotal, shardIndex },
+ params: {
+ queueKey,
+ ignoreArchive,
+ abortOnError,
+ shardTotal,
+ shardIndex,
+ keepSourceVideoOverride,
+ extractImmediately,
+ audioFormatOverride,
+ },
},
},
);
diff --git a/editor/app/channels/[slug]/videos/[id]/videoActions.ts b/editor/app/channels/[slug]/videos/[id]/videoActions.ts
@@ -9,7 +9,9 @@ import type {
ChannelConfig,
} from "yt-dlp-transcript-common/lib/channelConfig";
import { AUDIO_FORMAT_VALUES } from "yt-dlp-transcript-common/lib/channelConfig";
-import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { getPaths, type Paths } from "yt-dlp-transcript-common/lib/paths";
+import { checkDiskSpace } from "yt-dlp-transcript-common/lib/diskSpace";
+import { formatBytes } from "yt-dlp-transcript-common/lib/format";
import {
audioFilesToRemove,
isRealAudioFile,
@@ -171,6 +173,82 @@ export async function downloadVideoPipelineAction(
});
}
+// Re-fetch an already-downloaded video purely to archive its SOURCE container,
+// without disturbing the existing transcript. Forces the persistence rule on
+// (keepSourceVideoOverride=true) so downloadOneManaged downloads the full video,
+// app-extracts audio, and keeps the container (Phase 3 moves it to the saved
+// store). Works on a video already in the archive — downloadOneManaged has no
+// archive prefilter, so it always re-downloads.
+export async function redownloadToArchiveAction(
+ slug: string,
+ videoId: string,
+ queueKey?: string,
+): Promise<StreamActionResult> {
+ const r = await loadConfigOrError(slug);
+ if (!r.ok) return r;
+ const paths = getPaths();
+ const url = await findVideoSourceUrl(paths, slug, videoId, r.config);
+ if (!url) {
+ return {
+ ok: false,
+ error:
+ "Could not determine the video URL: no metadata.info.json and the playlist does not contain a matching entry.",
+ };
+ }
+ const err = await lowDiskError(paths);
+ if (err) return err;
+ const settings = getSettings();
+ return runManagedFunction({
+ kind: "redownload-archive",
+ queueKey: videoQueueKey(r.config, queueKey),
+ paths,
+ channelSlug: slug,
+ videoId,
+ fn: async (onLog, signal, _setProgress, ctx) => {
+ const task = makeTaskTracker(ctx, onLog).start({
+ id: videoId,
+ label: videoId,
+ kind: "download",
+ });
+ try {
+ onLog(`Re-downloading ${videoId} to archive its source video…\n`);
+ await downloadOneManaged({
+ channelSlug: slug,
+ channelConfig: r.config,
+ paths,
+ videoUrl: url,
+ onLog: task.onLog,
+ signal,
+ globalCookiesFromBrowser: settings.cookiesFromBrowser || undefined,
+ inlineTranscribeOnFallback: settings.inlineTranscribeOnFallback,
+ globalSkipLiveDownloads: settings.skipLiveDownloads,
+ appendArchive: true,
+ keepSourceVideoOverride: true,
+ });
+ revalidatePath(`/channels/${slug}/videos/${videoId}`);
+ revalidatePath(`/channels/${slug}`);
+ } finally {
+ task.end();
+ }
+ },
+ });
+}
+
+// Preflight disk-space gate shared with the per-video download actions.
+async function lowDiskError(
+ paths: Paths,
+): Promise<{ ok: false; error: string } | null> {
+ const disk = await checkDiskSpace(paths, getSettings());
+ if (disk.ok) return null;
+ return {
+ ok: false,
+ error:
+ `Low disk space: ${formatBytes(disk.freeBytes)} free, ` +
+ `${formatBytes(disk.thresholdBytes)} required. Free up space or ` +
+ `lower the floor in Settings.`,
+ };
+}
+
export async function whisperVideoAction(
slug: string,
videoId: string,
diff --git a/editor/app/jobs/jobKindLabels.ts b/editor/app/jobs/jobKindLabels.ts
@@ -13,6 +13,7 @@ const JOB_KIND_LABELS: Record<string, string> = {
"download-missing": "Download missing",
"download-missing-subs": "Download missing subs",
"import-one": "Import video",
+ "redownload-archive": "Archive source video",
"retry-bucket": "Retry",
"clean-audio-transcribed": "Clean audio",
"check-kept-deleted": "Check kept videos",
diff --git a/editor/app/jobs/runJobSpec.ts b/editor/app/jobs/runJobSpec.ts
@@ -102,6 +102,9 @@ export async function runJobSpec(spec: JobSpec): Promise<StreamActionResult> {
bool(p.abortOnError),
num(p.shardTotal),
num(p.shardIndex),
+ bool(p.keepSourceVideoOverride),
+ bool(p.extractImmediately),
+ str(p.audioFormatOverride) as AudioFormat | undefined,
);
case "sync":
return syncAction(spec.slug, queueKey);
