import path from "node:path"; import type { Paths } from "../lib/paths"; import type { ChannelConfig } from "../lib/channelConfig"; import { getSettings } from "../lib/settings"; import { diskGate } from "../lib/diskSpace"; import { resolveCookiePolicy } from "../lib/cookiePolicy"; import { isSavedVideo } from "../lib/savedVideo-server"; import { computeKeptVideoIds } from "./keptVideos"; import { findVideoSourceUrl } from "./undownloadedVideos"; import { downloadOneManaged, sourceFetchFailure, } from "../ytdlp/downloadOneManaged"; import { resolveSourceVideoQuality, type SourceVideoQuality, } from "../ytdlp/downloadFormat"; // Bulk "persist kept now" pass (Phase 5). Ensures every video currently in the // channel's keep-latest window has its source container saved to the store, for // the catch-up case where the window was widened (or persistence was enabled) // after those videos had already been downloaded audio-only. // // For each kept video whose source isn't saved yet, it re-fetches the container // via downloadOneManaged with keepSourceVideoOverride + forceMedia (which moves // the container into the saved store; a youtube-handling video that already has // a transcript gets no audio extracted), without disturbing the existing // transcript. Mirrors redownloadToArchiveAction (videoActions.ts) but loops over // the whole window. Videos already saved, or whose URL can't be resolved, are // skipped rather than failing the pass. export type PersistKeptResult = { // Total videos in the keep-latest window. kept: number; // Already had a saved-video pointer; left untouched. alreadySaved: number; // Newly persisted this pass. persisted: number; // Re-download attempted but failed. failed: number; // No resolvable source URL (no metadata + not in the playlist). skippedNoUrl: number; // Not attempted because free disk fell under the floor part-way through. A // refusal to start, not a failure — these videos are still kept, still // unpersisted, and the next pass picks them up. skippedLowDisk: number; }; export async function persistKept({ paths, channelSlug, channelConfig, onLog, signal, quality, }: { paths: Paths; channelSlug: string; channelConfig: ChannelConfig; onLog?: (line: string) => void; signal?: AbortSignal; // Per-run override of the source-video quality; absent = the channel's, // else the global setting (resolveSourceVideoQuality). quality?: SourceVideoQuality; }): Promise { const log = (line: string) => onLog?.(line.endsWith("\n") ? line : `${line}\n`); // downloadOneManaged requires a non-optional onLog/signal; supply inert // fallbacks so persistKept stays callable without a managed-job context. const downloadLog = (line: string) => onLog?.(line); const downloadSignal = signal ?? new AbortController().signal; const keepLatest = channelConfig.keepLatest ?? 0; const keptIds = await computeKeptVideoIds({ paths, channelSlug, keepLatest, }); const result: PersistKeptResult = { kept: keptIds.size, alreadySaved: 0, persisted: 0, failed: 0, skippedNoUrl: 0, skippedLowDisk: 0, }; if (keptIds.size === 0) { log( keepLatest > 0 ? "Persist kept: no videos in the keep-latest window yet." : "Persist kept: keep-latest is off for this channel; nothing to do.", ); return result; } const settings = getSettings(); const persistFormatPreset = resolveSourceVideoQuality({ override: quality, channel: channelConfig.sourceVideoQuality, global: settings.sourceVideoQuality, }); log(`Persist kept: source video quality ${persistFormatPreset}.`); const dataDir = path.join(paths.channelsDir, channelSlug, "data"); // Newest-first so the freshest videos are secured even if the pass is cancelled // partway through. computeKeptVideoIds returns an unordered set; sort by id desc // as a stable proxy (YYYYMMDD_-prefixed ids sort by recency, like the window). const ordered = [...keptIds].sort((a, b) => b.localeCompare(a)); // Set once the disk gate closes mid-pass. Everything after it is counted as // skipped rather than attempted, so the log says how much was left. let lowDisk = false; for (const videoId of ordered) { if (signal?.aborted) break; if (lowDisk) { result.skippedLowDisk += 1; continue; } const videoDir = path.join(dataDir, videoId); if (await isSavedVideo(videoDir)) { result.alreadySaved += 1; continue; } // PER ITEM, not once at the start. This pass writes full source containers — // the largest files the app produces — so the twentieth video must not // inherit the first one's headroom. Same shape as backfillReacquire's // per-item floor. Checked only for videos that will actually be fetched, so // an already-saved window costs no syscalls. const gate = await diskGate(paths, settings); if (!gate.ok) { lowDisk = true; result.skippedLowDisk += 1; log(` ${videoId}: ${gate.message} — stopping; remaining videos skipped.`); continue; } const url = await findVideoSourceUrl(paths, channelSlug, videoId, channelConfig); if (!url) { result.skippedNoUrl += 1; log(` ${videoId}: no resolvable source URL, skipping.`); continue; } log(` ${videoId}: re-fetching source container to persist…`); try { const record = await downloadOneManaged({ channelSlug, channelConfig, paths, videoUrl: url, onLog: downloadLog, signal: downloadSignal, cookiePolicy: resolveCookiePolicy(settings, channelConfig), inlineTranscribeOnFallback: settings.inlineTranscribeOnFallback, globalSkipLiveDownloads: settings.skipLiveDownloads, appendArchive: true, keepSourceVideoOverride: true, // "Persist kept" means persist: on a youtube-handling channel the media // pass is otherwise skipped for any video with a transcript or // captions, which is every kept one (release 10 slice N). forceMedia: true, persistFormatPreset, }); // A download that RETURNED is not a persisted source (release 11 slice // O3, review low 2): a failed forced media pass leaves the download `ok` // (the transcript is fine) with only its n: 3 attempt failed. const failure = sourceFetchFailure(record); if (failure) { result.failed += 1; log(` ${videoId}: persist failed — ${failure}`); } else { result.persisted += 1; } } catch (e) { result.failed += 1; log(` ${videoId}: persist failed — ${(e as Error).message}`); } } log( `Persist kept: ${result.persisted} persisted, ${result.alreadySaved} already saved, ` + `${result.failed} failed, ${result.skippedNoUrl} skipped (no URL)` + (result.skippedLowDisk > 0 ? `, ${result.skippedLowDisk} skipped (low disk)` : "") + ` of ${result.kept} kept.`, ); return result; }