import { removeMediaFile } from "../lib/mediaTier-server"; import path from "node:path"; import fs from "fs-extra"; import type { Paths } from "../lib/paths"; import { isDoNotClean } from "../lib/doNotClean-server"; import { hasDiarization } from "../lib/diarization-server"; import { getSettings } from "../lib/settings"; import { audioFilesToRemove } from "../lib/videoStatus"; import { readChannelConfig } from "./channels"; import { computeKeptVideoIds } from "./keptVideos"; import { pruneSavedVideos } from "./pruneSavedVideos"; import { excludedIds, verifyBeforeClean } from "./verifyBeforeClean"; const { pathExists, readdir } = fs; export type CleanAudioOptions = { channelSlug: string; paths: Paths; // Check each candidate is still available upstream before deleting its audio // (see verifyBeforeClean). Omitted → the global // SiteSettings.verifyAvailabilityBeforeClean, which defaults to true. verifyAvailability?: boolean; onLog?: (msg: string) => void; signal?: AbortSignal; }; export type CleanAudioResult = { inspected: number; cleanedDirs: number; removedFiles: number; // TOTAL not cleaned: protected (keep-latest / do-not-clean) + pinned + // unverified. skipped: number; // Newly marked do-not-clean this run because the source copy is gone. pinned: number; // Left alone because availability could not be resolved — no marker written, // retried on the next sweep. unverified: number; // Left alone because diarization is enabled and this video has no // diarization.json yet. Counted separately from `skipped` because it is a // TRANSIENT hold, not a protection: it clears itself as the diarize lane // catches up, and a number that only ever grows is the signal that the CPU // lane has fallen behind the GPU. awaitingDiarization: number; // Saved-store containers evicted because they rolled out of the keep-latest // window (the retention prune runs alongside the audio sweep). prunedSavedVideos: number; prunedBytes: number; }; type CleanCandidate = { id: string; videoDir: string; audioFiles: string[]; }; export async function cleanAudioFromTranscribed({ channelSlug, paths, verifyAvailability, onLog, signal, }: CleanAudioOptions): Promise { const log = onLog ?? ((m: string) => console.log(m)); const dataDir = path.join(paths.channelsDir, channelSlug, "data"); if (!(await pathExists(dataDir))) { log(`No data directory for ${channelSlug}`); return { inspected: 0, cleanedDirs: 0, removedFiles: 0, skipped: 0, pinned: 0, unverified: 0, awaitingDiarization: 0, prunedSavedVideos: 0, prunedBytes: 0, }; } const dirs = await readdir(dataDir); // The rolling keep-latest window is protected from cleanup just like the // explicit do-not-clean marker (the source media of recent videos is kept). const config = await readChannelConfig(paths, channelSlug); const keptIds = await computeKeptVideoIds({ paths, channelSlug, keepLatest: config?.keepLatest ?? 0, }); let cleanedDirs = 0; let removedFiles = 0; let protectedCount = 0; let awaitingDiarization = 0; // While diarization is enabled, a transcribed video that still has audio but // no diarization.json is NOT cleanable yet — its diarize job hasn't run. // // This guard is the whole reason deferring diarization to an async job is // safe. Without it the sweep would delete the audio out from under a queued // diarize job, silently and permanently: audio is the one input in this // pipeline that cannot be regenerated, and the failure would leave no trace. const diarization = getSettings().diarization; // --- Discover ------------------------------------------------------------- // Nothing is deleted in this pass. Protected videos are filtered out here, so // the availability gate never spends a spawn on a video we would keep anyway. const candidates: CleanCandidate[] = []; for (const id of dirs) { if (signal?.aborted) break; const videoDir = path.join(dataDir, id); const entries = await readdir(videoDir).catch(() => [] as string[]); if (!entries.includes("transcript.json")) continue; const audioFiles = audioFilesToRemove(entries); if (audioFiles.length === 0) continue; if (keptIds.has(id)) { log(`Skipped ${id} (in keep-latest window)`); protectedCount++; continue; } if (await isDoNotClean(videoDir)) { log(`Skipped ${id} (marked do not clean)`); protectedCount++; continue; } if (diarization.enabled && !(await hasDiarization(videoDir))) { log(`Skipped ${id} (awaiting diarization)`); awaitingDiarization++; continue; } candidates.push({ id, videoDir, audioFiles }); } // --- Verify --------------------------------------------------------------- const verify = verifyAvailability ?? getSettings().verifyAvailabilityBeforeClean; let excluded = new Set(); let pinned = 0; let unverified = 0; if (verify && candidates.length > 0 && !signal?.aborted) { const verdicts = await verifyBeforeClean({ channelSlug, paths, candidateIds: candidates.map((c) => c.id), onLog: log, signal, }); excluded = excludedIds(verdicts); pinned = verdicts.pinned.size; unverified = verdicts.unverified.size; // A video already pinned by someone else is protected, not newly pinned. protectedCount += verdicts.alreadyGone.size; } // --- Delete --------------------------------------------------------------- for (const candidate of candidates) { if (signal?.aborted) break; if (excluded.has(candidate.id)) continue; for (const f of candidate.audioFiles) { // Through its link when tiered (release 17): the bytes on the media tier go too. await removeMediaFile(candidate.videoDir, f); log(`Removed ${candidate.id}/${f}`); removedFiles++; } cleanedDirs++; } const skipped = protectedCount + pinned + unverified + awaitingDiarization; // Keep the historical wording when the gate had nothing to report, and break // the total down only when it would otherwise be misleading. // The diarization clause is appended only when it is non-zero, so the message // stays byte-identical for every install that doesn't use the capture lane — // a "0 awaiting diarization" on a box with diarization off is noise, and this // exact string is asserted in pre-clean-availability.spec. const diarNote = awaitingDiarization > 0 ? `, ${awaitingDiarization} awaiting diarization` : ""; const skippedNote = skipped === 0 ? "" : pinned === 0 && unverified === 0 && awaitingDiarization === 0 ? ` Skipped ${skipped} (do not clean).` : ` Skipped ${skipped} (${protectedCount} protected, ${pinned} gone-from-source pinned, ${unverified} unverified${diarNote}).`; log( `Cleaned ${removedFiles} audio file(s) from ${cleanedDirs} of ${dirs.length} video dir(s).${skippedNote}`, ); // Retention prune of the saved-video store: evict any keep-latest containers // that have rolled out of the window (pinned/manually-archived ones survive). const prune = await pruneSavedVideos({ channelSlug, paths, onLog: log, signal }); return { inspected: dirs.length, cleanedDirs, removedFiles, skipped, pinned, unverified, awaitingDiarization, prunedSavedVideos: prune.pruned, prunedBytes: prune.bytesFreed, }; }