commit 1de3212e41b9c81ff6089008f51156e39c813f10
parent e89a8cad6bbc1faddd7b942add0dedc92afa658d
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 31 Jul 2026 19:06:19 -0400
Check the video still exists upstream before deleting its audio
The transcribed-audio sweep hard-deletes a video's audio.* files once whisper
has produced a transcript -- remove(), no trash, no undo -- and nothing had
ever asked whether the video was still there. A video since removed, privated,
or put behind a membership, sitting outside the keep-latest window, lost its
source audio at exactly the moment the local copy became the only copy.
cleanAudioFromTranscribed is now discover -> verify -> delete. The gate
(verifyBeforeClean.ts) runs three tiers cheapest-first, so a typical run costs
one yt-dlp spawn per channel plus a handful for suspects rather than one per
cleanable video:
A. cached availability 0 spawns also the only tier that can catch
members_only, which stays listed in
the channel playlist
B. flat-playlist diff 1/channel narrows to candidates that dropped
out of the listing
C. per-suspect probe 1/suspect separates deleted/private from an
unlisted video that legitimately left
the listing
Anything gone gets a do-not-clean.json marker and is skipped. Anything the
check cannot resolve -- probe error, age gate, no webpage_url to probe -- is
left alone with NO marker and retried next run, so a rate-limited or offline
source cleans nothing rather than cleaning wrongly. A channel with no URL is
structurally unverifiable and fails OPEN; a playlist fetch that is attempted
and fails knows better and fails CLOSED.
On by default, with a Settings opt-out for offline or URL-less setups where
the check can never resolve.
Also:
- Lift the gone rule into availability.ts as isPermanentlyGone(); checkKeptDeleted
drops its local copy.
- The sweep shares isRealAudioFile instead of its own filter, so it stops
deleting audio.live_chat.json and the .part.good/.part.testing audio-check
snapshots -- which the reclaim estimate never counted, so the two had drifted.
- runAvailabilityCheck gains ignoreShard: a saved shard slice on disk otherwise
replaces an explicit onlyIds list wholesale.
- The e2e baseline settings disable the gate. one-transcribe-channel-with-audio
has no metadata.info.json, so with it on every candidate is an unprobeable
suspect and nothing is ever cleaned; pre-clean-availability.spec.ts opts in
explicitly and uses the availability-baseline fixture instead.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Diffstat:
12 files changed, 742 insertions(+), 33 deletions(-)
diff --git a/common/controller/checkAvailability.ts b/common/controller/checkAvailability.ts
@@ -55,6 +55,12 @@ export type CheckAvailabilityOpts = {
// Skip ids already known permanently gone (deleted / private / members_only,
// i.e. EXCLUDED_FROM_DOWNLOAD) — they will not change, so don't re-probe.
skipExpectedAbsent?: boolean;
+ // Bypass shard resolution entirely. Needed by callers that pass an explicit,
+ // already-narrowed onlyIds set: with no shardTotal/shardIndex but a saved
+ // shard-availability.json on disk, resolveShardItems returns the SAVED slice
+ // and silently discards the onlyIds filter. Set this when the caller's id list
+ // is the authority (the pre-clean gate probes a specific suspect set).
+ ignoreShard?: boolean;
onLog?: (msg: string) => void;
signal?: AbortSignal;
};
@@ -103,6 +109,7 @@ export async function runAvailabilityCheck({
saveShardOnly = false,
onlyIds,
skipExpectedAbsent = false,
+ ignoreShard = false,
onLog,
signal,
}: CheckAvailabilityOpts): Promise<CheckAvailabilityResult> {
@@ -127,15 +134,17 @@ export async function runAvailabilityCheck({
const onlyIdSet = new Set(onlyIds);
allDirs = allDirs.filter((id) => onlyIdSet.has(id));
}
- const shardResult = await resolveShardItems({
- paths,
- slug: channelSlug,
- op: "availability",
- fullItems: allDirs,
- totalShards: shardTotal,
- shardIndex: shardIndex,
- onLog: log,
- });
+ const shardResult = ignoreShard
+ ? { items: allDirs, source: "full" as const, config: null }
+ : await resolveShardItems({
+ paths,
+ slug: channelSlug,
+ op: "availability",
+ fullItems: allDirs,
+ totalShards: shardTotal,
+ shardIndex: shardIndex,
+ onLog: log,
+ });
const videoDirs = shardResult.items;
if (saveShardOnly) {
log(
diff --git a/common/controller/checkKeptDeleted.ts b/common/controller/checkKeptDeleted.ts
@@ -1,6 +1,6 @@
import path from "node:path";
import type { Paths } from "../lib/paths";
-import { EXCLUDED_FROM_DOWNLOAD, type Availability } from "../lib/availability";
+import { isPermanentlyGone } from "../lib/availability";
import { resolveEffectiveAvailability } from "../lib/availability-server";
import { loadDoNotClean, setDoNotClean } from "../lib/doNotClean-server";
import { readChannelConfig } from "./channels";
@@ -31,13 +31,6 @@ export type CheckKeptDeletedResult = {
pinned: number;
};
-function isGone(a: Availability | null): boolean {
- return (
- a !== null &&
- (EXCLUDED_FROM_DOWNLOAD as ReadonlyArray<Availability>).includes(a)
- );
-}
-
export async function checkKeptDeleted({
channelSlug,
paths,
@@ -79,7 +72,7 @@ export async function checkKeptDeleted({
if (signal?.aborted) break;
const videoDir = path.join(dataDir, id);
const availability = await resolveEffectiveAvailability(videoDir);
- if (!isGone(availability)) continue;
+ if (!isPermanentlyGone(availability)) continue;
deleted++;
// Only count/log a NEW pin — leave an existing marker (and its note) intact.
const already = await loadDoNotClean(videoDir);
diff --git a/common/controller/cleanAudioFromTranscribed.ts b/common/controller/cleanAudioFromTranscribed.ts
@@ -2,15 +2,22 @@ import path from "node:path";
import fs from "fs-extra";
import type { Paths } from "../lib/paths";
import { isDoNotClean } from "../lib/doNotClean-server";
+import { getSettings } from "../lib/settings";
+import { audioFilesToRemove } from "../lib/videoStatus";
import { readChannelConfig } from "./channels";
import { computeKeptVideoIds } from "./keptVideos";
import { pruneSavedVideos } from "./pruneSavedVideos";
+import { excludedIds, verifyBeforeClean } from "./verifyBeforeClean";
const { pathExists, readdir, remove } = fs;
export type CleanAudioOptions = {
channelSlug: string;
paths: Paths;
+ // Check each candidate is still available upstream before deleting its audio
+ // (see verifyBeforeClean). Omitted → the global
+ // SiteSettings.verifyAvailabilityBeforeClean, which defaults to true.
+ verifyAvailability?: boolean;
onLog?: (msg: string) => void;
signal?: AbortSignal;
};
@@ -19,16 +26,30 @@ export type CleanAudioResult = {
inspected: number;
cleanedDirs: number;
removedFiles: number;
+ // TOTAL not cleaned: protected (keep-latest / do-not-clean) + pinned +
+ // unverified.
skipped: number;
+ // Newly marked do-not-clean this run because the source copy is gone.
+ pinned: number;
+ // Left alone because availability could not be resolved — no marker written,
+ // retried on the next sweep.
+ unverified: number;
// Saved-store containers evicted because they rolled out of the keep-latest
// window (the retention prune runs alongside the audio sweep).
prunedSavedVideos: number;
prunedBytes: number;
};
+type CleanCandidate = {
+ id: string;
+ videoDir: string;
+ audioFiles: string[];
+};
+
export async function cleanAudioFromTranscribed({
channelSlug,
paths,
+ verifyAvailability,
onLog,
signal,
}: CleanAudioOptions): Promise<CleanAudioResult> {
@@ -41,6 +62,8 @@ export async function cleanAudioFromTranscribed({
cleanedDirs: 0,
removedFiles: 0,
skipped: 0,
+ pinned: 0,
+ unverified: 0,
prunedSavedVideos: 0,
prunedBytes: 0,
};
@@ -58,40 +81,73 @@ export async function cleanAudioFromTranscribed({
let cleanedDirs = 0;
let removedFiles = 0;
- let skipped = 0;
+ let protectedCount = 0;
+ // --- Discover -------------------------------------------------------------
+ // Nothing is deleted in this pass. Protected videos are filtered out here, so
+ // the availability gate never spends a spawn on a video we would keep anyway.
+ const candidates: CleanCandidate[] = [];
for (const id of dirs) {
if (signal?.aborted) break;
const videoDir = path.join(dataDir, id);
const entries = await readdir(videoDir).catch(() => [] as string[]);
if (!entries.includes("transcript.json")) continue;
- const audioFiles = entries.filter(
- (e) =>
- e.startsWith("audio.") &&
- !e.includes(".tmp-") &&
- !e.endsWith(".info.json") &&
- !e.endsWith(".part"),
- );
+ const audioFiles = audioFilesToRemove(entries);
if (audioFiles.length === 0) continue;
if (keptIds.has(id)) {
log(`Skipped ${id} (in keep-latest window)`);
- skipped++;
+ protectedCount++;
continue;
}
if (await isDoNotClean(videoDir)) {
log(`Skipped ${id} (marked do not clean)`);
- skipped++;
+ protectedCount++;
continue;
}
- for (const f of audioFiles) {
- await remove(path.join(videoDir, f));
- log(`Removed ${id}/${f}`);
+ candidates.push({ id, videoDir, audioFiles });
+ }
+
+ // --- Verify ---------------------------------------------------------------
+ const verify = verifyAvailability ?? getSettings().verifyAvailabilityBeforeClean;
+ let excluded = new Set<string>();
+ let pinned = 0;
+ let unverified = 0;
+ if (verify && candidates.length > 0 && !signal?.aborted) {
+ const verdicts = await verifyBeforeClean({
+ channelSlug,
+ paths,
+ candidateIds: candidates.map((c) => c.id),
+ onLog: log,
+ signal,
+ });
+ excluded = excludedIds(verdicts);
+ pinned = verdicts.pinned.size;
+ unverified = verdicts.unverified.size;
+ // A video already pinned by someone else is protected, not newly pinned.
+ protectedCount += verdicts.alreadyGone.size;
+ }
+
+ // --- Delete ---------------------------------------------------------------
+ for (const candidate of candidates) {
+ if (signal?.aborted) break;
+ if (excluded.has(candidate.id)) continue;
+ for (const f of candidate.audioFiles) {
+ await remove(path.join(candidate.videoDir, f));
+ log(`Removed ${candidate.id}/${f}`);
removedFiles++;
}
cleanedDirs++;
}
- const skippedNote = skipped > 0 ? ` Skipped ${skipped} (do not clean).` : "";
+ const skipped = protectedCount + pinned + unverified;
+ // Keep the historical wording when the gate had nothing to report, and break
+ // the total down only when it would otherwise be misleading.
+ const skippedNote =
+ skipped === 0
+ ? ""
+ : pinned === 0 && unverified === 0
+ ? ` Skipped ${skipped} (do not clean).`
+ : ` Skipped ${skipped} (${protectedCount} protected, ${pinned} gone-from-source pinned, ${unverified} unverified).`;
log(
`Cleaned ${removedFiles} audio file(s) from ${cleanedDirs} of ${dirs.length} video dir(s).${skippedNote}`,
);
@@ -105,6 +161,8 @@ export async function cleanAudioFromTranscribed({
cleanedDirs,
removedFiles,
skipped,
+ pinned,
+ unverified,
prunedSavedVideos: prune.pruned,
prunedBytes: prune.bytesFreed,
};
diff --git a/common/controller/verifyBeforeClean.test.ts b/common/controller/verifyBeforeClean.test.ts
@@ -0,0 +1,177 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import type { Availability } from "../lib/availability";
+import { excludedIds, verifyBeforeClean } from "./verifyBeforeClean";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test common/controller/verifyBeforeClean.test.ts
+//
+// Covers the tiers that need no network plus both failure policies — the two
+// cases that decide whether an irreversible delete happens on incomplete
+// information, and the ones an e2e run can't easily provoke. Tier B/C happy
+// paths are covered end-to-end in editor/e2e/pre-clean-availability.spec.ts.
+
+// getSettings() reads getPaths().settingsFile; point it at a path that does not
+// exist so these tests can never read the developer's real settings.json.
+process.env.SETTINGS_FILE = path.join(tmpdir(), "ttb-vbc-no-such-settings.json");
+
+async function withPaths(fn: (paths: Paths) => Promise<void>): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-vbc-"));
+ const paths = {
+ channelsDir: path.join(dir, "channels"),
+ // Deliberately nonexistent: any yt-dlp spawn fails instantly, so a test
+ // that expects "no probe" cannot pass by accident.
+ ytdlpBin: path.join(dir, "no-such-ytdlp"),
+ } as Paths;
+ try {
+ await fn(paths);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+async function seedConfig(
+ paths: Paths,
+ slug: string,
+ config: Record<string, unknown>,
+): Promise<void> {
+ const dir = path.join(paths.channelsDir, slug);
+ await mkdir(dir, { recursive: true });
+ await writeFile(path.join(dir, "config.json"), JSON.stringify(config));
+}
+
+async function seedVideo(
+ paths: Paths,
+ slug: string,
+ id: string,
+ opts: { availability?: Availability; doNotCleanNote?: string } = {},
+): Promise<string> {
+ const dir = path.join(paths.channelsDir, slug, "data", id);
+ await mkdir(dir, { recursive: true });
+ if (opts.availability) {
+ await writeFile(
+ path.join(dir, "availability.json"),
+ JSON.stringify({
+ checkedAt: new Date().toISOString(),
+ availability: opts.availability,
+ }),
+ );
+ }
+ if (opts.doNotCleanNote !== undefined) {
+ await writeFile(
+ path.join(dir, "do-not-clean.json"),
+ JSON.stringify({
+ setAt: new Date().toISOString(),
+ note: opts.doNotCleanNote,
+ }),
+ );
+ }
+ return dir;
+}
+
+async function readNote(videoDir: string): Promise<string | undefined> {
+ const raw = await readFile(path.join(videoDir, "do-not-clean.json"), "utf8");
+ return (JSON.parse(raw) as { note?: string }).note;
+}
+
+test("tier A pins an already-known-gone video without probing the source", async () => {
+ await withPaths(async (paths) => {
+ await seedConfig(paths, "ch", {
+ handling: "youtube",
+ url: "https://www.youtube.com/@ch/videos",
+ });
+ const dir = await seedVideo(paths, "ch", "gone1", {
+ availability: "deleted",
+ });
+
+ const verdicts = await verifyBeforeClean({
+ channelSlug: "ch",
+ paths,
+ candidateIds: ["gone1"],
+ onLog: () => {},
+ });
+
+ // Resolved from the cached sidecar, so the run returns before tier B — with
+ // a bogus ytdlpBin, any spawn would have surfaced as unverified instead.
+ assert.equal(verdicts.pinned.get("gone1"), "deleted");
+ assert.equal(verdicts.unverified.size, 0);
+ assert.equal(verdicts.alreadyGone.size, 0);
+ assert.ok((await readNote(dir))?.includes("pre-clean check"));
+ assert.deepEqual([...excludedIds(verdicts)], ["gone1"]);
+ });
+});
+
+test("a hand-written do-not-clean note survives a re-pin", async () => {
+ await withPaths(async (paths) => {
+ await seedConfig(paths, "ch", {
+ handling: "youtube",
+ url: "https://www.youtube.com/@ch/videos",
+ });
+ const dir = await seedVideo(paths, "ch", "gone2", {
+ availability: "private",
+ doNotCleanNote: "keep this, I need it",
+ });
+
+ const verdicts = await verifyBeforeClean({
+ channelSlug: "ch",
+ paths,
+ candidateIds: ["gone2"],
+ onLog: () => {},
+ });
+
+ assert.equal(verdicts.pinned.size, 0);
+ assert.ok(verdicts.alreadyGone.has("gone2"));
+ assert.equal(await readNote(dir), "keep this, I need it");
+ // Still excluded from the delete — protected, just not newly pinned.
+ assert.ok(excludedIds(verdicts).has("gone2"));
+ });
+});
+
+test("a channel with no URL fails OPEN: candidates stay cleanable", async () => {
+ await withPaths(async (paths) => {
+ // No `url` — structurally unverifiable, and no setting toggle should be
+ // needed to clean an imported or hand-built archive.
+ await seedConfig(paths, "ch", { handling: "transcribe" });
+ await seedVideo(paths, "ch", "live1");
+
+ const verdicts = await verifyBeforeClean({
+ channelSlug: "ch",
+ paths,
+ candidateIds: ["live1"],
+ onLog: () => {},
+ });
+
+ assert.equal(verdicts.pinned.size, 0);
+ assert.equal(verdicts.unverified.size, 0);
+ assert.equal(excludedIds(verdicts).size, 0);
+ });
+});
+
+test("an unreachable source fails CLOSED: nothing is cleared to delete", async () => {
+ await withPaths(async (paths) => {
+ // A URL is configured, so this channel IS checkable — and yt-dlp is absent,
+ // so we get no answer. That must never read as "still available".
+ await seedConfig(paths, "ch", {
+ handling: "youtube",
+ url: "https://www.youtube.com/@ch/videos",
+ });
+ await seedVideo(paths, "ch", "live1");
+ await seedVideo(paths, "ch", "live2");
+
+ const verdicts = await verifyBeforeClean({
+ channelSlug: "ch",
+ paths,
+ candidateIds: ["live1", "live2"],
+ onLog: () => {},
+ });
+
+ assert.equal(verdicts.pinned.size, 0);
+ // No marker is written for an inconclusive result — the next sweep retries.
+ assert.equal(verdicts.unverified.size, 2);
+ assert.equal(excludedIds(verdicts).size, 2);
+ });
+});
diff --git a/common/controller/verifyBeforeClean.ts b/common/controller/verifyBeforeClean.ts
@@ -0,0 +1,211 @@
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import { isPermanentlyGone, type Availability } from "../lib/availability";
+import { resolveEffectiveAvailability } from "../lib/availability-server";
+import { loadDoNotClean, setDoNotClean } from "../lib/doNotClean-server";
+import { readChannelConfig } from "./channels";
+import { runAvailabilityCheck } from "./checkAvailability";
+import { runQuickAvailabilityCheck } from "./quickAvailabilityCheck";
+
+// Availability gate for the transcribed-audio cleanup sweep.
+//
+// Cleaning deletes a video's source audio irreversibly (no trash, no undo), so
+// a video that has since been removed / privated / put behind a membership must
+// NOT be cleaned: our copy has become the only copy. This module decides, per
+// candidate, whether the sweep is allowed to delete.
+//
+// Three tiers, cheapest first, so a typical run costs ONE yt-dlp spawn per
+// channel plus a handful for suspects — not one spawn per cleanable video:
+//
+// A. cached verdict resolveEffectiveAvailability() 0 spawns
+// Already-known-gone videos are pinned without touching the network. This
+// is also the only tier that catches members_only: a members-only video
+// stays listed in its channel's flat playlist, so tier B never flags it.
+// B. fresh listing runQuickAvailabilityCheck() 1 spawn/channel
+// Candidates absent from the channel's current flat playlist are suspects.
+// C. confirm suspects runAvailabilityCheck(onlyIds) 1 spawn/suspect
+// Resolves deleted/private (pin) from unlisted (clean — an unlisted video
+// legitimately drops out of the listing but still resolves by URL).
+//
+// Anything the gate cannot resolve is `unverified`: skipped, NO marker written,
+// retried on the next sweep. The sweep never deletes on incomplete information.
+
+export type CleanVerdicts = {
+ // Newly marked do-not-clean this run (gone from source).
+ pinned: Map<string, Availability>;
+ // Gone from source but a marker was already present — left untouched so a
+ // hand-written note survives. Structurally rare: the sweep filters marked
+ // videos out before the gate runs.
+ alreadyGone: Set<string>;
+ // Probe inconclusive (error / needs_auth / never resolved). Skip, no marker.
+ unverified: Set<string>;
+};
+
+export type VerifyBeforeCleanOptions = {
+ channelSlug: string;
+ paths: Paths;
+ candidateIds: string[];
+ onLog?: (msg: string) => void;
+ signal?: AbortSignal;
+};
+
+function emptyVerdicts(): CleanVerdicts {
+ return { pinned: new Map(), alreadyGone: new Set(), unverified: new Set() };
+}
+
+// Every id the sweep must NOT delete, across all three verdict kinds.
+export function excludedIds(verdicts: CleanVerdicts): Set<string> {
+ return new Set([
+ ...verdicts.pinned.keys(),
+ ...verdicts.alreadyGone,
+ ...verdicts.unverified,
+ ]);
+}
+
+export async function verifyBeforeClean({
+ channelSlug,
+ paths,
+ candidateIds,
+ onLog,
+ signal,
+}: VerifyBeforeCleanOptions): Promise<CleanVerdicts> {
+ const log = onLog ?? ((m: string) => console.log(m));
+ const verdicts = emptyVerdicts();
+ if (candidateIds.length === 0) return verdicts;
+
+ const dataDir = path.join(paths.channelsDir, channelSlug, "data");
+
+ // Pin a video as do-not-clean, preserving any marker (and note) already there.
+ const pin = async (id: string, availability: Availability): Promise<void> => {
+ const videoDir = path.join(dataDir, id);
+ if (await loadDoNotClean(videoDir)) {
+ verdicts.alreadyGone.add(id);
+ return;
+ }
+ await setDoNotClean(
+ videoDir,
+ true,
+ `deleted from source (pre-clean check): ${availability}`,
+ );
+ verdicts.pinned.set(id, availability);
+ log(`Pinned ${id} as do-not-clean (${availability}) — gone from source.`);
+ };
+
+ // --- Tier A: cached verdicts, no spawns -----------------------------------
+ const remaining: string[] = [];
+ for (const id of candidateIds) {
+ if (signal?.aborted) return verdicts;
+ const availability = await resolveEffectiveAvailability(
+ path.join(dataDir, id),
+ );
+ if (isPermanentlyGone(availability)) {
+ await pin(id, availability);
+ continue;
+ }
+ remaining.push(id);
+ }
+ if (remaining.length === 0 || signal?.aborted) return verdicts;
+
+ // --- Tier B: one flat-playlist spawn --------------------------------------
+ // A channel with no URL is structurally unverifiable — there is no source to
+ // diff against and there never will be. Fail OPEN: cleaning an imported or
+ // hand-built archive must not require flipping a global setting off.
+ const config = await readChannelConfig(paths, channelSlug);
+ if (!config?.url) {
+ log(
+ `Verify before clean: ${channelSlug} has no URL — no source to check against; cleaning the remaining ${remaining.length} candidate(s) unverified.`,
+ );
+ return verdicts;
+ }
+
+ // Ids known on disk but ABSENT from the channel's current flat playlist.
+ let maybeMissing: Set<string>;
+ try {
+ const quick = await runQuickAvailabilityCheck({
+ channelSlug,
+ paths,
+ onLog: log,
+ // runQuickAvailabilityCheck requires a signal; the sweep's is optional.
+ signal: signal ?? new AbortController().signal,
+ });
+ maybeMissing = new Set(quick.ids);
+ } catch (err) {
+ // We know this channel is checkable and we got no answer (network, rate
+ // limit, extractor break). Fail CLOSED — decision #4 at channel granularity.
+ const reason = (err as Error)?.message ?? String(err);
+ for (const id of remaining) verdicts.unverified.add(id);
+ log(
+ `Verify before clean: could not verify availability (${reason}) — nothing cleaned this run.`,
+ );
+ return verdicts;
+ }
+
+ const suspects = remaining.filter((id) => maybeMissing.has(id));
+ if (suspects.length === 0) {
+ log(
+ `Verify before clean: all ${remaining.length} candidate(s) still listed on the source.`,
+ );
+ return verdicts;
+ }
+ if (signal?.aborted) return verdicts;
+
+ // --- Tier C: one spawn per suspect ----------------------------------------
+ log(
+ `Verify before clean: ${suspects.length} candidate(s) missing from the fresh listing — confirming individually…`,
+ );
+ try {
+ await runAvailabilityCheck({
+ channelSlug,
+ paths,
+ mode: "recheck-all",
+ onlyIds: suspects,
+ // Tier A already pinned everything known-gone; belt and braces.
+ skipExpectedAbsent: true,
+ // `suspects` is the authority — without this a saved shard-availability
+ // slice on disk would replace it wholesale (see CheckAvailabilityOpts).
+ ignoreShard: true,
+ // Deliberately 1. The clean-audio job runs on the CHANNEL queue while the
+ // availability actions run on the PLATFORM queue precisely so probes share
+ // the per-source rate-limit budget with downloads. Probing from in here
+ // bypasses that serialization, so keep the extra load to a trickle — do
+ // NOT "optimize" this upward.
+ concurrency: 1,
+ onLog: log,
+ signal,
+ });
+ } catch (err) {
+ const reason = (err as Error)?.message ?? String(err);
+ for (const id of suspects) verdicts.unverified.add(id);
+ log(
+ `Verify before clean: confirmation probe failed (${reason}) — leaving ${suspects.length} candidate(s) unverified.`,
+ );
+ return verdicts;
+ }
+
+ for (const id of suspects) {
+ if (signal?.aborted) break;
+ const availability = await resolveEffectiveAvailability(
+ path.join(dataDir, id),
+ );
+ if (isPermanentlyGone(availability)) {
+ await pin(id, availability);
+ continue;
+ }
+ // null = the probe never resolved this video (no webpage_url to probe, or
+ // it was skipped). That is not "available" — it is "we don't know", which
+ // is exactly the case this gate exists to refuse to delete on.
+ if (availability === null) {
+ verdicts.unverified.add(id);
+ log(`Left ${id} unverified (no availability could be resolved).`);
+ continue;
+ }
+ if (availability === "error" || availability === "needs_auth") {
+ verdicts.unverified.add(id);
+ log(`Left ${id} unverified (${availability}) — will retry next sweep.`);
+ continue;
+ }
+ // public / unlisted → still there, safe to clean.
+ }
+
+ return verdicts;
+}
diff --git a/common/lib/availability.ts b/common/lib/availability.ts
@@ -31,6 +31,18 @@ export const EXCLUDED_FROM_DOWNLOAD: ReadonlyArray<Availability> = [
"private",
];
+// The single home for "this video is gone from the source for good". Used by
+// the keep-latest deletion sweep (checkKeptDeleted) and by the pre-clean
+// availability gate (verifyBeforeClean) to decide what to pin as do-not-clean.
+// A null availability means "we have never resolved this video" — NOT gone.
+// Narrows on the way through, so a caller that pins the video can pass the
+// availability straight into the marker note without re-asserting the type.
+export function isPermanentlyGone(
+ a: Availability | null | undefined,
+): a is Availability {
+ return a != null && EXCLUDED_FROM_DOWNLOAD.includes(a);
+}
+
// Availability classes a cookie (auth) retry can potentially recover — the
// gate for the managed downloader's auth-retry attempts and the membership
// rule for the per-channel "Needs cookies" snapshot bucket. Note the overlap
diff --git a/common/lib/settings.ts b/common/lib/settings.ts
@@ -135,6 +135,13 @@ export type SiteSettings = {
// manual download-bearing pipeline actions return a "Downloads are paused"
// result. Enumeration/store-playlist stay allowed. Default false.
downloadsPaused: boolean;
+ // Whether the transcribed-audio cleanup sweep checks each candidate is still
+ // available upstream before deleting its audio, pinning (do-not-clean) any
+ // video found permanently gone. The delete is irreversible and a gone video's
+ // audio is irreplaceable, so this defaults to true. Turn it off for an offline
+ // or URL-less setup, where the check can never resolve and cleanup would
+ // otherwise never delete anything. See verifyBeforeClean.ts.
+ verifyAvailabilityBeforeClean: boolean;
// Whether site builds generate downloadable transcript/live-chat archive zips
// (into public/archives, linked on the Downloads page). Global default; a site
// can opt out via site.json `archives: false`, and a single build can skip via
@@ -724,6 +731,7 @@ function defaults(): SiteSettings {
skipLiveDownloads: true,
transcriptionsPaused: false,
downloadsPaused: false,
+ verifyAvailabilityBeforeClean: true,
buildArchives: true,
archiveStorage: { bucket: "", publicBaseUrl: "" },
reportDebouncePreset: DEFAULT_REPORT_DEBOUNCE_PRESET,
@@ -928,6 +936,9 @@ export function getSettings(): SiteSettings {
if (typeof merged.downloadsPaused !== "boolean") {
merged.downloadsPaused = false;
}
+ if (typeof merged.verifyAvailabilityBeforeClean !== "boolean") {
+ merged.verifyAvailabilityBeforeClean = true;
+ }
if (typeof merged.buildArchives !== "boolean") {
merged.buildArchives = true;
}
@@ -1121,6 +1132,8 @@ export async function writeSettings(next: SiteSettings): Promise<void> {
skipLiveDownloads: next.skipLiveDownloads !== false,
transcriptionsPaused: next.transcriptionsPaused === true,
downloadsPaused: next.downloadsPaused === true,
+ verifyAvailabilityBeforeClean:
+ next.verifyAvailabilityBeforeClean !== false,
buildArchives: next.buildArchives !== false,
archiveStorage: {
bucket:
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,7 @@
# Changelog
## [Unreleased]
+- **Cleaning audio now checks the video still exists upstream, and keeps it forever if it doesn't.** The transcribed-audio sweep hard-deletes a video's `audio.*` files once whisper has produced a transcript — `remove()`, no trash, no undo — and nothing had ever asked whether the video was still *there*. So a video YouTube had since removed, privated, or put behind a membership, sitting outside the keep-latest window, got its source audio deleted precisely when that local copy had become the only copy. Before deleting anything, the sweep now resolves each candidate's availability and writes a `do-not-clean.json` marker on any video found permanently gone (`deleted` / `private` / `members_only` — the same rule the keep-latest deletion pass uses, now shared as `isPermanentlyGone`), protecting it from this and every future sweep. The check is **cheap-first, not one probe per video**: a cached availability verdict costs nothing and is the only tier that catches `members_only` (a members-only video stays listed in its channel's playlist, so a listing diff can never flag it); then **one** flat-playlist call per channel narrows the field to candidates that have dropped out of the listing; only those few get a per-video probe, which is also what distinguishes a deleted video from an *unlisted* one that legitimately left the listing and is still fetchable by URL. Anything the check cannot resolve — a probe error, an age-gate, a video with no URL to probe — is **left alone with no marker written** and retried next run: the sweep never deletes on incomplete information, and a rate-limited or offline source therefore cleans nothing rather than cleaning wrongly. The summary line breaks the total down (`Skipped 4 (0 protected, 3 gone-from-source pinned, 1 unverified)`) whenever the check acted. On by default; **Check availability before cleaning audio** in Settings turns it off for an offline setup or channels with no URL, where the check can never resolve and cleanup would otherwise stop deleting anything. Two related fixes ride along: the sweep now shares `isRealAudioFile` with the rest of the app instead of its own hand-rolled filter, so it no longer deletes the `audio.live_chat.json` sidecar or the `.part.good`/`.part.testing` audio-check snapshots (which the reclaim estimate never counted, so the two had quietly drifted); and `runAvailabilityCheck` gains `ignoreShard`, because a saved shard slice on disk would otherwise replace an explicit `onlyIds` list wholesale. Scoped to the primary sweep only — the wrong-format, extra-format and auto-sub purges are unchanged, as are the explicit per-video deletes, which still ignore markers deliberately. See `common/controller/verifyBeforeClean.ts`, `common/controller/cleanAudioFromTranscribed.ts`, `common/lib/availability.ts`, and `editor/e2e/pre-clean-availability.spec.ts`.
- **The monitor widget can now reclaim disk, not just report it.** The widget's cleanable-data strip showed a single global number ("4.2 GB reclaimable") with nothing to act on — reclaiming it meant leaving the widget for `/cleanup` or `/actionable`. A new opt-in **"Needs cleaning"** section (URL flag `cleanlist=1`, plus a **Channels needing cleanup** checkbox in the builder and the in-widget gear) lists the channels actually holding that audio, each with its reclaim estimate (`⌫ 2.5 MB`, the video count in the tooltip), capped at 6 channels with a `+N more` line like the needs-work list. With `controls=1` each row gains the same per-channel **Clean audio** button as the `/actionable` page — the existing `window.confirm` still guards the delete — so a pinned interactive widget clears disk pressure the way it already clears a download backlog. This is also the first surface on which a channel that is *fully downloaded and transcribed* but still holding reclaimable audio is actionable: the needs-work list is fed by a backlog route with a download/transcribe precondition, so such a channel never appeared there. It costs no extra polling — the per-channel rows come from the same `/api/widget/cleanable` snapshot read that already backed the total, and the total is now a sum over those rows so the section and the strip above it can't disagree. The dashboard's needs-work panel and its shared route are untouched. See `editor/app/cleanup/lib/loadCleanup.ts` (`cleanableChannels`), `editor/app/api/widget/cleanable/route.ts`, `editor/app/widget/{lib/config.ts,components/{MonitorWidget,WidgetConfigForm}.tsx}`, and `editor/e2e/widget.spec.ts`.
- **A corpus-wide digest backfill can now be started, left alone, and watched.** The digest layer could generate, but only one channel at a time from that channel's own page — a full-archive pass meant 63 manual launches, and a server restart silently ended it with nothing to say so. There is now a **Start Digest Sweep** control on the dashboard that walks every channel in turn, **heaviest first by remaining audio-hours** (cost is audio, not videos: one VOD channel outweighs every duplicate mirror in the archive combined), and it **survives a restart** — the sweep is re-armed at boot the way the auto-download and auto-transcribe runners already were. It stores no work-list, so a resumed sweep re-does nothing: what still needs generating is re-derived from disk every time, which also means a transcript that finishes mid-sweep, or a duplicate cluster you confirm, is simply picked up on the next pass. A separate **Pause Digests** control holds a running sweep at zero without ending it (the pause flag has existed since the digest layer shipped and nothing could set it). **The digest lane now yields the GPU to transcription**: the two were deliberately on separate queues so they wouldn't serialise, whose unintended consequence was that the model and whisper competed for the same card — measured at 90 seconds per audio-hour against the 27 an idle machine managed. While transcription is working the digest lane steps aside and resumes when the card is free; it can be turned off in Settings. Coverage is now visible — a digest instrument on the dashboard with the corpus percentage (to two decimals, because rounding 0.13% up to 1% flatters an 80-day job), a **No digest** column on the channels table, and a per-channel count on the needs-work rows. Progress bars also **work during a regeneration** for the first time: they re-counted digest files from disk, and a regenerated digest is rewritten in place, so a job that was working sat at 0% for its whole run. Time-remaining estimates for digest work are now computed in **seconds per audio-hour** rather than by averaging videos, which for this archive is wrong by more than an order of magnitude between a VOD channel and a shorts channel. New `common/bin/digest-plan.ts` prices the whole backfill in audio-hours before you commit hardware to it.
- **Duplicate clusters awaiting review can finally be confirmed or rejected.** A cluster whose evidence is only a shared title and runtime shares no AI digests and reaches no built site until a human confirms it — and the code to record that confirmation existed, complete, with **no way to reach it from anywhere in the app**. Roughly 166 clusters were therefore stuck permanently. `/actionable` duplicate cards now carry **Confirm** / **Not a duplicate** / **Undo**, badges that show what has already been decided, and confirming immediately shares the canonical copy's digest to any aligned member rather than making you wait for its next sweep.
diff --git a/editor/app/settings/actions.ts b/editor/app/settings/actions.ts
@@ -64,6 +64,8 @@ export async function saveSettingsAction(
const inlineTranscribeOnFallback =
formData.get("inlineTranscribeOnFallback") === "on";
const skipLiveDownloads = formData.get("skipLiveDownloads") === "on";
+ const verifyAvailabilityBeforeClean =
+ formData.get("verifyAvailabilityBeforeClean") === "on";
const buildArchives = formData.get("buildArchives") === "on";
const archiveStorage = {
bucket: String(formData.get("archiveStorageBucket") ?? "").trim(),
@@ -322,6 +324,7 @@ export async function saveSettingsAction(
// controls own them). Preserve the current values on an unrelated save.
transcriptionsPaused: getSettings().transcriptionsPaused,
downloadsPaused: getSettings().downloadsPaused,
+ verifyAvailabilityBeforeClean,
buildArchives,
archiveStorage,
reportDebouncePreset,
diff --git a/editor/app/settings/components/SettingsForm.tsx b/editor/app/settings/components/SettingsForm.tsx
@@ -209,6 +209,30 @@ export function SettingsForm({ initial, apps, digestApps }: Props) {
<label className="flex items-start gap-2 text-sm">
<input
type="checkbox"
+ name="verifyAvailabilityBeforeClean"
+ defaultChecked={initial.verifyAvailabilityBeforeClean}
+ className="mt-1"
+ />
+ <span className="flex flex-col gap-1">
+ <span className="font-medium">
+ Check availability before cleaning audio
+ </span>
+ <span className="text-xs text-muted-foreground">
+ Before the transcribed-audio cleanup deletes anything, check each
+ video is still available at the source. A video that has been
+ removed, made private, or put behind a membership is marked
+ “do not clean” and kept — our copy is now the only copy.
+ Anything the check cannot resolve is left alone and retried next
+ run. Costs one extra yt-dlp call per channel, plus one per video
+ missing from the channel listing. On by default; turn it off for an
+ offline setup or channels without a URL, where the check can never
+ resolve and cleanup would stop deleting anything.
+ </span>
+ </span>
+ </label>
+ <label className="flex items-start gap-2 text-sm">
+ <input
+ type="checkbox"
name="buildArchives"
defaultChecked={initial.buildArchives}
className="mt-1"
diff --git a/editor/e2e/fixtures/test-settings.default.json b/editor/e2e/fixtures/test-settings.default.json
@@ -2,5 +2,6 @@
"adminTitle": "Test Admin",
"maxTranscriptPageBytes": 8388608,
"sleepBetweenDownloadsSeconds": 0,
- "minFreeDiskGB": 0
+ "minFreeDiskGB": 0,
+ "verifyAvailabilityBeforeClean": false
}
diff --git a/editor/e2e/pre-clean-availability.spec.ts b/editor/e2e/pre-clean-availability.spec.ts
@@ -0,0 +1,207 @@
+import path, { dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { readFile, writeFile } from "node:fs/promises";
+import { test, expect } from "@playwright/test";
+import {
+ pathExists,
+ readJson,
+ resetData,
+ resolvePath,
+ writeSettings,
+} from "./helpers";
+import { baseUrl } from "./baseUrl";
+
+// The transcribed-audio sweep deletes source audio irreversibly, so it first
+// checks each candidate is still available upstream and pins (do-not-clean)
+// anything permanently gone. These tests pin down all three tiers of that gate
+// plus the global opt-out.
+//
+// The availability-baseline fixture is purpose-built for it: each data dir's
+// metadata.info.json carries a webpage_url whose id doubles as the sentinel that
+// fake-ytdlp matches on to produce a deterministic availability bucket.
+
+const here = dirname(fileURLToPath(import.meta.url));
+const CHANNEL = "availability-test";
+const FIXTURE = "availability-baseline";
+const channelRoot = `test-transcripts/channels/${CHANNEL}`;
+
+function dataRel(id: string, file: string): string {
+ return `${channelRoot}/data/${id}/${file}`;
+}
+
+async function defaultSettings(): Promise<Record<string, unknown>> {
+ const raw = await readFile(
+ path.join(here, "fixtures", "test-settings.default.json"),
+ "utf8",
+ );
+ return JSON.parse(raw) as Record<string, unknown>;
+}
+
+// Make a video cleanable: a whisper transcript plus audio still on disk.
+async function seedCleanable(id: string): Promise<void> {
+ await writeFile(
+ resolvePath(dataRel(id, "transcript.json")),
+ '{"transcription":[]}\n',
+ );
+ await writeFile(resolvePath(dataRel(id, "audio.m4a")), "fake-audio-bytes\n");
+}
+
+// Write the sidecar controlling which ids fake-ytdlp's flat-playlist branch
+// emits. It lives in the channel cwd, where availability spawns run.
+async function setFreshPlaylist(ids: string[]): Promise<void> {
+ await writeFile(
+ resolvePath(`${channelRoot}/.fake-ytdlp-flat-playlist.json`),
+ JSON.stringify({ ids }),
+ );
+}
+
+async function readInvocations(): Promise<string> {
+ try {
+ return await readFile(
+ resolvePath(`${channelRoot}/fake-ytdlp.invocations`),
+ "utf8",
+ );
+ } catch {
+ return "";
+ }
+}
+
+// The five videos each test makes cleanable. vidpublic2 stays untranscribed.
+const CLEANABLE = [
+ "vidpublic1",
+ "viddeleted1",
+ "vidprivate1",
+ "vidneedsauth1",
+ "vidmembers1",
+];
+
+async function seedAll(): Promise<void> {
+ for (const id of CLEANABLE) await seedCleanable(id);
+ // vidmembers1 is already known members_only. A members-only video stays
+ // listed in its channel's flat playlist, so tier B never flags it — tier A's
+ // cached verdict is the only thing that can catch it, with zero spawns.
+ await writeFile(
+ resolvePath(dataRel("vidmembers1", "availability.json")),
+ JSON.stringify({
+ checkedAt: new Date().toISOString(),
+ availability: "members_only",
+ }),
+ );
+ await fetch(`${baseUrl}/api/test/invalidate-cache`).catch(() => {});
+}
+
+async function runCleanAudio(page: import("@playwright/test").Page) {
+ await page.goto(`/channels/${CHANNEL}`);
+ await page.getByRole("button", { name: "Cleanup stage summary" }).click();
+ const button = page.getByRole("button", { name: "Clean audio", exact: true });
+ await expect(button).toBeEnabled();
+ await button.click();
+ return page.getByLabel("Clean audio output");
+}
+
+test("cleanup pins videos gone from the source and leaves unresolvable ones alone", async ({
+ page,
+}) => {
+ test.setTimeout(120_000);
+ await resetData(FIXTURE);
+ await seedAll();
+ await writeSettings({
+ ...(await defaultSettings()),
+ verifyAvailabilityBeforeClean: true,
+ });
+ // The fresh listing keeps the public videos and the members-only one; the
+ // deleted / private / age-gated videos have dropped out of it.
+ await setFreshPlaylist(["vidpublic1", "vidpublic2", "vidmembers1"]);
+
+ const log = await runCleanAudio(page);
+ await expect(log).toContainText(
+ "Cleaned 1 audio file(s) from 1 of 6 video dir(s). Skipped 4 (0 protected, 3 gone-from-source pinned, 1 unverified).",
+ { timeout: 60_000 },
+ );
+
+ // Still listed and still public — cleaned.
+ expect(await pathExists(dataRel("vidpublic1", "audio.m4a"))).toBe(false);
+ expect(await pathExists(dataRel("vidpublic1", "transcript.json"))).toBe(true);
+
+ // Gone from the source — audio kept and pinned, with a note saying why.
+ for (const [id, availability] of [
+ ["viddeleted1", "deleted"],
+ ["vidprivate1", "private"],
+ ["vidmembers1", "members_only"],
+ ] as const) {
+ expect(await pathExists(dataRel(id, "audio.m4a"))).toBe(true);
+ const marker = await readJson<{ note?: string }>(
+ dataRel(id, "do-not-clean.json"),
+ );
+ expect(marker.note).toContain(availability);
+ expect(marker.note).toContain("pre-clean check");
+ }
+
+ // The fail-safe case: an age-gated video probes as needs_auth, which resolves
+ // nothing. Audio is kept but NO marker is written, so the next sweep retries.
+ expect(await pathExists(dataRel("vidneedsauth1", "audio.m4a"))).toBe(true);
+ expect(await pathExists(dataRel("vidneedsauth1", "do-not-clean.json"))).toBe(
+ false,
+ );
+
+ // Tier A resolved the members-only video from its cached availability, so it
+ // never cost a probe. The three suspects did.
+ const invocations = await readInvocations();
+ expect(invocations).not.toContain("dump-json:https://www.youtube.com/watch?v=vidmembers1");
+ expect(invocations).toContain("dump-json:https://www.youtube.com/watch?v=viddeleted1");
+ expect(invocations).toContain("dump-json:https://www.youtube.com/watch?v=vidprivate1");
+ expect(invocations).toContain("dump-json:https://www.youtube.com/watch?v=vidneedsauth1");
+});
+
+test("an unverified video is retried on the next sweep", async ({ page }) => {
+ test.setTimeout(120_000);
+ await resetData(FIXTURE);
+ await seedAll();
+ await writeSettings({
+ ...(await defaultSettings()),
+ verifyAvailabilityBeforeClean: true,
+ });
+ await setFreshPlaylist(["vidpublic1", "vidpublic2", "vidmembers1"]);
+
+ const log = await runCleanAudio(page);
+ await expect(log).toContainText("1 unverified", { timeout: 60_000 });
+
+ // Second sweep: the unpinned, unverified video is probed again (a pinned one
+ // would have been filtered out as do-not-clean before the gate even ran).
+ const secondLog = await runCleanAudio(page);
+ await expect(secondLog).toContainText("1 unverified", { timeout: 60_000 });
+
+ const probes = (await readInvocations())
+ .split("\n")
+ .filter((line) =>
+ line.startsWith("dump-json:https://www.youtube.com/watch?v=vidneedsauth1"),
+ );
+ expect(probes.length).toBe(2);
+ expect(await pathExists(dataRel("vidneedsauth1", "audio.m4a"))).toBe(true);
+});
+
+test("the availability check can be turned off", async ({ page }) => {
+ test.setTimeout(120_000);
+ await resetData(FIXTURE);
+ await seedAll();
+ await writeSettings({
+ ...(await defaultSettings()),
+ verifyAvailabilityBeforeClean: false,
+ });
+ await setFreshPlaylist(["vidpublic1", "vidpublic2", "vidmembers1"]);
+
+ const log = await runCleanAudio(page);
+ await expect(log).toContainText(
+ "Cleaned 5 audio file(s) from 5 of 6 video dir(s).",
+ { timeout: 60_000 },
+ );
+
+ // Everything cleanable is cleaned, including videos known to be gone, and no
+ // marker is written — the pre-check is the only thing that pins.
+ for (const id of CLEANABLE) {
+ expect(await pathExists(dataRel(id, "audio.m4a"))).toBe(false);
+ expect(await pathExists(dataRel(id, "do-not-clean.json"))).toBe(false);
+ }
+ // No probes at all: neither the flat-playlist listing nor any per-video probe.
+ expect(await readInvocations()).not.toContain("dump-json:");
+});
