commit 6779e4e3789b805fb9767ed10737cd0031ab6761
parent 564df959ba21fc38369e28de1d82c2dff707bcd4
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 11 Aug 2026 18:36:02 -0400
Say what is holding the audio, and what to run to get it back
/cleanup answered one question — how much space can I reclaim right now — and
said nothing about the rest of the disk. The sweep skips videos for four
different reasons and reported them only as a line in a job log after the fact:
no bytes attached, nothing ranked, nothing visible before the run.
The snapshot now attributes every audio byte to the FIRST gate it hits, in
cleanAudioFromTranscribed's own order (no transcript -> keep-latest -> pin ->
awaiting diarization), so the five figures are an attribution and never overlap:
a pinned, undiarized video is counted once, under the pin. It costs no extra
I/O — audioSizes, the file listing, keptIds/doNotCleanIds, the availability
verdict and the per-kind backfill map are all already in hand in the per-video
loop. attributeAudioHold is pure and exported so the partition invariant
(four gates + transcribedWithAudio === totalAudioBytes) has a test.
The page draws that as a sieve: everything on disk enters at the top, each gate
siphons its share, and the remainder steps down to the reclaimable figure the
hero already printed. Below it, a release ledger ranks the channels holding the
most, split by what it costs to release — a run of a lane that is already weeks
deep, or a setting that frees the space the moment it changes.
And the part nobody could have known: the cleanup guard fires on
settings.diarization.enabled ALONE, but allBackfillKinds drops the diarization
kind unless segModel and embModel are both set, and the kind defers anything
over maxAudioHours. Audio in either state is held from cleaning permanently and
no queued job will ever release it. diarizationNeverClears is that number, drawn
hatched inside gate 4 with its own figure. What to do about it is left alone on
purpose: audio is the one input in this pipeline that cannot be regenerated.
Channel is the honest granularity throughout — per-video byte sizes survive
nowhere outside the report, and ranking videos would need the corpus walk
noCorpusWalkInRenderPaths.test.ts bans from render paths. The hero, the sidebar
badge and /actionable's Est. reclaim are untouched: the availability gate is
reported as an annotation on the remainder, not folded in as a fifth gate.
The ledger's "Not counted" rows link to the channel card rather than rendering a
second ChannelCleanupToggle, which would duplicate an accessible name that
cleanup-page.spec.ts clicks. Transcribe carries ids for the top channels per
gate and links to the channel's own bucket control past that, because the action
takes an id list and the payload has to be bounded.
Verified: tsc clean in editor/ and common/; common tests 683/683; editor build
clean (/cleanup still dynamic); cleanup-holds, cleanup-page, cleanup-actionable,
do-not-clean and pre-clean-availability green in one 12/12 sweep.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Diffstat:
9 files changed, 1408 insertions(+), 3 deletions(-)
diff --git a/common/controller/channelSnapshot.test.ts b/common/controller/channelSnapshot.test.ts
@@ -1,6 +1,12 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { digestWorkOf, foldBackfillEntry } from "./channelSnapshot";
+import {
+ attributeAudioHold,
+ diarizationWillNeverClear,
+ digestWorkOf,
+ emptyHeldAudio,
+ foldBackfillEntry,
+} from "./channelSnapshot";
import {
emptyBackfillCounts,
reachableBackfillWork,
@@ -151,3 +157,129 @@ test("digestWorkOf survives an entry written before `partial` and `eligible`", (
assert.equal(work.partial, 0);
assert.equal(work.present, null);
});
+
+// --- The audio-hold partition ------------------------------------------------
+//
+// The /cleanup sieve says what is holding each byte of audio, and its five
+// figures are only trustworthy as a PARTITION: a video that is both pinned and
+// undiarized must be counted once, under the pin, or the page reports a bigger
+// hold than exists. attributeAudioHold is where that rule lives, and it has to
+// stay in step with cleanAudioFromTranscribed's discover loop — which is a
+// cascade of `continue`s, so first-gate-wins is the whole of the semantics.
+
+type Vid = Parameters<typeof attributeAudioHold>[0] & { bytes: number };
+
+function video(over: Partial<Vid> = {}): Vid {
+ // A plain reclaimable video: transcribed, unprotected, already diarized.
+ return {
+ bytes: 100,
+ hasWhisper: true,
+ inKeepLatestWindow: false,
+ doNotClean: false,
+ diarizationGuardOn: true,
+ hasDiarization: true,
+ ...over,
+ };
+}
+
+// The same fold generateChannelSnapshot performs, over a synthetic corpus.
+function partition(videos: Vid[]) {
+ const held = emptyHeldAudio();
+ let total = 0;
+ let reclaimable = 0;
+ for (const v of videos) {
+ total += v.bytes;
+ const gate = attributeAudioHold(v);
+ if (gate === "reclaimable") reclaimable += v.bytes;
+ else held[gate] += v.bytes;
+ }
+ return { held, total, reclaimable };
+}
+
+test("every audio byte lands in exactly one gate", () => {
+ const { held, total, reclaimable } = partition([
+ video({ bytes: 2800, hasWhisper: false }),
+ video({ bytes: 184, inKeepLatestWindow: true }),
+ video({ bytes: 31, doNotClean: true }),
+ video({ bytes: 1100, hasDiarization: false }),
+ video({ bytes: 2300 }),
+ ]);
+ assert.deepEqual(held, {
+ noTranscript: 2800,
+ keepLatest: 184,
+ doNotClean: 31,
+ awaitingDiarization: 1100,
+ diarizationNeverClears: 0,
+ });
+ assert.equal(reclaimable, 2300);
+ // THE INVARIANT. This is what stops the sieve's remainder column from
+ // drifting away from the reclaimable figure the page already leads with.
+ const sumHeld =
+ held.noTranscript +
+ held.keepLatest +
+ held.doNotClean +
+ held.awaitingDiarization;
+ assert.equal(sumHeld + reclaimable, total);
+});
+
+test("first gate wins — a pinned, undiarized video is counted ONCE", () => {
+ // Under the PIN, because the sweep leaves at the first `continue` it hits.
+ // Counting it at both gates would inflate the hold and prescribe a diarize run
+ // that would not release it anyway.
+ const { held } = partition([
+ video({ bytes: 50, doNotClean: true, hasDiarization: false }),
+ ]);
+ assert.equal(held.doNotClean, 50);
+ assert.equal(held.awaitingDiarization, 0);
+});
+
+test("gate order matches the sweep: no-transcript beats every protection", () => {
+ const { held } = partition([
+ video({
+ bytes: 7,
+ hasWhisper: false,
+ inKeepLatestWindow: true,
+ doNotClean: true,
+ hasDiarization: false,
+ }),
+ ]);
+ assert.equal(held.noTranscript, 7);
+ assert.equal(held.keepLatest, 0);
+ assert.equal(held.doNotClean, 0);
+ assert.equal(held.awaitingDiarization, 0);
+});
+
+test("the diarization hold exists only while the guard is on", () => {
+ // The sweep reads settings.diarization.enabled and nothing else. With it off,
+ // an undiarized video is reclaimable TODAY — the estimate must say so.
+ assert.equal(
+ attributeAudioHold(
+ video({ diarizationGuardOn: false, hasDiarization: false }),
+ ),
+ "reclaimable",
+ );
+ assert.equal(
+ attributeAudioHold(
+ video({ diarizationGuardOn: true, hasDiarization: false }),
+ ),
+ "awaitingDiarization",
+ );
+});
+
+test("only the reachable diarization states will ever clear a hold", () => {
+ // The one number nothing else in the app can produce. `undefined` is the kind
+ // being disabled outright (models unconfigured): the guard still holds the
+ // audio, and no queued job will ever release it.
+ for (const state of ["missing", "stale", "partial"] as const) {
+ assert.equal(diarizationWillNeverClear(state), false);
+ }
+ for (const state of [
+ "deferred",
+ "missing-input",
+ "not-applicable",
+ "blocked",
+ undefined,
+ ] as const) {
+ assert.equal(diarizationWillNeverClear(state), true);
+ }
+});
diff --git a/common/controller/channelSnapshot.ts b/common/controller/channelSnapshot.ts
@@ -14,6 +14,7 @@ import {
AUTH_RETRY_CLASSES,
AVAILABILITY_VALUES,
EXCLUDED_FROM_DOWNLOAD,
+ isPermanentlyGone,
type Availability,
} from "../lib/availability";
import { resolveCookiePolicy } from "../lib/cookiePolicy";
@@ -32,6 +33,7 @@ import {
presentBackfillWork,
reachableBackfillWork,
DIGEST_KIND_ID,
+ DIARIZATION_KIND_ID,
type BackfillClassification,
type BackfillSnapshotEntry,
} from "../lib/backfillKinds";
@@ -255,8 +257,101 @@ export type ChannelSnapshot = {
// lack it; readers must default to 0.
foreignAudio?: number;
};
+ // Every audio.* byte this channel has on disk, held or not. The denominator
+ // the four gates below partition. Optional: older snapshots lack it; readers
+ // must default to 0 — and MUST render that as "—", not "0", because a zero
+ // here would claim a measurement nobody took.
+ totalAudioBytes?: number;
+ // WHY the audio that isn't reclaimable isn't reclaimable, in the sweep's own
+ // order (cleanAudioFromTranscribed's discover loop). A video leaves at the
+ // FIRST gate it hits, so these are an attribution and never overlapping sets:
+ //
+ // totalAudioBytes === noTranscript + keepLatest + doNotClean
+ // + awaitingDiarization + cleanupBytes.transcribedWithAudio
+ //
+ // pinned by channelSnapshot.test.ts, because the five numbers are only
+ // trustworthy as a partition — a double-count would read as a bigger hold.
+ heldAudioBytes?: HeldAudio;
+ // The same partition by VIDEO COUNT. Only videos that actually hold audio are
+ // counted: a dir with no audio.* file holds nothing at any gate.
+ heldAudioCounts?: HeldAudio;
+ // Of the RECLAIMABLE set, the bytes whose CACHED availability is already
+ // permanently-gone / needs_auth / error — what verifyBeforeClean is likely to
+ // refuse at sweep time. An annotation on the remainder, deliberately NOT a
+ // fifth gate: the real gate needs a live probe, and folding a guess into the
+ // hero figure would move the sidebar badge and /actionable's Est. reclaim too.
+ reclaimableAtRiskBytes?: number;
+};
+
+// The four holds, in sweep order. Same shape for bytes and for counts.
+export type HeldAudio = {
+ noTranscript: number;
+ keepLatest: number;
+ doNotClean: number;
+ awaitingDiarization: number;
+ // The SUBSET of awaitingDiarization that the diarize lane will never produce
+ // on its own: over maxAudioHours (`deferred`), no diarizable input, an
+ // untranscribable transcript the kind declines, or the kind reporting itself
+ // disabled because its models aren't configured. Running Diarize until the end
+ // of time does not move this number, and nothing else in the app says so.
+ diarizationNeverClears: number;
};
+export function emptyHeldAudio(): HeldAudio {
+ return {
+ noTranscript: 0,
+ keepLatest: 0,
+ doNotClean: 0,
+ awaitingDiarization: 0,
+ diarizationNeverClears: 0,
+ };
+}
+
+// Which gate a video's audio leaves the sweep at, or "reclaimable" if it
+// survives all four. Pure, exported, and tested — this is the one rule that
+// keeps the five figures from double-counting, and it has to stay in step with
+// cleanAudioFromTranscribed's discover loop by inspection, so it is written in
+// the same order with the same predicates.
+export type AudioHoldGate =
+ | "noTranscript"
+ | "keepLatest"
+ | "doNotClean"
+ | "awaitingDiarization"
+ | "reclaimable";
+
+export function attributeAudioHold(v: {
+ // entries.includes("transcript.json") — the sweep's gate, NOT isVideoTranscribed:
+ // an ASR-only video has an English VTT and is still uncleanable.
+ hasWhisper: boolean;
+ inKeepLatestWindow: boolean;
+ doNotClean: boolean;
+ // settings.diarization.enabled — the guard the sweep actually reads.
+ diarizationGuardOn: boolean;
+ hasDiarization: boolean;
+}): AudioHoldGate {
+ if (!v.hasWhisper) return "noTranscript";
+ if (v.inKeepLatestWindow) return "keepLatest";
+ if (v.doNotClean) return "doNotClean";
+ if (v.diarizationGuardOn && !v.hasDiarization) return "awaitingDiarization";
+ return "reclaimable";
+}
+
+// Will the diarize lane ever produce this video's sidecar? `undefined` means the
+// kind isn't enabled at all (allBackfillKinds dropped it — models unconfigured),
+// which is the case the guard cannot see: the sweep holds the audio on
+// settings.diarization.enabled alone, so the hold is permanent and silent.
+//
+// Only the reachable states clear: missing / stale / partial are what the lane
+// picks up. `deferred` (over the duration cap), `missing-input` and
+// `not-applicable` (an untranscribable transcript) are all holds nothing queued
+// will release.
+export function diarizationWillNeverClear(
+ state: BackfillClassification | undefined,
+): boolean {
+ if (state === undefined) return true;
+ return !(state === "missing" || state === "stale" || state === "partial");
+}
+
export type MissingNeverFetched = {
id: string;
// The URL the video was last seen at. "" only when it was never observed in a
@@ -735,10 +830,30 @@ export async function generateChannelSnapshot(
let foreignAudioBytes = 0;
let transcribed = 0;
let downloaded = 0;
+ // WHY the audio that isn't reclaimable isn't. The sweep reports this only as a
+ // line in a job log after the fact ("Skipped 412 (388 protected, …)"), with no
+ // bytes attached to it, so nothing could rank the holds or say which of them a
+ // queued job will ever clear. Free to compute here: every input is already in
+ // hand on the perVideo entry, so this adds ZERO I/O to a pass that runs over
+ // ~79,000 videos.
+ let totalAudioBytes = 0;
+ const heldAudioBytes = emptyHeldAudio();
+ const heldAudioCounts = emptyHeldAudio();
+ let reclaimableAtRiskBytes = 0;
+ // Is the lane that would release an awaiting-diarization video even running?
+ // allBackfillKinds drops a kind whose enabled() is false, so an ABSENT entry
+ // means "no diarize job will ever be produced" — while the sweep's guard holds
+ // the audio on settings.diarization.enabled alone. That gap is a permanent,
+ // silent hold, and this is the only place in the app that can see it.
+ const diarizationKindEnabled = backfillKinds.some(
+ (k) => k.id === DIARIZATION_KIND_ID,
+ );
for (const {
id,
files,
audioSizes,
+ backfill,
+ effectiveAvailability,
outcome,
coverage,
vttProvenance,
@@ -746,6 +861,58 @@ export async function generateChannelSnapshot(
} of perVideo) {
if (isVideoTranscribed(files)) transcribed++;
if (isVideoDownloaded(files)) downloaded++;
+
+ // --- Hold attribution ---------------------------------------------------
+ // FIRST, before the short-circuits below: they `continue` past videos that
+ // still have audio on disk, and audio dropped here would leave the partition
+ // (held + reclaimable === total) silently short.
+ const audioBytes = files.audioFiles.reduce(
+ (sum, name) => sum + (audioSizes[name] ?? 0),
+ 0,
+ );
+ totalAudioBytes += audioBytes;
+ if (files.audioFiles.length > 0) {
+ const gate = attributeAudioHold({
+ // The sweep's own gate — entries.includes("transcript.json"). NOT
+ // isVideoTranscribed, which counts any English VTT: an ASR-only video is
+ // absent from downloadedNoTranscript and still uncleanable.
+ //
+ // A corrupt-full-source container is forced to this gate to stay in step
+ // with the transcribedWithAudio bucket, which short-circuits it below:
+ // it is an artifact, not usable audio, and the estimate must not offer
+ // the kept file up for deletion.
+ hasWhisper:
+ files.hasWhisper && outcome?.status !== "corrupt-full-source",
+ inKeepLatestWindow: keptIds.has(id),
+ doNotClean: doNotCleanIds.has(id),
+ diarizationGuardOn: diarizationEnabled,
+ hasDiarization: files.hasDiarization,
+ });
+ if (gate === "reclaimable") {
+ // What verifyBeforeClean is likely to refuse at sweep time, from the
+ // CACHED verdict only — no probe, and deliberately not a fifth gate.
+ if (
+ effectiveAvailability &&
+ (isPermanentlyGone(effectiveAvailability) ||
+ effectiveAvailability === "needs_auth" ||
+ effectiveAvailability === "error")
+ ) {
+ reclaimableAtRiskBytes += audioBytes;
+ }
+ } else {
+ heldAudioBytes[gate] += audioBytes;
+ heldAudioCounts[gate] += 1;
+ if (
+ gate === "awaitingDiarization" &&
+ diarizationWillNeverClear(
+ diarizationKindEnabled ? backfill[DIARIZATION_KIND_ID] : undefined,
+ )
+ ) {
+ heldAudioBytes.diarizationNeverClears += audioBytes;
+ heldAudioCounts.diarizationNeverClears += 1;
+ }
+ }
+ }
// A download that completed but stayed malformed after one re-download. The
// raw container (e.g. audio.mp4) is kept on disk for inspection. It IS an
// artifact (so it won't be re-queued for download), but it is NOT usable
@@ -1079,6 +1246,10 @@ export async function generateChannelSnapshot(
multipleAudioFormats: multipleAudioFormatsBytes,
foreignAudio: foreignAudioBytes,
},
+ totalAudioBytes,
+ heldAudioBytes,
+ heldAudioCounts,
+ reclaimableAtRiskBytes,
};
await writeChannelSnapshot(paths, slug, snapshot);
diff --git a/common/lib/backfillKinds.ts b/common/lib/backfillKinds.ts
@@ -945,6 +945,12 @@ export function operationLabel(id: string): string {
// snapshot entry does — silently, as "nothing to do".
export const DIGEST_KIND_ID = "digest";
+// The diarization operation's id, named once for the same reason. The cleanup
+// accounting needs it to ask a question no other surface asks: whether the lane
+// that would release a held video is even running (allBackfillKinds drops a kind
+// whose enabled() is false, so an ABSENT entry is the answer, not a zero).
+export const DIARIZATION_KIND_ID = "diarization";
+
// One entry per backfill known to the system.
export const BACKFILL_KINDS: readonly BackfillKind[] = [
diarization,
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,8 @@
# Changelog
## [Unreleased]
+- **Cleanup now says what is holding the audio it can't reclaim, and what to run to get it back.** The page led with one number — how much space you can free right now — and said nothing about the rest of the disk. The sweep skips videos for four different reasons and reported them only as a line in a job log after the fact, with no bytes attached and nothing ranked. A **sieve** now runs down the page: all the audio on disk enters at the top, each gate siphons off its share (no transcript yet, the keep-latest window, do-not-clean pins, awaiting diarization), and the remainder steps down to the reclaimable figure the page already led with. A video leaves at the *first* gate it hits, exactly as the sweep's own cascade does, so the five figures are an attribution and never overlap — a pinned, undiarized video is counted once, under the pin. Below it, the **release ledger** ranks the channels holding the most, split by what it costs to get the space back: a run of a lane that is already weeks deep, or a setting that frees it the moment it changes. "Not counted" sits in the second group and is not called a hold — excluding a channel hides its bytes from the total, it never protected them, which makes it the fastest win on the page.
+- **Some of that audio is held forever, and nothing anywhere said so.** The cleanup guard fires on `diarization.enabled` alone, but the diarize lane will never produce a sidecar for a video that is over `maxAudioHours`, has no diarizable input, or belongs to a channel whose diarization models were never configured — the kind reports itself disabled and no job is ever queued. Those videos were held from cleaning permanently, and running *Diarize speakers* until the end of time would not have moved the number. The sieve now draws that slice hatched inside gate ④ with its own figure and says it in words: *will not clear on its own*. Deciding what to do about it — raise the cap, configure the models, or let the sweep past the guard — is deliberately left to you: audio is the one input in this pipeline that cannot be regenerated. Channel is the honest granularity throughout; per-video byte sizes exist nowhere outside the report, and inventing them would have cost a corpus walk on every render.
- **A channel page now opens on the channel's whole lifecycle instead of a viewport of video rows.** The first screen is a single transit line — Playlist → Download → Transcode → Transcribe → Digest → Backfill — where each station carries its count and its coverage, and the gap between two stations carries the shortfall, because the gap *is* the work. Only the largest shortfall is emphasised, so the bottleneck is the thing your eye lands on. Work the lane can actually do today sits on the line; everything that left it (needs cookies, deleted, untranscribable, waiting on a transcript, needs media re-acquired) hangs below it in muted type, on a different axis, so the two can never be read as one number. Below the line: one brand-coloured **next action** that disappears when there is nothing to do, a strip of anomaly chips that says "All clear" rather than vanishing, and the stage panels — **one at a time**, chosen by a tab strip and carried in the URL as `?stage=`, so a stage is a link you can share. With no stage selected you get an overview listing every stage's status at once. Previously the page stacked a full-height video browser above eight simultaneously-expanded stage panels, which put the channel's actual pipeline a full screen and a collapsed `<details>` out of reach.
- **The video browser has moved to its own page and no longer builds a DOM node per video.** `/channels/<slug>/videos` is a two-pane workspace: a virtualized list on the left, the selected video's detail on the right, both always visible. The largest channel has ~11,000 videos and the old list rendered every one of them into the page; it now renders the twenty or so you can see. Filters, search and `?video=` selection work exactly as before. The old collapse toggles are gone — they existed only because the pane was squeezed onto a page it shared with everything else, and a page of its own removes the reason for them.
- **Command bookmarks are gone.** Saving a job under a label and re-launching it from the strip above **Jobs** / **Active jobs**, and the `/jobs/bookmarks` page that managed those labels, have all been removed. The feature was used for three days after it shipped and never again: every job it could re-launch is still one click away from its own channel-page control, and **Retry** — which re-runs a finished job straight from the descriptor stored with it — covers re-running something you have already run. Retry and **Retry all failed** are untouched and keep working exactly as before, including on jobs old enough to have been evicted from memory. Bucket jobs still re-derive their work from the channel's current state on every re-run rather than replaying a stale list. Your existing `transcripts/.bookmarks/bookmarks.json` is left on disk as a record; nothing reads it any more, and you can delete it whenever you like.
diff --git a/editor/app/cleanup/components/HoldSieve.tsx b/editor/app/cleanup/components/HoldSieve.tsx
@@ -0,0 +1,311 @@
+import Link from "next/link";
+import { formatBytes } from "yt-dlp-transcript-common/lib/format";
+import {
+ STATION_DOT,
+ STATION_TEXT,
+ formatCount,
+} from "../../channels/[slug]/components/flow/tone";
+import type { StageTone } from "../../channels/[slug]/lib/stageStatus";
+import type { CleanupSummary } from "../lib/loadCleanup";
+
+// THE SIEVE. The cleanup sweep is a cascade of `continue`s — a video leaves at
+// the FIRST gate it hits — so this is the shape the page takes: all the audio on
+// disk enters at the top, each gate siphons off its share, and the remainder
+// steps down the column to the reclaimable figure the hero already prints.
+//
+// The gates are NUMBERED because they genuinely are a sequence. The order IS the
+// attribution rule, and it is the only thing stopping the five figures from
+// double-counting each other.
+//
+// No new colours, no new type. The rail and its station dots are the transit
+// vocabulary the channel page established (flow/tone.ts, which the two maps are
+// imported from rather than copied — laneState.ts warns that a third copy would
+// guarantee drift), and the bars use the same --chart-N tokens as the hero bar
+// directly above.
+
+type Gate = {
+ n: number;
+ label: string;
+ // What clears it. Split so the prescription can be a link where one exists.
+ clears: React.ReactNode;
+ bytes: number;
+ count: number;
+ // Bytes inside this gate that NOTHING queued will ever release.
+ neverClears?: number;
+};
+
+export function HoldSieve({ summary }: { summary: CleanupSummary }) {
+ const {
+ totalAudioBytes,
+ heldBytes,
+ heldCounts,
+ measuredReclaimBytes,
+ atRiskBytes,
+ staleSnapshotCount,
+ } = summary;
+
+ // Nothing measured at all: every figure would be a zero nobody computed.
+ if (totalAudioBytes <= 0) {
+ return (
+ <section
+ aria-label="audio hold sieve"
+ className="flex flex-col gap-2 rounded-xl border border-border bg-card p-5"
+ >
+ <Eyebrow>Audio on disk</Eyebrow>
+ <p
+ className="font-display text-3xl font-semibold tabular-nums text-muted-foreground"
+ aria-label="total audio on disk"
+ >
+ —
+ </p>
+ <p className="text-sm text-muted-foreground">
+ {staleSnapshotCount > 0 ? (
+ <>
+ No report yet says what is holding this audio.{" "}
+ <RefreshLink /> to measure it.
+ </>
+ ) : (
+ <>No audio on disk.</>
+ )}
+ </p>
+ </section>
+ );
+ }
+
+ const gates: Gate[] = [
+ {
+ n: 1,
+ label: "no transcript yet",
+ clears: "run: Transcribe",
+ bytes: heldBytes.noTranscript,
+ count: heldCounts.noTranscript,
+ },
+ {
+ n: 2,
+ label: "keep-latest window",
+ clears: "setting: keepLatest",
+ bytes: heldBytes.keepLatest,
+ count: heldCounts.keepLatest,
+ },
+ {
+ n: 3,
+ label: "do-not-clean pins",
+ clears: "unpin on the video",
+ bytes: heldBytes.doNotClean,
+ count: heldCounts.doNotClean,
+ },
+ {
+ n: 4,
+ label: "awaiting diarization",
+ clears: "run: Diarize",
+ bytes: heldBytes.awaitingDiarization,
+ count: heldCounts.awaitingDiarization,
+ neverClears: heldBytes.diarizationNeverClears,
+ },
+ ];
+
+ // The remainder after each gate, which is what the column steps down.
+ let remainder = totalAudioBytes;
+ const steps = gates.map((gate) => {
+ remainder -= gate.bytes;
+ return { gate, remainder: Math.max(0, remainder) };
+ });
+
+ const pct = (n: number) => (n / totalAudioBytes) * 100;
+
+ return (
+ <section
+ aria-label="audio hold sieve"
+ className="flex flex-col gap-4 rounded-xl border border-border bg-card p-5"
+ >
+ {/* Everything on disk enters here. */}
+ <div className="flex flex-wrap items-end justify-between gap-x-4 gap-y-1">
+ <Eyebrow>Audio on disk</Eyebrow>
+ <p
+ className="font-display text-3xl font-semibold tabular-nums text-foreground"
+ aria-label="total audio on disk"
+ >
+ ~{formatBytes(totalAudioBytes)}
+ </p>
+ </div>
+ <Bar>
+ <span className="h-full bg-[var(--chart-1)]" style={{ width: "100%" }} />
+ </Bar>
+
+ {/* The rail. One gate per station, in the sweep's own order. */}
+ <ol className="flex flex-col border-l border-border pl-4 sm:pl-5">
+ {steps.map(({ gate, remainder: left }) => (
+ <GateRow key={gate.n} gate={gate} remainder={left} pct={pct} />
+ ))}
+ </ol>
+
+ {/* What survives all four — the number the page already leads with. */}
+ <div className="flex flex-col gap-1.5">
+ <div className="flex flex-wrap items-end justify-between gap-x-4 gap-y-1">
+ <span className="flex items-center gap-2">
+ <span
+ aria-hidden="true"
+ className={`inline-block h-2 w-2 shrink-0 rounded-full ${STATION_DOT.ok}`}
+ />
+ <Eyebrow>Reclaimable now</Eyebrow>
+ </span>
+ <p
+ className="font-display text-3xl font-semibold tabular-nums text-foreground"
+ aria-label="reclaimable after holds"
+ >
+ ~{formatBytes(measuredReclaimBytes)}
+ </p>
+ </div>
+ <Bar>
+ <span
+ className="h-full bg-[var(--chart-1)]"
+ style={{ width: `${pct(measuredReclaimBytes)}%` }}
+ />
+ </Bar>
+ </div>
+
+ {atRiskBytes > 0 && (
+ <p className="font-mono text-[11px] text-muted-foreground">
+ the upstream check may still refuse ~{formatBytes(atRiskBytes)} cached
+ as gone-from-source
+ </p>
+ )}
+ {staleSnapshotCount > 0 && (
+ <p className="font-mono text-[11px] text-muted-foreground">
+ {staleSnapshotCount} channel{staleSnapshotCount === 1 ? "" : "s"} not
+ in this measurement — <RefreshLink /> to include{" "}
+ {staleSnapshotCount === 1 ? "it" : "them"}.
+ </p>
+ )}
+ </section>
+ );
+}
+
+function GateRow({
+ gate,
+ remainder,
+ pct,
+}: {
+ gate: Gate;
+ remainder: number;
+ pct: (n: number) => number;
+}) {
+ const held = gate.bytes > 0;
+ const never = gate.neverClears ?? 0;
+ // A gate holding nothing is an outline, not a filled dot — the tone map's own
+ // rule. A gate holding audio nothing will ever release is the one danger on
+ // the page, and it earns it.
+ const tone: StageTone = never > 0 ? "danger" : held ? "attention" : "neutral";
+ const clearsBytes = Math.max(0, gate.bytes - never);
+
+ return (
+ <li
+ aria-label={`gate ${gate.n} ${gate.label}`}
+ data-gate={gate.n}
+ data-held-bytes={gate.bytes}
+ data-held-count={gate.count}
+ className="relative flex flex-col gap-1 py-2"
+ >
+ {/* The tick joining this station to the rail. */}
+ <span
+ aria-hidden="true"
+ className="absolute -left-4 top-[1.1rem] h-px w-3 bg-border sm:-left-5 sm:w-4"
+ />
+ <span
+ aria-hidden="true"
+ className={`absolute -left-[1.3rem] top-4 inline-block h-2 w-2 rounded-full ${STATION_DOT[tone]} sm:-left-[1.55rem]`}
+ />
+
+ <div className="flex flex-wrap items-baseline gap-x-3 gap-y-0.5">
+ <span
+ aria-hidden="true"
+ className="font-mono text-[11px] tabular-nums text-muted-foreground"
+ >
+ {gate.n}
+ </span>
+ <span className={`text-sm ${STATION_TEXT[tone]}`}>{gate.label}</span>
+ <span
+ className="font-mono text-xs tabular-nums text-foreground"
+ aria-label={`${gate.label} held`}
+ >
+ {gate.bytes > 0 ? `~${formatBytes(gate.bytes)}` : "—"}
+ </span>
+ <span className="font-mono text-xs tabular-nums text-muted-foreground">
+ {gate.count > 0 ? formatCount(gate.count) : "—"} vids
+ </span>
+ <span
+ className="ml-auto font-mono text-xs tabular-nums text-muted-foreground"
+ aria-label={`remaining after gate ${gate.n}`}
+ >
+ → ~{formatBytes(remainder)}
+ </span>
+ </div>
+
+ {/* What is left, and what this gate just took out of it. */}
+ <Bar>
+ <span
+ className="h-full bg-[var(--chart-1)]"
+ style={{ width: `${pct(remainder)}%` }}
+ />
+ <span
+ className="h-full bg-[var(--chart-3)]"
+ style={{ width: `${pct(clearsBytes)}%` }}
+ />
+ {never > 0 && (
+ // The one risk this page takes. Not a new colour — the SAME token,
+ // struck through with a repeating gradient, because the honest thing
+ // to say about this slice is that it is the rest of the bar plus a
+ // permanent defect. It is the one number nothing else in the app can
+ // tell you, and the reason the page is worth building.
+ <span
+ className="h-full bg-[var(--chart-5)]"
+ style={{
+ width: `${pct(never)}%`,
+ backgroundImage:
+ "repeating-linear-gradient(45deg, transparent 0 3px, var(--card) 3px 5px)",
+ }}
+ />
+ )}
+ </Bar>
+
+ <p className="font-mono text-[11px] text-muted-foreground">
+ {gate.clears}
+ {never > 0 && (
+ <>
+ {" · "}
+ <span className="text-destructive">
+ ~{formatBytes(never)} will not clear on its own
+ </span>
+ </>
+ )}
+ </p>
+ </li>
+ );
+}
+
+function Bar({ children }: { children: React.ReactNode }) {
+ return (
+ <div
+ className="flex h-2 w-full overflow-hidden rounded-full bg-muted"
+ role="presentation"
+ >
+ {children}
+ </div>
+ );
+}
+
+function Eyebrow({ children }: { children: React.ReactNode }) {
+ return (
+ <p className="font-mono text-[11px] uppercase tracking-[0.18em] text-muted-foreground">
+ {children}
+ </p>
+ );
+}
+
+function RefreshLink() {
+ return (
+ <Link href="/actionable" className="underline hover:text-foreground">
+ Refresh reports
+ </Link>
+ );
+}
diff --git a/editor/app/cleanup/components/ReleaseLedger.tsx b/editor/app/cleanup/components/ReleaseLedger.tsx
@@ -0,0 +1,370 @@
+"use client";
+
+import { useState } from "react";
+import Link from "next/link";
+import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog";
+import { formatBytes } from "yt-dlp-transcript-common/lib/format";
+import { formatCount } from "../../channels/[slug]/components/flow/tone";
+import { cancelJobAction } from "../../jobs/actions";
+import {
+ transcribeBucketAction,
+ transcribeAutoSubsBucketAction,
+ diarizeChannelAction,
+} from "../../channels/[slug]/whisperActions";
+import type { CleanupLedger, LedgerRow } from "../lib/loadCleanup";
+
+// THE RELEASE LEDGER. The sieve says what is holding the audio; this says what
+// to run to get it back, split by WHAT IT COSTS to release — a run of a lane
+// that is already weeks deep, or a setting that frees the space the moment it
+// changes. That split is a true property of the data, and it is the sharpest
+// answer to "what frees space quickest".
+//
+// Channels rank by held bytes, descending, inside each group.
+
+const ROW_LIMIT = 5;
+
+export function ReleaseLedger({ ledger }: { ledger: CleanupLedger }) {
+ const { noTranscript, awaitingDiarization, notCounted, keepLatest, doNotClean } =
+ ledger;
+ const hasRuns = noTranscript.bytes > 0 || awaitingDiarization.bytes > 0;
+ const hasSettings =
+ notCounted.bytes > 0 || keepLatest.bytes > 0 || doNotClean.bytes > 0;
+ if (!hasRuns && !hasSettings) return null;
+
+ return (
+ <section
+ aria-label="release ledger"
+ className="flex flex-col gap-6 rounded-xl border border-border bg-card p-5"
+ >
+ {hasRuns && (
+ <div className="flex flex-col gap-4">
+ <Eyebrow>Costs a run</Eyebrow>
+
+ {noTranscript.bytes > 0 && (
+ <GateGroup
+ n={1}
+ label="no transcript"
+ bytes={noTranscript.bytes}
+ count={noTranscript.count}
+ rows={noTranscript.rows}
+ action={(row) => <TranscribeButtons row={row} />}
+ />
+ )}
+
+ {awaitingDiarization.bytes > 0 && (
+ <GateGroup
+ n={4}
+ label="awaiting diarization"
+ bytes={awaitingDiarization.bytes}
+ count={awaitingDiarization.count}
+ rows={awaitingDiarization.rows}
+ // Diarize takes no id list, so every row can act — the cap on the
+ // Transcribe rows is a payload bound, not a display rule.
+ action={(row) => (
+ <StreamActionLog
+ trigger={() => diarizeChannelAction(row.slug)}
+ cancelAction={cancelJobAction}
+ buttonLabel="Diarize"
+ runningLabel="Diarizing…"
+ label={`Diarize ${row.slug}`}
+ />
+ )}
+ note={
+ awaitingDiarization.neverClearsBytes > 0 ? (
+ <span className="text-destructive">
+ ~{formatBytes(awaitingDiarization.neverClearsBytes)} of this
+ will not clear on its own — those videos are over the
+ diarization limit, have no diarizable input, or the lane is
+ not configured to run at all.
+ </span>
+ ) : undefined
+ }
+ />
+ )}
+ </div>
+ )}
+
+ {hasSettings && (
+ <div className="flex flex-col gap-3">
+ <Eyebrow>Frees on a setting</Eyebrow>
+
+ {/* Stated precisely, and deliberately NOT called a hold.
+ excludeFromCleanup is display-only — the sweep never reads it — so
+ this space is already reclaimable today. Excluding a channel hides
+ it from the total; it does not protect it. */}
+ {notCounted.bytes > 0 && (
+ <SettingRow
+ label="Not counted"
+ bytes={notCounted.bytes}
+ meta={`${notCounted.channels} channel${notCounted.channels === 1 ? "" : "s"}`}
+ hint="already cleanable — just hidden from the total"
+ rows={notCounted.rows}
+ />
+ )}
+ {keepLatest.bytes > 0 && (
+ <SettingRow
+ label="Keep-latest window"
+ bytes={keepLatest.bytes}
+ meta={`${formatCount(keepLatest.count)} vids · ${keepLatest.channels} channel${keepLatest.channels === 1 ? "" : "s"}`}
+ hint="lower keepLatest on a channel"
+ />
+ )}
+ {doNotClean.bytes > 0 && (
+ <SettingRow
+ label="Do-not-clean pins"
+ bytes={doNotClean.bytes}
+ meta={`${formatCount(doNotClean.count)} vids · ${doNotClean.channels} channel${doNotClean.channels === 1 ? "" : "s"}`}
+ hint="unpin from the video"
+ />
+ )}
+ </div>
+ )}
+ </section>
+ );
+}
+
+function GateGroup({
+ n,
+ label,
+ bytes,
+ count,
+ rows,
+ action,
+ note,
+}: {
+ n: number;
+ label: string;
+ bytes: number;
+ count: number;
+ rows: LedgerRow[];
+ action: (row: LedgerRow) => React.ReactNode;
+ note?: React.ReactNode;
+}) {
+ const { shown, hidden, expanded, toggle } = useExpandableList(rows, ROW_LIMIT);
+ return (
+ <div className="flex flex-col gap-1.5" aria-label={`release ${label}`}>
+ <p className="flex flex-wrap items-baseline gap-x-2 text-sm">
+ <span
+ aria-hidden="true"
+ className="font-mono text-[11px] tabular-nums text-muted-foreground"
+ >
+ {n}
+ </span>
+ <span className="text-foreground">{label}</span>
+ <span className="font-mono text-xs tabular-nums text-muted-foreground">
+ — ~{formatBytes(bytes)} · {formatCount(count)} vids
+ </span>
+ </p>
+ {note && (
+ <p className="font-mono text-[11px] leading-relaxed">{note}</p>
+ )}
+ <ul className="flex flex-col">
+ {shown.map((row) => (
+ <li
+ key={row.slug}
+ aria-label={`${label} ${row.slug}`}
+ data-held-bytes={row.bytes}
+ className="flex flex-wrap items-center gap-x-3 gap-y-1 border-b border-border/60 py-1.5 last:border-b-0"
+ >
+ <Link
+ href={`/channels/${row.slug}`}
+ className="min-w-0 max-w-[14rem] truncate font-mono text-xs text-foreground hover:text-brand"
+ title={row.slug}
+ >
+ {row.name}
+ </Link>
+ <span className="font-mono text-xs tabular-nums text-foreground">
+ ~{formatBytes(row.bytes)}
+ </span>
+ <span className="font-mono text-xs tabular-nums text-muted-foreground">
+ {formatCount(row.count)} vids
+ </span>
+ <span className="ml-auto flex flex-wrap items-center gap-1.5">
+ {action(row)}
+ </span>
+ </li>
+ ))}
+ <ListOverflowToggle
+ hidden={hidden}
+ expanded={expanded}
+ onToggle={toggle}
+ noun="channel"
+ />
+ </ul>
+ </div>
+ );
+}
+
+// The Transcribe half of the prescription. Two buttons at most, and the second
+// only when the channel HAS ASR-only videos: those carry an English VTT, so they
+// are absent from downloadedNoTranscript and are still holding their audio.
+function TranscribeButtons({ row }: { row: LedgerRow }) {
+ const ids = row.transcribeIds;
+ const auto = row.autoSubsIds;
+ if (!ids) {
+ // Past the id cap — the same control lives on the channel page.
+ return (
+ <Link
+ href={`/channels/${row.slug}?stage=transcribe`}
+ className="font-mono text-xs text-muted-foreground underline underline-offset-2 hover:text-foreground"
+ >
+ on the channel
+ </Link>
+ );
+ }
+ return (
+ <>
+ {ids.length > 0 && (
+ <StreamActionLog
+ trigger={() =>
+ transcribeBucketAction(
+ row.slug,
+ ids,
+ undefined,
+ undefined,
+ false,
+ "downloadedNoTranscript",
+ )
+ }
+ cancelAction={cancelJobAction}
+ buttonLabel={`Transcribe ${ids.length}`}
+ runningLabel="Transcribing…"
+ label={`Transcribe ${row.slug}`}
+ />
+ )}
+ {auto && auto.length > 0 && (
+ <StreamActionLog
+ trigger={() => transcribeAutoSubsBucketAction(row.slug, auto)}
+ cancelAction={cancelJobAction}
+ buttonLabel={`Transcribe ${auto.length} ASR-only`}
+ runningLabel="Transcribing…"
+ label={`Transcribe ASR-only ${row.slug}`}
+ />
+ )}
+ {ids.length === 0 && (!auto || auto.length === 0) && (
+ <Link
+ href={`/channels/${row.slug}?stage=transcribe`}
+ className="font-mono text-xs text-muted-foreground underline underline-offset-2 hover:text-foreground"
+ >
+ on the channel
+ </Link>
+ )}
+ </>
+ );
+}
+
+function SettingRow({
+ label,
+ bytes,
+ meta,
+ hint,
+ rows,
+}: {
+ label: string;
+ bytes: number;
+ meta: string;
+ hint: string;
+ rows?: LedgerRow[];
+}) {
+ const { shown, hidden, expanded, toggle } = useExpandableList(
+ rows ?? [],
+ ROW_LIMIT,
+ );
+ return (
+ <div className="flex flex-col gap-1" aria-label={`release ${label}`}>
+ <p className="flex flex-wrap items-baseline gap-x-3 gap-y-0.5 text-sm">
+ <span className="text-foreground">{label}</span>
+ <span
+ className="font-mono text-xs tabular-nums text-foreground"
+ aria-label={`${label} bytes`}
+ >
+ ~{formatBytes(bytes)}
+ </span>
+ <span className="font-mono text-xs tabular-nums text-muted-foreground">
+ {meta}
+ </span>
+ <span className="font-mono text-[11px] text-muted-foreground">
+ {hint}
+ </span>
+ </p>
+ {rows && rows.length > 0 && (
+ <ul className="flex flex-col pl-3">
+ {shown.map((row) => (
+ <li
+ key={row.slug}
+ aria-label={`${label} ${row.slug}`}
+ className="flex flex-wrap items-center gap-x-3 py-0.5"
+ >
+ <Link
+ href={`/channels/${row.slug}`}
+ className="min-w-0 max-w-[14rem] truncate font-mono text-xs text-muted-foreground hover:text-brand"
+ title={row.slug}
+ >
+ {row.name}
+ </Link>
+ <span className="font-mono text-xs tabular-nums text-muted-foreground">
+ ~{formatBytes(row.bytes)}
+ </span>
+ </li>
+ ))}
+ <ListOverflowToggle
+ hidden={hidden}
+ expanded={expanded}
+ onToggle={toggle}
+ noun="channel"
+ />
+ </ul>
+ )}
+ </div>
+ );
+}
+
+// Mirrors the widget's list disclosure (MonitorWidget.tsx) — same rule: `hidden`
+// is computed against the LIMIT, not against what is shown, so the control keeps
+// its label once expanded instead of vanishing mid-use.
+function useExpandableList<T>(items: T[], limit: number) {
+ const [expanded, setExpanded] = useState(false);
+ const hidden = Math.max(0, items.length - limit);
+ return {
+ shown: expanded ? items : items.slice(0, limit),
+ hidden,
+ expanded,
+ toggle: () => setExpanded((e) => !e),
+ };
+}
+
+function ListOverflowToggle({
+ hidden,
+ expanded,
+ onToggle,
+ noun,
+}: {
+ hidden: number;
+ expanded: boolean;
+ onToggle: () => void;
+ noun: string;
+}) {
+ if (hidden === 0) return null;
+ return (
+ <li>
+ <button
+ type="button"
+ onClick={onToggle}
+ aria-expanded={expanded}
+ className="text-xs text-muted-foreground underline underline-offset-2 hover:text-foreground"
+ >
+ {expanded
+ ? "Show fewer"
+ : `+${hidden} more ${hidden === 1 ? noun : `${noun}s`}`}
+ </button>
+ </li>
+ );
+}
+
+function Eyebrow({ children }: { children: React.ReactNode }) {
+ return (
+ <p className="font-mono text-[11px] uppercase tracking-[0.18em] text-muted-foreground">
+ {children}
+ </p>
+ );
+}
diff --git a/editor/app/cleanup/lib/loadCleanup.ts b/editor/app/cleanup/lib/loadCleanup.ts
@@ -1,7 +1,11 @@
import type { Paths } from "yt-dlp-transcript-common/lib/paths";
import type { ChannelBrief } from "yt-dlp-transcript-common/controller/channels";
import { getChannelBriefs } from "../../lib/requestCache";
-import type { ChannelSnapshot } from "yt-dlp-transcript-common/controller/channelSnapshot";
+import {
+ emptyHeldAudio,
+ type ChannelSnapshot,
+ type HeldAudio,
+} from "yt-dlp-transcript-common/controller/channelSnapshot";
import { loadFailedTranscriptions } from "yt-dlp-transcript-common/controller/failedTranscriptions";
import { loadFailedTranscodings } from "yt-dlp-transcript-common/controller/failedTranscodings";
@@ -22,6 +26,21 @@ export type CleanupRow = {
// Housekeeping list sizes (failed transcriptions / transcodings).
failedTranscriptions: number;
failedTranscodings: number;
+ // --- What is holding the audio this row can't reclaim --------------------
+ // Straight off the snapshot the brief already carries (CleanupRow.snapshot has
+ // been loaded and unread since this file was written) — no new disk reads. A
+ // report written before the accounting existed has `measured: false`, and its
+ // figures must render "—", never "0": a zero would claim a measurement nobody
+ // took.
+ measured: boolean;
+ totalAudioBytes: number;
+ heldBytes: HeldAudio;
+ heldCounts: HeldAudio;
+ reclaimableAtRiskBytes: number;
+ // Ids for the release prescriptions. `downloadedNoTranscript` is the bucket
+ // the Transcribe button runs, and `downloadedAutoSubsOnly` its ASR-only twin.
+ transcribeIds: string[];
+ autoSubsIds: string[];
};
export type CleanupSummary = {
@@ -38,6 +57,26 @@ export type CleanupSummary = {
includedForeignBytes: number;
includedCount: number;
excludedCount: number;
+ // --- The sieve, summed over MEASURED, INCLUDED channels ------------------
+ // Measured-only on purpose: a channel whose report predates the hold
+ // accounting contributes zeros to every gate, and quietly adding its
+ // reclaimable bytes to the tail would make the column stop adding up. It is
+ // counted in `staleSnapshotCount` and named instead.
+ totalAudioBytes: number;
+ heldBytes: HeldAudio;
+ heldCounts: HeldAudio;
+ // The remainder the four gates leave — equal to the hero figure whenever every
+ // included channel has been measured.
+ measuredReclaimBytes: number;
+ // Of that remainder, what the pre-clean availability check is likely to refuse
+ // (cached gone-from-source / needs-auth / error). An annotation, not a gate.
+ atRiskBytes: number;
+ // Included channels with no hold accounting yet — the "—" case.
+ staleSnapshotCount: number;
+ // Reclaimable bytes sitting in EXCLUDED channels. Not a hold: excludeFromCleanup
+ // is display-only (the sweep never reads it), so this space is already
+ // reclaimable today — it is merely hidden from the total.
+ excludedReclaimBytes: number;
};
function transcribedBytesOf(snap: ChannelSnapshot | null): number {
@@ -60,9 +99,35 @@ function rowOf(
hasTargetFormat: Boolean(channel.config.audioFormat),
failedTranscriptions,
failedTranscodings,
+ measured: snapshot?.heldAudioBytes !== undefined,
+ totalAudioBytes: snapshot?.totalAudioBytes ?? 0,
+ heldBytes: snapshot?.heldAudioBytes ?? emptyHeldAudio(),
+ heldCounts: snapshot?.heldAudioCounts ?? emptyHeldAudio(),
+ reclaimableAtRiskBytes: snapshot?.reclaimableAtRiskBytes ?? 0,
+ transcribeIds: snapshot?.buckets.downloadedNoTranscript ?? [],
+ autoSubsIds: snapshot?.buckets.downloadedAutoSubsOnly ?? [],
};
}
+function addHeld(into: HeldAudio, from: HeldAudio): void {
+ into.noTranscript += from.noTranscript;
+ into.keepLatest += from.keepLatest;
+ into.doNotClean += from.doNotClean;
+ into.awaitingDiarization += from.awaitingDiarization;
+ into.diarizationNeverClears += from.diarizationNeverClears;
+}
+
+export function heldTotalOf(held: HeldAudio): number {
+ // diarizationNeverClears is a SUBSET of awaitingDiarization, so it is never
+ // added — summing it would double-count the one hold that matters most.
+ return (
+ held.noTranscript +
+ held.keepLatest +
+ held.doNotClean +
+ held.awaitingDiarization
+ );
+}
+
export async function loadCleanupSummary(
paths: Paths,
): Promise<CleanupSummary> {
@@ -91,6 +156,13 @@ export async function loadCleanupSummary(
let includedForeignBytes = 0;
let includedCount = 0;
let excludedCount = 0;
+ let totalAudioBytes = 0;
+ let measuredReclaimBytes = 0;
+ let atRiskBytes = 0;
+ let staleSnapshotCount = 0;
+ let excludedReclaimBytes = 0;
+ const heldBytes = emptyHeldAudio();
+ const heldCounts = emptyHeldAudio();
for (const r of rows) {
totalReclaimBytes += r.transcribedBytes;
if (r.included) {
@@ -98,8 +170,18 @@ export async function loadCleanupSummary(
includedReclaimBytes += r.transcribedBytes;
includedExtraFormatsBytes += r.extraFormatsBytes;
includedForeignBytes += r.foreignBytes;
+ if (r.measured) {
+ totalAudioBytes += r.totalAudioBytes;
+ measuredReclaimBytes += r.transcribedBytes;
+ atRiskBytes += r.reclaimableAtRiskBytes;
+ addHeld(heldBytes, r.heldBytes);
+ addHeld(heldCounts, r.heldCounts);
+ } else {
+ staleSnapshotCount++;
+ }
} else {
excludedCount++;
+ excludedReclaimBytes += r.transcribedBytes;
}
}
@@ -111,6 +193,13 @@ export async function loadCleanupSummary(
includedForeignBytes,
includedCount,
excludedCount,
+ totalAudioBytes,
+ heldBytes,
+ heldCounts,
+ measuredReclaimBytes,
+ atRiskBytes,
+ staleSnapshotCount,
+ excludedReclaimBytes,
};
}
@@ -158,3 +247,117 @@ export async function cleanableTotalBytes(paths: Paths): Promise<number> {
const rows = await cleanableChannels(paths);
return rows.reduce((sum, r) => sum + r.bytes, 0);
}
+
+// --- The release ledger ------------------------------------------------------
+//
+// The prescription under the sieve, split by WHAT IT COSTS to release, because
+// that is a true property of the data and the sharpest answer to "what frees
+// space quickest": a run of the GPU/CPU lanes, or a setting nobody has to wait
+// for. Channels rank by held bytes descending inside each group.
+
+// How many channels per group get an action button. The button hands the job a
+// LIST OF IDS, so the cap is a payload bound, not a display choice: past this
+// the row still ranks and still reports its bytes, and links to the channel page
+// where the same bucket control lives. Starting the biggest run is what this
+// page is for; the long tail belongs to the channel.
+export const LEDGER_ACTION_LIMIT = 6;
+
+export type LedgerRow = {
+ slug: string;
+ name: string;
+ bytes: number;
+ count: number;
+ // Undefined past LEDGER_ACTION_LIMIT — the row renders a link instead.
+ transcribeIds?: string[];
+ // Only when the channel HAS ASR-only videos: those carry an English VTT, so
+ // they are absent from downloadedNoTranscript and need the auto-subs lane.
+ autoSubsIds?: string[];
+};
+
+export type CleanupLedger = {
+ // Costs a run.
+ noTranscript: { bytes: number; count: number; rows: LedgerRow[] };
+ awaitingDiarization: {
+ bytes: number;
+ count: number;
+ neverClearsBytes: number;
+ rows: LedgerRow[];
+ };
+ // Frees on a setting.
+ notCounted: { bytes: number; channels: number; rows: LedgerRow[] };
+ keepLatest: { bytes: number; count: number; channels: number };
+ doNotClean: { bytes: number; count: number; channels: number };
+};
+
+function rankRows(
+ rows: CleanupRow[],
+ bytesOf: (r: CleanupRow) => number,
+ countOf: (r: CleanupRow) => number,
+ opts: { withIds?: boolean } = {},
+): LedgerRow[] {
+ return rows
+ .filter((r) => bytesOf(r) > 0 || countOf(r) > 0)
+ .sort((a, b) => bytesOf(b) - bytesOf(a))
+ .map((r, i) => ({
+ slug: r.channel.slug,
+ name: r.channel.config.name ?? r.channel.slug,
+ bytes: bytesOf(r),
+ count: countOf(r),
+ ...(opts.withIds && i < LEDGER_ACTION_LIMIT
+ ? {
+ transcribeIds: r.transcribeIds,
+ ...(r.autoSubsIds.length > 0 ? { autoSubsIds: r.autoSubsIds } : {}),
+ }
+ : {}),
+ }));
+}
+
+export function buildCleanupLedger(summary: CleanupSummary): CleanupLedger {
+ const included = summary.rows.filter((r) => r.included && r.measured);
+ const excluded = summary.rows.filter((r) => !r.included);
+ return {
+ noTranscript: {
+ bytes: summary.heldBytes.noTranscript,
+ count: summary.heldCounts.noTranscript,
+ rows: rankRows(
+ included,
+ (r) => r.heldBytes.noTranscript,
+ (r) => r.heldCounts.noTranscript,
+ { withIds: true },
+ ),
+ },
+ awaitingDiarization: {
+ bytes: summary.heldBytes.awaitingDiarization,
+ count: summary.heldCounts.awaitingDiarization,
+ neverClearsBytes: summary.heldBytes.diarizationNeverClears,
+ rows: rankRows(
+ included,
+ (r) => r.heldBytes.awaitingDiarization,
+ (r) => r.heldCounts.awaitingDiarization,
+ ),
+ },
+ // NOT a hold, and stated precisely: excludeFromCleanup is display-only (the
+ // sweep never reads it), so this space is reclaimable TODAY. Filing it under
+ // "held" would be the page's first lie — and it is in fact the fastest win
+ // on the page: no compute, no setting change, just run the sweep.
+ notCounted: {
+ bytes: summary.excludedReclaimBytes,
+ channels: excluded.length,
+ rows: rankRows(
+ excluded,
+ (r) => r.transcribedBytes,
+ (r) => r.snapshot?.buckets.transcribedWithAudio?.length ?? 0,
+ ),
+ },
+ keepLatest: {
+ bytes: summary.heldBytes.keepLatest,
+ count: summary.heldCounts.keepLatest,
+ channels: included.filter((r) => r.heldBytes.keepLatest > 0).length,
+ },
+ doNotClean: {
+ bytes: summary.heldBytes.doNotClean,
+ count: summary.heldCounts.doNotClean,
+ channels: included.filter((r) => r.heldBytes.doNotClean > 0).length,
+ },
+ };
+}
diff --git a/editor/app/cleanup/page.tsx b/editor/app/cleanup/page.tsx
@@ -3,8 +3,10 @@ import Link from "next/link";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import { formatBytes } from "yt-dlp-transcript-common/lib/format";
import { Alert, AlertDescription } from "yt-dlp-transcript-common/components/ui/alert";
-import { loadCleanupSummary } from "./lib/loadCleanup";
+import { buildCleanupLedger, loadCleanupSummary } from "./lib/loadCleanup";
import { ChannelCleanupCard } from "./components/ChannelCleanupCard";
+import { HoldSieve } from "./components/HoldSieve";
+import { ReleaseLedger } from "./components/ReleaseLedger";
export const dynamic = "force-dynamic";
@@ -107,6 +109,12 @@ export default async function CleanupPage() {
</Alert>
</section>
+ {/* Why the rest of it is still on disk, and what to run to get it back.
+ The hero answers "how much can I reclaim right now"; these two answer
+ the question an operator with weeks of queued GPU work actually has. */}
+ <HoldSieve summary={summary} />
+ <ReleaseLedger ledger={buildCleanupLedger(summary)} />
+
{!hasAnything ? (
<p className="text-sm text-muted-foreground border border-dashed border-border rounded p-4">
No channels yet.
diff --git a/editor/e2e/cleanup-holds.spec.ts b/editor/e2e/cleanup-holds.spec.ts
@@ -0,0 +1,202 @@
+import { mkdir, stat, writeFile } from "node:fs/promises";
+import { test, expect } from "@playwright/test";
+import {
+ generateReport,
+ readJson,
+ resetData,
+ resolvePath,
+ writeChannelConfig,
+ writeSettings,
+} from "./helpers";
+import { baseUrl } from "./baseUrl";
+
+// /cleanup's hold accounting: WHY the audio that isn't reclaimable isn't, and
+// which of those holds a queued job will ever clear.
+//
+// The sweep skips videos for four reasons and reports them only as a line in a
+// job log after the fact, with no bytes attached. The snapshot now attributes
+// every audio byte to the FIRST gate it hits (cleanAudioFromTranscribed's own
+// order), and the page renders that as a sieve. Two things have to hold or the
+// page lies: the five figures must be a partition (no double-counting), and a
+// report written before the accounting existed must render "—", never "0".
+
+const SLUG = "test-transcribe";
+const STALE = "stale-report";
+const SNAPSHOT_REL = `test-transcripts/channels/${SLUG}/snapshot.json`;
+
+function dataRel(videoId: string, file: string): string {
+ return `test-transcripts/channels/${SLUG}/data/${videoId}/${file}`;
+}
+
+async function seedTranscript(videoId: string): Promise<void> {
+ await writeFile(
+ resolvePath(dataRel(videoId, "transcript.json")),
+ '{"transcription":[]}\n',
+ );
+}
+
+async function audioBytes(videoId: string): Promise<number> {
+ return (await stat(resolvePath(dataRel(videoId, "audio.m4a")))).size;
+}
+
+type Held = {
+ noTranscript: number;
+ keepLatest: number;
+ doNotClean: number;
+ awaitingDiarization: number;
+ diarizationNeverClears: number;
+};
+
+type HoldSnapshot = {
+ totalAudioBytes?: number;
+ heldAudioBytes?: Held;
+ heldAudioCounts?: Held;
+ cleanupBytes?: { transcribedWithAudio: number };
+};
+
+test("the sieve attributes every held byte to exactly one gate, and names the hold nothing will clear", async ({
+ page,
+}) => {
+ // Generating a report and then walking three pages does not fit the default
+ // 30s budget once the box is busy: this measured 19.3s idle and timed out at
+ // 30s while another suite was running. The assertions are cheap; the setup is
+ // not, so the test gets room rather than a retry.
+ test.setTimeout(120_000);
+ await resetData("one-transcribe-channel-with-audio");
+ // The guard the sweep actually reads is `diarization.enabled` ALONE. Enabled
+ // without segModel/embModel is the case the page exists to expose: the guard
+ // holds the audio, the diarize KIND reports itself disabled, and so no queued
+ // job will ever release it. (Deliberately safe in a test: a disabled kind is
+ // dropped from the backfill lane, so nothing is dispatched.)
+ await writeSettings({
+ adminTitle: "Test Admin",
+ maxTranscriptPageBytes: 8388608,
+ sleepBetweenDownloadsSeconds: 0,
+ minFreeDiskGB: 0,
+ verifyAvailabilityBeforeClean: false,
+ syncScheduler: { fullSweepIntervalMinutes: 0 },
+ diarization: { enabled: true },
+ });
+
+ // vidA — transcribed and PINNED (and undiarized: the first-gate-wins case).
+ // vidB — audio, no transcript at all.
+ // vidC — transcribed, undiarized.
+ // vidD — transcribed AND diarized: the only reclaimable one.
+ await seedTranscript("vidA");
+ await seedTranscript("vidC");
+ await mkdir(resolvePath(dataRel("vidD", "")), { recursive: true });
+ await writeFile(resolvePath(dataRel("vidD", "audio.m4a")), "x".repeat(4096));
+ await seedTranscript("vidD");
+ await writeFile(
+ resolvePath(dataRel("vidD", "diarization.json")),
+ JSON.stringify({ segments: [], engine: "test", createdAt: "2026-01-01" }),
+ );
+ await fetch(`${baseUrl}/api/test/invalidate-cache`).catch(() => {});
+
+ await page.goto(`/channels/${SLUG}/videos/vidA`);
+ await page
+ .getByRole("button", { name: "mark video vidA do not clean" })
+ .click();
+ await expect(page.getByLabel("media archived")).toBeVisible();
+
+ await generateReport(page, SLUG);
+ await page.goto(`/channels/${SLUG}`);
+
+ const [aBytes, bBytes, cBytes, dBytes] = await Promise.all([
+ audioBytes("vidA"),
+ audioBytes("vidB"),
+ audioBytes("vidC"),
+ audioBytes("vidD"),
+ ]);
+ const snap = await readJson<HoldSnapshot>(SNAPSHOT_REL);
+ const held = snap.heldAudioBytes;
+ const counts = snap.heldAudioCounts;
+ if (!held || !counts) throw new Error("snapshot carries no hold accounting");
+
+ // One gate each, by the sweep's order.
+ expect(held.noTranscript).toBe(bBytes);
+ expect(held.doNotClean).toBe(aBytes);
+ expect(held.awaitingDiarization).toBe(cBytes);
+ expect(held.keepLatest).toBe(0);
+ expect(counts.noTranscript).toBe(1);
+ expect(counts.doNotClean).toBe(1);
+ expect(counts.awaitingDiarization).toBe(1);
+
+ // FIRST GATE WINS. vidA is both pinned and undiarized; it is counted once,
+ // under the pin — counting it twice would report a hold bigger than the disk.
+ expect(counts.doNotClean + counts.awaitingDiarization).toBe(2);
+
+ // The permanent hold: the diarize lane is not running at all, so every byte
+ // it holds is a byte no queued job will ever release.
+ expect(held.diarizationNeverClears).toBe(cBytes);
+
+ // THE INVARIANT — the four gates plus the reclaimable remainder are the whole
+ // of the audio on disk.
+ const reclaimable = snap.cleanupBytes?.transcribedWithAudio ?? 0;
+ expect(reclaimable).toBe(dBytes);
+ expect(
+ held.noTranscript +
+ held.keepLatest +
+ held.doNotClean +
+ held.awaitingDiarization +
+ reclaimable,
+ ).toBe(snap.totalAudioBytes);
+ expect(snap.totalAudioBytes).toBe(aBytes + bBytes + cBytes + dBytes);
+
+ // --- The page ------------------------------------------------------------
+ await page.goto("/cleanup");
+ const sieve = page.getByLabel("audio hold sieve");
+ await expect(sieve).toBeVisible();
+
+ const gate = (n: number) => sieve.locator(`li[data-gate="${n}"]`);
+ await expect(gate(1)).toHaveAttribute("data-held-bytes", String(bBytes));
+ await expect(gate(1)).toHaveAttribute("data-held-count", "1");
+ await expect(gate(2)).toHaveAttribute("data-held-bytes", "0");
+ await expect(gate(3)).toHaveAttribute("data-held-bytes", String(aBytes));
+ await expect(gate(4)).toHaveAttribute("data-held-bytes", String(cBytes));
+
+ // A gate that holds nothing shows "—", not "0".
+ await expect(gate(2).getByLabel("keep-latest window held")).toHaveText("—");
+
+ // The permanent hold is stated on the page, not just in the file.
+ await expect(gate(4)).toContainText("will not clear on its own");
+
+ // The remainder steps down to the figure the hero already prints.
+ await expect(sieve.getByLabel("reclaimable after holds")).toHaveText(/4\.0 KB/);
+
+ // And the ledger prescribes the two runs, ranked by held bytes.
+ const ledger = page.getByLabel("release ledger");
+ // The row is labelled by slug and reads as the channel's NAME, so assert the
+ // row exists rather than that the slug is printed in it.
+ await expect(ledger.getByLabel(`no transcript ${SLUG}`)).toHaveAttribute(
+ "data-held-bytes",
+ String(bBytes),
+ );
+ await expect(
+ ledger.getByRole("button", { name: "Diarize", exact: true }),
+ ).toBeVisible();
+});
+
+test("a report written before the hold accounting renders '—', not a zero nobody measured", async ({
+ page,
+}) => {
+ await resetData();
+ await writeChannelConfig(STALE);
+ // Exactly what an older snapshot looks like: reclaim estimate, no accounting.
+ await writeFile(
+ resolvePath(`test-transcripts/channels/${STALE}/snapshot.json`),
+ JSON.stringify({
+ generatedAt: "2026-01-01T00:00:00.000Z",
+ totals: { videos: 1, transcribed: 1, downloaded: 1 },
+ buckets: { transcribedWithAudio: ["vidA"] },
+ cleanupBytes: { transcribedWithAudio: 1024 },
+ }),
+ );
+ await fetch(`${baseUrl}/api/test/invalidate-cache`).catch(() => {});
+
+ await page.goto("/cleanup");
+ const sieve = page.getByLabel("audio hold sieve");
+ await expect(sieve.getByLabel("total audio on disk")).toHaveText("—");
+ // And it says whose measurement is missing rather than implying there is none.
+ await expect(sieve).toContainText("Refresh reports");
+});
