commit 828265ccbe471894ee444380fbddb8f100ad4d70
parent 08ae4c72167a60944a87b1bad345d05feb696511
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 21 Sep 2026 02:21:30 -0400
storage: evict fetched clip windows, by age
`data/<id>/clips/` is the one thing in the corpus with no garbage collection.
The retention sweep is pointer-driven and never sees a window; the cleanup lanes
are about `audio.*`. So a channel walked by many reports accumulates windows
silently, on a platter, forever, and until now nothing removed one.
BY AGE, AND THAT IS A LIMITATION, NOT A PREFERENCE. Whether a window is still
wanted is a fact about a umtool manifest — a report being rendered to video
cites spans — and the editor cannot see those manifests: they live in a umtool
project, possibly on another machine, possibly not written yet. There is no
reference count to consult and no honest way to invent one. So every surface
says so: the card's caveat block, the job log's second line, and the route's
header comment. An evicted window is re-fetchable, so getting it wrong costs a
fetch, not data.
The age is the MTIME, not the provenance sidecar's `fetchedAt`: a window whose
sidecar was never written (a fetch interrupted after the media landed) would
otherwise be un-evictable forever, which is exactly backwards — an orphan is the
most evictable thing in the directory. `rsync -a` preserves mtimes, so a
relocated channel's windows carry their real ages across a move.
AN UNMOUNTED DRIVE IS NOT AN EMPTY CHANNEL. The kind declares `needsMedia: true`
(the sharpest case for that flag in the table: this one DELETES), which covers a
single-channel run; the controller re-asks `inspectChannelMedia` per channel for
the corpus-wide run, where there is no slug for `runManagedFunction` to check.
Without it the pass would report a clean eviction of zero bytes about a platter
full of windows. `in-transition` is refused for a sharper reason still: there
may be two copies and the mover's verify pass compares trees, so deleting under
it turns a copy that had verified into a failed move of a channel that was fine.
The preview is the SAME walk (`dryRun`), so the number the operator reads comes
from the code that would do the work rather than a second estimate that can
disagree. The sidecar is only ever removed with the window it describes.
Surfaced as a /storage card (like the saved-video store, and for the same
reason: eviction is not a fact about one location), `POST /api/ops/evict-clips`,
and `evict-clips` in the ops script.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
10 files changed, 790 insertions(+), 0 deletions(-)
diff --git a/common/controller/evictClipWindows.test.ts b/common/controller/evictClipWindows.test.ts
@@ -0,0 +1,220 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ mkdir,
+ mkdtemp,
+ rm,
+ stat,
+ symlink,
+ utimes,
+ writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { evictClipWindows, evictClipWindowsSummary } from "./evictClipWindows";
+import type { Paths } from "../lib/paths";
+
+// `clips/` had no garbage collection at all before this: the retention sweep is
+// pointer-driven and never sees a window, and the cleanup lanes are about
+// `audio.*`. What these pin is the two halves of the rule — by AGE, and never
+// while a channel's media is in transition.
+
+const DAY = 24 * 60 * 60 * 1000;
+
+async function seed(
+ paths: Paths,
+ slug: string,
+ windows: Record<string, { ageDays: number; size: number; sidecar?: boolean }>,
+): Promise<string> {
+ const clipsDir = path.join(paths.channelsDir, slug, "data", "v1", "clips");
+ await mkdir(clipsDir, { recursive: true });
+ for (const [name, spec] of Object.entries(windows)) {
+ const file = path.join(clipsDir, name);
+ await writeFile(file, Buffer.alloc(spec.size));
+ const when = new Date(Date.now() - spec.ageDays * DAY);
+ await utimes(file, when, when);
+ if (spec.sidecar !== false) {
+ const json = file.replace(/\.mp4$/, ".json");
+ await writeFile(json, JSON.stringify({ requestedBy: "umtool" }));
+ await utimes(json, when, when);
+ }
+ }
+ return clipsDir;
+}
+
+async function withTmp(fn: (paths: Paths) => Promise<void>): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-evict-"));
+ const transcriptsDir = path.join(dir, "corpus");
+ const paths = {
+ transcriptsDir,
+ channelsDir: path.join(transcriptsDir, "channels"),
+ } as Paths;
+ await mkdir(paths.channelsDir, { recursive: true });
+ try {
+ await fn(paths);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+const exists = (p: string) =>
+ stat(p).then(
+ () => true,
+ () => false,
+ );
+
+test("an old window goes with its sidecar; a recent one is left alone", async () => {
+ await withTmp(async (paths) => {
+ const clipsDir = await seed(paths, "alpha", {
+ "10.00-40.00.mp4": { ageDays: 90, size: 1000 },
+ "60.00-70.00.mp4": { ageDays: 1, size: 500 },
+ });
+ const r = await evictClipWindows({ paths, olderThanDays: 30 });
+ assert.equal(r.windows, 1);
+ assert.equal(r.bytes, 1000);
+ assert.equal(r.kept, 1);
+ assert.equal(r.keptBytes, 500);
+ assert.equal(r.videos, 1);
+ assert.equal(r.channels, 1);
+ // THE SIDECAR GOES WITH THE MEDIA and never on its own — a window with no
+ // provenance is one nobody can explain afterwards.
+ assert.equal(await exists(path.join(clipsDir, "10.00-40.00.mp4")), false);
+ assert.equal(await exists(path.join(clipsDir, "10.00-40.00.json")), false);
+ assert.equal(await exists(path.join(clipsDir, "60.00-70.00.mp4")), true);
+ assert.equal(await exists(path.join(clipsDir, "60.00-70.00.json")), true);
+ });
+});
+
+test("a dry run reports the same numbers and deletes nothing", async () => {
+ await withTmp(async (paths) => {
+ const clipsDir = await seed(paths, "alpha", {
+ "10.00-40.00.mp4": { ageDays: 90, size: 1000 },
+ });
+ const r = await evictClipWindows({
+ paths,
+ olderThanDays: 30,
+ dryRun: true,
+ });
+ assert.equal(r.dryRun, true);
+ assert.equal(r.windows, 1);
+ assert.equal(r.bytes, 1000);
+ assert.equal(await exists(path.join(clipsDir, "10.00-40.00.mp4")), true);
+ assert.match(evictClipWindowsSummary(r), /^Would evict 1 window/);
+ });
+});
+
+test("a window whose sidecar was never written is still evictable", async () => {
+ // An interrupted fetch leaves exactly this, and it is the single most
+ // evictable thing in the directory — which is why the age is the MTIME and
+ // not the provenance sidecar's `fetchedAt`.
+ await withTmp(async (paths) => {
+ const clipsDir = await seed(paths, "alpha", {
+ "10.00-40.00.mp4": { ageDays: 90, size: 1000, sidecar: false },
+ });
+ const r = await evictClipWindows({ paths, olderThanDays: 30 });
+ assert.equal(r.windows, 1);
+ assert.equal(await exists(path.join(clipsDir, "10.00-40.00.mp4")), false);
+ });
+});
+
+test("a channel whose media is in transition is SKIPPED, not evicted from", async () => {
+ // `.relocating.json` means there may be two copies and the mover's verify
+ // pass compares trees — deleting under it turns a copy that had verified into
+ // a failed move of a channel that was fine.
+ await withTmp(async (paths) => {
+ const clipsDir = await seed(paths, "alpha", {
+ "10.00-40.00.mp4": { ageDays: 90, size: 1000 },
+ });
+ await writeFile(
+ path.join(paths.channelsDir, "alpha", ".relocating.json"),
+ JSON.stringify({
+ target: "/mnt/platter/alpha/data",
+ direction: "out",
+ startedAt: new Date().toISOString(),
+ phase: "copy",
+ }),
+ );
+ const r = await evictClipWindows({ paths, olderThanDays: 30 });
+ assert.equal(r.windows, 0);
+ assert.equal(r.channels, 0);
+ assert.equal(r.skipped.length, 1);
+ assert.match(r.skipped[0], /^alpha: media in-transition/);
+ assert.equal(await exists(path.join(clipsDir, "10.00-40.00.mp4")), true);
+ // A skip is the ANSWER, so it reaches the operator in the summary rather
+ // than being swallowed by a clean-looking zero.
+ assert.match(evictClipWindowsSummary(r), /Skipped: alpha:/);
+ });
+});
+
+test("one slug walks one channel; no slug walks them all", async () => {
+ await withTmp(async (paths) => {
+ await seed(paths, "alpha", {
+ "10.00-40.00.mp4": { ageDays: 90, size: 1000 },
+ });
+ await seed(paths, "beta", {
+ "10.00-40.00.mp4": { ageDays: 90, size: 2000 },
+ });
+ const one = await evictClipWindows({
+ paths,
+ slug: "alpha",
+ olderThanDays: 30,
+ dryRun: true,
+ });
+ assert.equal(one.bytes, 1000);
+ const all = await evictClipWindows({
+ paths,
+ olderThanDays: 30,
+ dryRun: true,
+ });
+ assert.equal(all.bytes, 3000);
+ assert.equal(all.channels, 2);
+ });
+});
+
+test("olderThanDays 0 takes everything, and a file that is not a window is never touched", async () => {
+ await withTmp(async (paths) => {
+ const clipsDir = await seed(paths, "alpha", {
+ "10.00-40.00.mp4": { ageDays: 0, size: 1000 },
+ });
+ // Not a `<from>-<to>` name: nothing this pass owns, so it stays. The walk
+ // is driven by `parseClipWindowName`, not by an extension.
+ await writeFile(path.join(clipsDir, "notes.txt"), "keep me");
+ const r = await evictClipWindows({ paths, olderThanDays: 0 });
+ assert.equal(r.windows, 1);
+ assert.equal(await exists(path.join(clipsDir, "notes.txt")), true);
+ });
+});
+
+test("a channel with no clips/ at all contributes nothing and does not throw", async () => {
+ await withTmp(async (paths) => {
+ await mkdir(path.join(paths.channelsDir, "alpha", "data", "v1"), {
+ recursive: true,
+ });
+ const r = await evictClipWindows({ paths, olderThanDays: 30 });
+ assert.equal(r.channels, 1);
+ assert.equal(r.videos, 0);
+ assert.equal(r.windows, 0);
+ });
+});
+
+test("an unmounted drive is a SKIP, never a clean eviction of nothing", async () => {
+ // Every other enumerator swallows ENOENT on `data/` as "no videos". Here
+ // that would report zero bytes to reclaim about a platter full of windows.
+ await withTmp(async (paths) => {
+ const channelDir = path.join(paths.channelsDir, "alpha");
+ await mkdir(channelDir, { recursive: true });
+ const target = path.join(paths.transcriptsDir, "platter", "alpha", "data");
+ await writeFile(
+ path.join(channelDir, "config.json"),
+ JSON.stringify({ url: "https://example.com/c", dataDir: target }),
+ );
+ // A link with no target: an unmounted drive, exactly.
+ await symlink(target, path.join(channelDir, "data"));
+
+ const r = await evictClipWindows({ paths, olderThanDays: 30 });
+ assert.equal(r.channels, 0);
+ assert.equal(r.windows, 0);
+ assert.equal(r.skipped.length, 1);
+ assert.match(r.skipped[0], /^alpha: media unreachable/);
+ });
+});
diff --git a/common/controller/evictClipWindows.ts b/common/controller/evictClipWindows.ts
@@ -0,0 +1,225 @@
+import path from "node:path";
+import { readdir, rm, stat } from "node:fs/promises";
+import type { Paths } from "../lib/paths";
+import {
+ CLIPS_DIR_NAME,
+ clipWindowSidecar,
+ parseClipWindowName,
+} from "../lib/clipWindow";
+import { inspectChannelMedia } from "../lib/channelMedia";
+import { formatBytes } from "../lib/format";
+
+// THE GARBAGE COLLECTION `clips/` NEVER HAD.
+//
+// A clip window is a few seconds of a video's source media, fetched on purpose
+// and kept beside the video it came from so the next tool that wants those
+// seconds does not spend a source's patience again. Nothing has ever removed
+// one: the retention sweep is POINTER-driven (`pruneSavedVideos` walks saved
+// video pointers and never sees a `clips/` file), and the cleanup lanes are
+// about `audio.*`. So a channel walked by many reports accumulates windows
+// silently, on a platter, forever.
+//
+// EVICTION IS BY AGE, AND THAT IS A LIMITATION, NOT A DESIGN PREFERENCE.
+// Whether a window is still WANTED is a fact about a umtool manifest — a report
+// being rendered to video cites spans, and `report-to-video` resolves each one
+// against whatever windows exist. The editor cannot see those manifests: they
+// live in a umtool project, possibly on another machine, possibly not written
+// yet. There is no reference count to consult and no honest way to invent one.
+//
+// So the rule is "older than N days" and EVERY SURFACE MUST SAY SO. An operator
+// evicting a window a report still cites has not lost data — the window is
+// re-fetchable, which is the whole reason it is safe to treat as a cache — but
+// they have spent a fetch, and they are entitled to know that before clicking
+// rather than after.
+//
+// THE MTIME IS THE AGE, not the provenance sidecar's `fetchedAt`. A window with
+// no sidecar (a fetch interrupted after the media landed) would otherwise be
+// un-evictable forever, which is exactly backwards: an orphan is the single
+// most evictable thing in the directory. `rsync -a` preserves mtimes, so a
+// relocated channel's windows carry their real ages across a move.
+
+export type EvictClipWindowsOptions = {
+ paths: Paths;
+ // One channel, or every channel when absent.
+ slug?: string;
+ // A window is evicted when it has not been touched for this many days.
+ // 0 means "everything", which is a legitimate ask (the operator is emptying
+ // a drive) and is why this is not clamped to a minimum.
+ olderThanDays: number;
+ // Report what WOULD go, delete nothing. The /storage control previews with
+ // this before it commits, so the number the operator reads is produced by
+ // the same walk that does the work.
+ dryRun?: boolean;
+ onLog?: (message: string) => void;
+ signal?: AbortSignal;
+};
+
+export type EvictClipWindowsResult = {
+ dryRun: boolean;
+ // Channels actually walked (a skipped one is not one of these).
+ channels: number;
+ // Video dirs that held at least one window, evicted or not.
+ videos: number;
+ windows: number;
+ bytes: number;
+ // Windows left alone because they are newer than the cutoff.
+ kept: number;
+ keptBytes: number;
+ // `<slug>: <reason>` for every channel the walk refused to touch. A skip is
+ // not a failure — it is the answer — so it travels in the result rather than
+ // throwing.
+ skipped: string[];
+};
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+// One channel's windows. Returns the same counters the whole run sums.
+async function evictChannel(
+ opts: EvictClipWindowsOptions,
+ slug: string,
+ cutoffMs: number,
+ out: EvictClipWindowsResult,
+): Promise<void> {
+ const channelDir = path.join(opts.paths.channelsDir, slug);
+
+ // AN UNMOUNTED DRIVE IS NOT AN EMPTY CHANNEL, and `inspectChannelMedia` is
+ // the one module that can tell the two apart (AGENTS.md: a path that reads
+ // `data/` guards there). Every enumerator else swallows ENOENT as "no
+ // videos" — which here would report a clean eviction of zero bytes about a
+ // platter full of windows, and an operator would read that as "nothing to
+ // reclaim".
+ //
+ // `in-transition` is a refusal for a sharper reason than unreachability:
+ // there may be two copies, `data/` may be a link whose target is half
+ // written, and the mover's verify pass compares trees. Deleting under it
+ // turns a copy that had verified into a failed move of a channel that was
+ // fine. (The JOB also declares `needsMedia: true`, which covers a
+ // single-channel run before it starts; this covers the corpus-wide one,
+ // where there is no slug for that guard to check.)
+ const media = await inspectChannelMedia(opts.paths, slug);
+ if (media.status !== "ok" && media.status !== "in-place") {
+ out.skipped.push(
+ `${slug}: media ${media.status}${media.detail ? ` (${media.detail})` : ""} — nothing was touched`,
+ );
+ return;
+ }
+
+ const dataDir = path.join(channelDir, "data");
+ let videoIds: string[];
+ try {
+ videoIds = await readdir(dataDir);
+ } catch {
+ // Past the guard above this really is "no videos": a channel whose media
+ // is in place and whose `data/` has never been created.
+ out.channels += 1;
+ return;
+ }
+ out.channels += 1;
+
+ for (const id of videoIds) {
+ if (opts.signal?.aborted) return;
+ const clipsDir = path.join(dataDir, id, CLIPS_DIR_NAME);
+ let entries: string[];
+ try {
+ entries = await readdir(clipsDir);
+ } catch {
+ continue;
+ }
+ let sawWindow = false;
+ for (const name of entries) {
+ // THE WINDOWS, NOT THE SIDECARS. A `.json` is removed with the media it
+ // describes and never on its own — walking them independently would let
+ // one pass delete a sidecar whose window survived, and a window with no
+ // provenance is one nobody can explain afterwards.
+ const span = parseClipWindowName(name);
+ if (!span) continue;
+ const file = path.join(clipsDir, name);
+ let st;
+ try {
+ st = await stat(file);
+ } catch {
+ continue;
+ }
+ if (!st.isFile()) continue;
+ sawWindow = true;
+ if (st.mtimeMs > cutoffMs) {
+ out.kept += 1;
+ out.keptBytes += st.size;
+ continue;
+ }
+ out.windows += 1;
+ out.bytes += st.size;
+ opts.onLog?.(
+ `${opts.dryRun ? "would evict" : "evicting"} ${slug}/${id}/${name} ` +
+ `(${formatBytes(st.size)}, last touched ${new Date(st.mtimeMs)
+ .toISOString()
+ .slice(0, 10)})`,
+ );
+ if (opts.dryRun) continue;
+ await rm(file, { force: true });
+ // `force` so a window whose sidecar was never written is not an error —
+ // an interrupted fetch leaves exactly that, and it is the most evictable
+ // thing in the directory.
+ await rm(path.join(clipsDir, clipWindowSidecar(span.from, span.to)), {
+ force: true,
+ });
+ }
+ if (sawWindow) out.videos += 1;
+ // The now-empty `clips/` is left in place deliberately: `rmdir` here would
+ // race a fetch that has just created it, and an empty directory is
+ // invisible to every video-dir enumerator anyway (FACTS: the predicates are
+ // anchored, so a directory named `clips` matches none of them).
+ }
+}
+
+export async function evictClipWindows(
+ opts: EvictClipWindowsOptions,
+): Promise<EvictClipWindowsResult> {
+ const out: EvictClipWindowsResult = {
+ dryRun: Boolean(opts.dryRun),
+ channels: 0,
+ videos: 0,
+ windows: 0,
+ bytes: 0,
+ kept: 0,
+ keptBytes: 0,
+ skipped: [],
+ };
+ const days = Math.max(0, opts.olderThanDays);
+ const cutoffMs = Date.now() - days * DAY_MS;
+
+ let slugs: string[];
+ if (opts.slug) {
+ slugs = [opts.slug];
+ } else {
+ // `isDirectory()` for the reason channel listing does it: a symlinked
+ // `<slug>/` is not a channel (AGENTS.md — only `data/` may be a link), and
+ // a stray file in `channels/` is not one either.
+ const entries = await readdir(opts.paths.channelsDir, {
+ withFileTypes: true,
+ }).catch(() => []);
+ slugs = entries
+ .filter((e) => e.isDirectory())
+ .map((e) => e.name)
+ .sort();
+ }
+
+ for (const slug of slugs) {
+ if (opts.signal?.aborted) break;
+ await evictChannel(opts, slug, cutoffMs, out);
+ }
+ return out;
+}
+
+// The one sentence every surface reports a run with, so the job log, the page
+// and the ops API cannot word it three ways.
+export function evictClipWindowsSummary(r: EvictClipWindowsResult): string {
+ const head = r.dryRun
+ ? `Would evict ${r.windows} window(s), ${formatBytes(r.bytes)}`
+ : `Evicted ${r.windows} window(s), ${formatBytes(r.bytes)}`;
+ return (
+ `${head} across ${r.videos} video(s) in ${r.channels} channel(s); ` +
+ `${r.kept} newer window(s) (${formatBytes(r.keptBytes)}) left alone.` +
+ (r.skipped.length ? ` Skipped: ${r.skipped.join("; ")}` : "")
+ );
+}
diff --git a/common/jobs/jobKinds.ts b/common/jobs/jobKinds.ts
@@ -399,6 +399,23 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
queueKeyStrategy: "custom",
needsMedia: false,
},
+ // FETCHED CLIP WINDOWS, BY AGE. `data/<id>/clips/` had no garbage collection
+ // at all: the retention sweep is pointer-driven and never sees a window, and
+ // the cleanup lanes are about `audio.*`. This is the only thing that removes
+ // one.
+ //
+ // `needsMedia: true`, and it is the sharpest case for the flag in the table:
+ // it DELETES. Against an unmounted drive every `readdir` of `data/` throws
+ // and the walk would report a clean eviction of zero bytes — the operator
+ // would read "nothing to reclaim" about a platter full of windows.
+ "evict-clips": {
+ kind: "evict-clips",
+ label: "Evict fetched windows",
+ drainable: false,
+ replayable: false,
+ queueKeyStrategy: "parallel",
+ needsMedia: true,
+ },
// THE SAVED-VIDEO STORE, ONTO A LOCATION AND BACK. Same mechanism as the
// channel move (relocateDir.ts is literally the same code) over one directory
// that belongs to no channel — so no `channelSlug`, and `needsMedia: false`
diff --git a/editor/app/api/ops/evict-clips/route.ts b/editor/app/api/ops/evict-clips/route.ts
@@ -0,0 +1,31 @@
+import { evictClipWindowsAction } from "../../../storage/actions";
+import { jobResponse, ops, optBool, optString } from "../_lib";
+
+export const dynamic = "force-dynamic";
+
+// POST { olderThanDays: number, slug?: string, dryRun?: boolean }
+// -> { ok: true, jobId }
+//
+// AN ADAPTER, like every route here: one call to the action the /storage button
+// posts. `olderThanDays` is required on purpose — there is no sensible default
+// for a control that deletes, and a route that picked one would be a rule the
+// UI does not have.
+//
+// Eviction is BY AGE. Nothing in the editor can know whether a umtool report
+// still cites a window, so a caller automating this should say so wherever it
+// reports the result — see controller/evictClipWindows.ts.
+export async function POST(request: Request) {
+ return ops(request, ["olderThanDays", "slug", "dryRun"], async (body) => {
+ const raw = body.olderThanDays;
+ const olderThanDays = typeof raw === "number" ? raw : Number.NaN;
+ const slug = optString(body, "slug");
+ const dryRun = optBool(body, "dryRun");
+ return jobResponse(
+ await evictClipWindowsAction({
+ ...(slug ? { slug } : {}),
+ olderThanDays,
+ ...(dryRun ? { dryRun: true } : {}),
+ }),
+ );
+ });
+}
diff --git a/editor/app/storage/actions.ts b/editor/app/storage/actions.ts
@@ -28,6 +28,7 @@ import { savedVideosMarkerPath } from "yt-dlp-transcript-common/controller/reloc
import { channelMediaBusyReason } from "../channels/lib/mediaBusy";
import { enqueueRepointJob } from "./lib/repointJob";
import { enqueueSavedVideosRelocation } from "./lib/savedVideosJob";
+import { enqueueEvictClipWindows } from "./lib/evictClipsJob";
import { savedVideosStoreBusyReason } from "./lib/storeBusy";
// THE SIX THINGS AN OPERATOR MAY DO TO A STORAGE LOCATION.
@@ -383,3 +384,38 @@ export async function clearSavedVideosMarkerAction(): Promise<LocationResult> {
revalidatePath("/storage");
return { ok: true, note: "Marker cleared. Nothing was moved." };
}
+
+// EVICT FETCHED CLIP WINDOWS.
+//
+// `data/<id>/clips/` is the one thing in the corpus with no garbage collection:
+// the retention sweep is pointer-driven and never sees a window, and the
+// cleanup lanes are about `audio.*`. This is the only control that removes one.
+//
+// BY AGE, and the button says so. Whether a window is still wanted is a fact
+// about a umtool manifest — a report being rendered to video cites spans — and
+// the editor cannot see those manifests. There is no reference count to consult
+// and no honest way to invent one, so the rule is "older than N days" and the
+// operator is told that before they click, not after.
+//
+// NO BUSY CHECK HERE, deliberately: `evict-clips` declares `needsMedia: true`,
+// so `runManagedFunction` refuses a per-channel run whose media is unreachable
+// or mid-relocation, and the controller re-asks the same question per channel
+// for the corpus-wide run (where there is no slug for that guard to check).
+// A second opinion in an action is the thing the ops API's header warns about.
+export async function evictClipWindowsAction(opts: {
+ slug?: string;
+ olderThanDays: number;
+ dryRun?: boolean;
+}): Promise<StreamActionResult> {
+ if (!Number.isFinite(opts.olderThanDays) || opts.olderThanDays < 0) {
+ return {
+ ok: false,
+ error: "The age must be a number of days, zero or more.",
+ };
+ }
+ return enqueueEvictClipWindows({
+ ...(opts.slug ? { slug: opts.slug } : {}),
+ olderThanDays: Math.floor(opts.olderThanDays),
+ ...(opts.dryRun ? { dryRun: true } : {}),
+ });
+}
diff --git a/editor/app/storage/components/ClipWindowsCard.tsx b/editor/app/storage/components/ClipWindowsCard.tsx
@@ -0,0 +1,114 @@
+"use client";
+
+import { useState } from "react";
+import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog";
+import { formatBytes } from "yt-dlp-transcript-common/lib/format";
+import { cancelJobAction } from "../../jobs/actions";
+import { evictClipWindowsAction } from "../actions";
+
+// FETCHED CLIP WINDOWS — the one thing in the corpus nothing prunes.
+//
+// A window is a few seconds of a video's source media, fetched so the next tool
+// that wants those seconds does not spend a source's patience again. The
+// retention sweep is pointer-driven and never sees one; the cleanup lanes are
+// about `audio.*`. So they accumulate on the platter, forever, and until this
+// card there was no control that removed one.
+//
+// A CARD RATHER THAN A ROW ACTION, for the reason the saved-video store gets
+// one: eviction is not a fact about a location. It is corpus-wide by default
+// (the windows are wherever the channels are, on every drive at once), and the
+// figure it acts on is already in each row's Media line.
+//
+// THE LIMITATION IS THE FIRST THING IT SAYS. Whether a window is still wanted
+// is a fact about a umtool manifest — a report being rendered to video cites
+// spans — and the editor cannot see those manifests: they live in a umtool
+// project, possibly on another machine, possibly not written yet. There is no
+// reference count to consult and no honest way to invent one. So the rule is
+// "older than N days", and an operator evicting something a report still cites
+// has spent a fetch, not lost data. They are entitled to know that beforehand.
+//
+// PREVIEW FIRST, and the preview is the same walk. `dryRun` runs the identical
+// pass and deletes nothing, so the number in the log is produced by the code
+// that would do the work rather than by a second estimate that can disagree.
+//
+// ⚠️ Both panels are rendered UNCONDITIONALLY and only `disabled` changes —
+// `StreamActionLog` calls router.refresh() the instant a run ends, and a panel
+// unmounted by its own result takes its log with it (plans/FACTS.md).
+
+const AGES = [0, 7, 30, 90, 180] as const;
+
+export function ClipWindowsCard({ clipsBytes }: { clipsBytes: number }) {
+ const [days, setDays] = useState<number>(30);
+ return (
+ <article
+ aria-label="clip windows"
+ className="flex flex-col gap-3 rounded-xl border border-border bg-card px-4 py-3"
+ >
+ <div className="flex flex-wrap items-center gap-3">
+ <h2 className="text-base font-semibold">Fetched clip windows</h2>
+ <span
+ aria-label="clip windows bytes"
+ className="text-sm text-muted-foreground tabular-nums"
+ >
+ {clipsBytes > 0
+ ? `${formatBytes(clipsBytes)} across the corpus`
+ : "none measured"}
+ </span>
+ </div>
+
+ <p className="text-sm text-muted-foreground max-w-3xl">
+ A window is a few seconds of a video’s source media, fetched for
+ another tool and kept beside the video it came from. Nothing prunes one:
+ the retention sweep is pointer-driven and the cleanup lanes are about{" "}
+ <code>audio.*</code>. They are already counted in each location’s
+ Media figure above.
+ </p>
+ <p
+ aria-label="clip eviction caveat"
+ className="text-sm rounded border border-border bg-muted px-3 py-2 max-w-3xl"
+ >
+ <strong>Eviction is by age only.</strong> Nothing here can know whether
+ a report still cites a window — those manifests live in umtool projects
+ this editor cannot see. An evicted window is re-fetchable, so the cost
+ of getting this wrong is one fetch, not data. Preview first.
+ </p>
+
+ <label className="flex items-center gap-2 text-sm">
+ <span className="text-muted-foreground">Older than</span>
+ <select
+ aria-label="clip eviction age"
+ value={days}
+ onChange={(e) => setDays(Number(e.target.value))}
+ className="rounded border border-border bg-background px-2 py-1 text-sm"
+ >
+ {AGES.map((d) => (
+ <option key={d} value={d}>
+ {d === 0 ? "any age (everything)" : `${d} days`}
+ </option>
+ ))}
+ </select>
+ </label>
+
+ <div className="flex flex-wrap items-start gap-4">
+ <StreamActionLog
+ key="evict-clips-preview-log"
+ trigger={() =>
+ evictClipWindowsAction({ olderThanDays: days, dryRun: true })
+ }
+ cancelAction={cancelJobAction}
+ buttonLabel="Preview eviction"
+ runningLabel="Walking…"
+ label="Preview eviction"
+ />
+ <StreamActionLog
+ key="evict-clips-log"
+ trigger={() => evictClipWindowsAction({ olderThanDays: days })}
+ cancelAction={cancelJobAction}
+ buttonLabel="Evict fetched windows"
+ runningLabel="Evicting…"
+ label="Evict fetched windows"
+ />
+ </div>
+ </article>
+ );
+}
diff --git a/editor/app/storage/components/StorageLocationsTable.tsx b/editor/app/storage/components/StorageLocationsTable.tsx
@@ -21,6 +21,7 @@ import {
} from "../actions";
import { LocationForm } from "./LocationForm";
import { SavedVideosStoreCard } from "./SavedVideosStoreCard";
+import { ClipWindowsCard } from "./ClipWindowsCard";
// THE LOCATIONS, ONE CARD EACH, AND WHAT MAY BE DONE TO THEM.
//
@@ -76,6 +77,16 @@ export function StorageLocationsTable({ payload }: { payload: StorageRowsPayload
<SavedVideosStoreCard store={payload.savedVideos} />
)}
+ {/* THE OTHER CORPUS-WIDE PILE OF BYTES, beside the store and for the same
+ reason: it is not a fact about any one location (windows are wherever
+ the channels are, on every drive at once) and it is the only control
+ that removes one. The total is the rows' own clip figures summed —
+ computed here rather than carried on the payload, because it is a
+ fold of numbers the payload already has. */}
+ <ClipWindowsCard
+ clipsBytes={payload.rows.reduce((n, r) => n + r.clipsBytes, 0)}
+ />
+
<section className="flex flex-col gap-2">
<h2 className="text-base font-semibold">Add a location</h2>
{adding ? (
diff --git a/editor/app/storage/lib/evictClipsJob.ts b/editor/app/storage/lib/evictClipsJob.ts
@@ -0,0 +1,70 @@
+import { revalidatePath } from "next/cache";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import {
+ runManagedFunction,
+ type StreamActionResult,
+} from "yt-dlp-transcript-common/jobs/streamCommand";
+import {
+ evictClipWindows,
+ evictClipWindowsSummary,
+} from "yt-dlp-transcript-common/controller/evictClipWindows";
+
+// ONE ENQUEUE OF THE CLIP-WINDOW EVICTION, for the callers that have one: the
+// control on /storage and the `/api/ops/evict-clips` adapter.
+//
+// Deliberately NOT in `actions.ts`: that file carries "use server", so every
+// non-type export in it is a server action, and a shared helper exported from
+// there would put an unguarded enqueue on the wire under its own endpoint. Same
+// reason `lib/repointJob.ts` and `lib/savedVideosJob.ts` exist.
+//
+// A JOB AND NOT AN INLINE ACTION, unlike the five small things on /storage: a
+// corpus-wide pass is a readdir of every video directory on the machine
+// (eleven thousand on the largest channel here), it DELETES, and the operator
+// is entitled to a record and a log naming every file that went.
+//
+// `channelSlug` is set only for a one-channel run. That is what makes the
+// snapshot regen land where it should: the run changes `totalMediaBytes` and
+// `totalClipsBytes`, so the report IS stale afterwards — and a corpus-wide run
+// deliberately regenerates nothing rather than queueing a full walk of every
+// channel behind itself.
+export async function enqueueEvictClipWindows(opts: {
+ slug?: string;
+ olderThanDays: number;
+ dryRun?: boolean;
+}): Promise<StreamActionResult> {
+ const paths = getPaths();
+ return runManagedFunction({
+ kind: "evict-clips",
+ // Parallel: it touches `clips/`, which no other kind reads or writes, and
+ // the `needsMedia` guard is what keeps it off a channel mid-relocation.
+ queueKey: "",
+ paths,
+ ...(opts.slug ? { channelSlug: opts.slug } : {}),
+ fn: async (onLog, signal) => {
+ onLog(
+ `${opts.dryRun ? "Previewing" : "Evicting"} clip windows ` +
+ `${opts.slug ? `in ${opts.slug}` : "across every channel"} ` +
+ `not touched for ${opts.olderThanDays} day(s).`,
+ );
+ // SAY THE LIMITATION IN THE LOG, not only in the button's help text. A
+ // window a umtool report still cites is indistinguishable from one
+ // nothing wants — the manifests live in a umtool project the editor
+ // cannot see — so eviction is by age and the record says so.
+ onLog(
+ "By age only: nothing here can know whether a report still cites a " +
+ "window. An evicted one is re-fetchable; it costs a fetch, not data.",
+ );
+ const result = await evictClipWindows({
+ paths,
+ ...(opts.slug ? { slug: opts.slug } : {}),
+ olderThanDays: opts.olderThanDays,
+ ...(opts.dryRun ? { dryRun: true } : {}),
+ onLog,
+ signal,
+ });
+ onLog(evictClipWindowsSummary(result));
+ revalidatePath("/storage");
+ if (opts.slug) revalidatePath(`/channels/${opts.slug}`);
+ },
+ });
+}
diff --git a/editor/e2e/storage-locations.spec.ts b/editor/e2e/storage-locations.spec.ts
@@ -5,6 +5,7 @@ import {
rename,
rm,
symlink,
+ utimes,
writeFile,
} from "node:fs/promises";
import { join } from "node:path";
@@ -558,3 +559,67 @@ test("a store move to an unmounted root refuses before it creates anything", asy
}>("test-settings.json");
expect(settings.storage.savedVideosLocationId ?? "").toBe("");
});
+
+// EVICTING FETCHED CLIP WINDOWS.
+//
+// `data/<id>/clips/` is the one thing in the corpus with no garbage collection:
+// the retention sweep is pointer-driven and never sees a window, and the
+// cleanup lanes are about `audio.*`. This is the only control that removes one,
+// and it evicts BY AGE — nothing in the editor can know whether a umtool report
+// still cites a window, which is why the card says so and why the assertion
+// below is that the recent one SURVIVES.
+test("Evict fetched windows removes an old one and leaves a recent one", async ({
+ page,
+}) => {
+ test.setTimeout(120_000);
+ await resetData("one-youtube-channel-with-data");
+ const clipsDir = resolvePath(
+ `test-transcripts/channels/${SLUG}/data/${VIDEO}/clips`,
+ );
+ await mkdir(clipsDir, { recursive: true });
+ const old = join(clipsDir, "10.00-40.00.mp4");
+ const recent = join(clipsDir, "60.00-70.00.mp4");
+ for (const [file, ageDays] of [
+ [old, 90],
+ [recent, 1],
+ ] as const) {
+ await writeFile(file, "x".repeat(1024));
+ await writeFile(
+ file.replace(/\.mp4$/, ".json"),
+ JSON.stringify({ requestedBy: "umtool", reason: "e2e" }),
+ );
+ const when = new Date(Date.now() - ageDays * 24 * 60 * 60 * 1000);
+ await utimes(file, when, when);
+ }
+ await writeSettings({ adminTitle: "Test Admin", minFreeDiskGB: 0 });
+
+ await page.goto("/storage");
+ const card = page.getByLabel("clip windows");
+ await expect(card).toBeVisible();
+ // THE LIMITATION IS ON THE PAGE, not only in a comment. An operator about to
+ // delete a cache they cannot reference-count is entitled to read why.
+ await expect(page.getByLabel("clip eviction caveat")).toContainText(
+ "by age only",
+ );
+
+ // --- preview deletes nothing -------------------------------------------
+ await page.getByLabel("clip eviction age").selectOption("30");
+ await page.getByRole("button", { name: "Preview eviction" }).click();
+ await expect(page.getByLabel("Preview eviction output")).toContainText(
+ /Would evict 1 window/,
+ { timeout: 60_000 },
+ );
+ expect(await pathExists(old)).toBe(true);
+
+ // --- and then it does ---------------------------------------------------
+ await page.getByRole("button", { name: "Evict fetched windows" }).click();
+ await expect(page.getByLabel("Evict fetched windows output")).toContainText(
+ /Evicted 1 window/,
+ { timeout: 60_000 },
+ );
+ // The sidecar goes WITH the window it describes, never on its own.
+ expect(await pathExists(old)).toBe(false);
+ expect(await pathExists(old.replace(/\.mp4$/, ".json"))).toBe(false);
+ expect(await pathExists(recent)).toBe(true);
+ expect(await pathExists(recent.replace(/\.mp4$/, ".json"))).toBe(true);
+});
diff --git a/scripts/archilyzer-ops.mjs b/scripts/archilyzer-ops.mjs
@@ -62,6 +62,7 @@ const ACTIONS = [
"build-site",
"relocate",
"relocate-back",
+ "evict-clips",
"lane",
];
