import path from "node:path"; import { readdir, rm, stat } from "node:fs/promises"; import type { Paths } from "../lib/paths"; import { CLIPS_DIR_NAME, clipWindowSidecar, parseClipWindowName, } from "../lib/clipWindow"; import { inspectChannelMedia } from "../lib/channelMedia"; import { formatBytes } from "../lib/format"; // THE GARBAGE COLLECTION `clips/` NEVER HAD. // // A clip window is a few seconds of a video's source media, fetched on purpose // and kept beside the video it came from so the next tool that wants those // seconds does not spend a source's patience again. Nothing has ever removed // one: the retention sweep is POINTER-driven (`pruneSavedVideos` walks saved // video pointers and never sees a `clips/` file), and the cleanup lanes are // about `audio.*`. So a channel walked by many reports accumulates windows // silently, on a platter, forever. // // EVICTION IS BY AGE, AND THAT IS A LIMITATION, NOT A DESIGN PREFERENCE. // Whether a window is still WANTED is a fact about a umtool manifest — a report // being rendered to video cites spans, and `report-to-video` resolves each one // against whatever windows exist. The editor cannot see those manifests: they // live in a umtool project, possibly on another machine, possibly not written // yet. There is no reference count to consult and no honest way to invent one. // // So the rule is "older than N days" and EVERY SURFACE MUST SAY SO. An operator // evicting a window a report still cites has not lost data — the window is // re-fetchable, which is the whole reason it is safe to treat as a cache — but // they have spent a fetch, and they are entitled to know that before clicking // rather than after. // // THE MTIME IS THE AGE, not the provenance sidecar's `fetchedAt`. A window with // no sidecar (a fetch interrupted after the media landed) would otherwise be // un-evictable forever, which is exactly backwards: an orphan is the single // most evictable thing in the directory. `rsync -a` preserves mtimes, so a // relocated channel's windows carry their real ages across a move. export type EvictClipWindowsOptions = { paths: Paths; // One channel, or every channel when absent. slug?: string; // A window is evicted when it has not been touched for this many days. // 0 means "everything", which is a legitimate ask (the operator is emptying // a drive) and is why this is not clamped to a minimum. olderThanDays: number; // Report what WOULD go, delete nothing. The /storage control previews with // this before it commits, so the number the operator reads is produced by // the same walk that does the work. dryRun?: boolean; onLog?: (message: string) => void; signal?: AbortSignal; }; export type EvictClipWindowsResult = { dryRun: boolean; // Channels actually walked (a skipped one is not one of these). channels: number; // Video dirs that held at least one window, evicted or not. videos: number; windows: number; bytes: number; // Windows left alone because they are newer than the cutoff. kept: number; keptBytes: number; // `: ` for every channel the walk refused to touch. A skip is // not a failure — it is the answer — so it travels in the result rather than // throwing. skipped: string[]; }; const DAY_MS = 24 * 60 * 60 * 1000; // One channel's windows. Returns the same counters the whole run sums. async function evictChannel( opts: EvictClipWindowsOptions, slug: string, cutoffMs: number, out: EvictClipWindowsResult, ): Promise { const channelDir = path.join(opts.paths.channelsDir, slug); // AN UNREADABLE TEXT TIER IS NOT AN EMPTY CHANNEL, and `inspectChannelMedia` // is the one module that can tell the two apart (AGENTS.md: a path that reads // `data/` guards there). Every enumerator else swallows ENOENT as "no // videos" — which here would report a clean eviction of zero bytes, and an // operator would read that as "nothing to reclaim". // // THE TEXT GUARD (release 17): `clips/` is on the corpus disk and is never // tiered, so a channel whose MEDIA is moving, stalled or unmounted is evicted // as usual — a media move carries `media/`, never `data/`. Skipped: a // `legacy` channel (the retired whole-directory layout, its `data/` — clips // included — on the far drive, where a move's verify compares trees) and a // tier migration in flight, which rebuilds `data/` itself. (The JOB also // declares `needsText`, which covers a single-channel run before it starts; // this covers the corpus-wide one, where there is no slug for that guard.) const media = await inspectChannelMedia(opts.paths, slug, undefined, { fresh: true, }); if (!media.text.readable) { out.skipped.push( `${slug}: media ${media.status}${media.detail ? ` (${media.detail})` : ""} — nothing was touched`, ); return; } const dataDir = path.join(channelDir, "data"); let videoIds: string[]; try { videoIds = await readdir(dataDir); } catch { // Past the guard above this really is "no videos": a channel whose text // tier is on the corpus disk and whose `data/` has never been created. out.channels += 1; return; } out.channels += 1; for (const id of videoIds) { if (opts.signal?.aborted) return; const clipsDir = path.join(dataDir, id, CLIPS_DIR_NAME); let entries: string[]; try { entries = await readdir(clipsDir); } catch { continue; } let sawWindow = false; for (const name of entries) { // THE WINDOWS, NOT THE SIDECARS. A `.json` is removed with the media it // describes and never on its own — walking them independently would let // one pass delete a sidecar whose window survived, and a window with no // provenance is one nobody can explain afterwards. const span = parseClipWindowName(name); if (!span) continue; const file = path.join(clipsDir, name); let st; try { st = await stat(file); } catch { continue; } if (!st.isFile()) continue; sawWindow = true; if (st.mtimeMs > cutoffMs) { out.kept += 1; out.keptBytes += st.size; continue; } out.windows += 1; out.bytes += st.size; opts.onLog?.( `${opts.dryRun ? "would evict" : "evicting"} ${slug}/${id}/${name} ` + `(${formatBytes(st.size)}, last touched ${new Date(st.mtimeMs) .toISOString() .slice(0, 10)})`, ); if (opts.dryRun) continue; await rm(file, { force: true }); // `force` so a window whose sidecar was never written is not an error — // an interrupted fetch leaves exactly that, and it is the most evictable // thing in the directory. await rm(path.join(clipsDir, clipWindowSidecar(span.from, span.to)), { force: true, }); } if (sawWindow) out.videos += 1; // The now-empty `clips/` is left in place deliberately: `rmdir` here would // race a fetch that has just created it, and an empty directory is // invisible to every video-dir enumerator anyway (FACTS: the predicates are // anchored, so a directory named `clips` matches none of them). } } export async function evictClipWindows( opts: EvictClipWindowsOptions, ): Promise { const out: EvictClipWindowsResult = { dryRun: Boolean(opts.dryRun), channels: 0, videos: 0, windows: 0, bytes: 0, kept: 0, keptBytes: 0, skipped: [], }; // REFUSED, NOT CLAMPED, and NaN is why. `Math.max(0, NaN)` is NaN, the cutoff // is then NaN, and `st.mtimeMs > NaN` is false for every file — so a bad // number would not have evicted nothing, it would have evicted EVERYTHING. // The action validates too, but this function is exported and the failure // mode is silent and total. if (!Number.isFinite(opts.olderThanDays) || opts.olderThanDays < 0) { throw new Error( `olderThanDays must be a finite number of days, zero or more ` + `(got ${String(opts.olderThanDays)})`, ); } const days = opts.olderThanDays; const cutoffMs = Date.now() - days * DAY_MS; let slugs: string[]; if (opts.slug) { slugs = [opts.slug]; } else { // `isDirectory()` for the reason channel listing does it: a symlinked // `/` is not a channel (AGENTS.md — only `data/` may be a link), and // a stray file in `channels/` is not one either. const entries = await readdir(opts.paths.channelsDir, { withFileTypes: true, }).catch(() => []); slugs = entries .filter((e) => e.isDirectory()) .map((e) => e.name) .sort(); } for (const slug of slugs) { if (opts.signal?.aborted) break; await evictChannel(opts, slug, cutoffMs, out); } return out; } // The one sentence every surface reports a run with, so the job log, the page // and the ops API cannot word it three ways. export function evictClipWindowsSummary(r: EvictClipWindowsResult): string { const head = r.dryRun ? `Would evict ${r.windows} window(s), ${formatBytes(r.bytes)}` : `Evicted ${r.windows} window(s), ${formatBytes(r.bytes)}`; return ( `${head} across ${r.videos} video(s) in ${r.channels} channel(s); ` + `${r.kept} newer window(s) (${formatBytes(r.keptBytes)}) left alone.` + (r.skipped.length ? ` Skipped: ${r.skipped.join("; ")}` : "") ); }