commit 93d0eefb55df91b60f68c9d01f0593bfd882b801
parent 45a37f16b987efe990b0ab460a84ff0ab1039931
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 21 Sep 2026 02:00:28 -0400
snapshot: clips/ is bytes on the platter, so count them
`data/<id>/clips/` — the clip windows umtool and the video page fetch a few
seconds at a time — was counted by nothing. The `mediaBytes` loop did
`if (!st.isFile()) continue` under a comment asserting "a video dir is flat",
which the clip-window feature made untrue the day it landed (FACTS: "clips/ HAS
NO GARBAGE COLLECTION, and its bytes are counted by nothing"). Meanwhile
`rsync -a` of the data dir carries the windows, so a relocation moved bytes the
estimate never priced.
THE DECISION: `totalMediaBytes` GROWS to include clips. It is defined as every
byte under `data/<id>/` — what the volume is holding and what a move carries —
and a window is exactly that. `totalClipsBytes` is a new optional "of which",
not a sibling: the two overlap on purpose. Split out because clips are the one
part of a channel's bytes that is a CACHE (nothing prunes a window), so an
operator looking at a row wants to know which of the two the number is.
The risk this carries, stated: `totalMediaBytes` is read by the /storage rows,
the /channels Size column, the free-up selection and the relocation preflight.
Every one of them now sees a slightly larger, and correct, number for a channel
that has been walked by a report. Nothing partitions on it, so nothing breaks;
the numbers move.
`totalClipsBytes` is optional on the same terms as `totalMediaBytes`: absent
means "unknown until Refresh report", never 0 (a fresh snapshot always writes
it, including 0).
The recursion is ONE LEVEL and ONLY into `clips/`: `clipWindow-server.ts`
writes `<from>-<to>.<ext>` and `<from>-<to>.json` flat and nothing else does, so
anything deeper would be machinery for a case that cannot happen. Every other
directory still counts nothing — a blanket recursion here would be the second
walk this loop exists to avoid.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
2 files changed, 128 insertions(+), 3 deletions(-)
diff --git a/common/controller/channelSnapshot.test.ts b/common/controller/channelSnapshot.test.ts
@@ -358,3 +358,65 @@ test("generateChannelSnapshot refuses an unreachable channel rather than writing
await rm(dir, { recursive: true, force: true });
}
});
+
+test("a video dir's clips/ counts into totalMediaBytes AND into totalClipsBytes", async () => {
+ // `clips/` is the one subdirectory a video dir has, and until this it was
+ // counted by nothing: the loop did `if (!st.isFile()) continue` under a
+ // comment saying a video dir is flat, which the clip-window feature made
+ // untrue. The bytes were on the platter and `rsync -a` carried them, so
+ // /storage and every relocation estimate under-reported a channel that had
+ // been walked by a report.
+ //
+ // The two numbers OVERLAP on purpose: totalMediaBytes is every byte under
+ // data/<id>/, totalClipsBytes is the "of which".
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-snap-clips-"));
+ try {
+ const paths = { channelsDir: path.join(dir, "channels") } as Paths;
+ const channelDir = path.join(paths.channelsDir, "alpha");
+ const videoDir = path.join(channelDir, "data", "v1");
+ await mkdir(path.join(videoDir, "clips"), { recursive: true });
+ // A directory that is NOT clips/ still counts nothing — no blanket recursion.
+ await mkdir(path.join(videoDir, "scratch"), { recursive: true });
+ await writeFile(
+ path.join(channelDir, "config.json"),
+ JSON.stringify({ url: "https://example.com/c" }),
+ );
+ await writeFile(path.join(channelDir, "playlist"), "");
+ await writeFile(path.join(videoDir, "audio.m4a"), Buffer.alloc(100));
+ await writeFile(path.join(videoDir, "scratch", "junk.bin"), Buffer.alloc(999));
+ const meta = JSON.stringify({ id: "v1" });
+ await writeFile(path.join(videoDir, "metadata.info.json"), meta);
+ await writeFile(path.join(videoDir, "clips", "1.00-3.00.mp4"), Buffer.alloc(50));
+ await writeFile(path.join(videoDir, "clips", "1.00-3.00.json"), Buffer.alloc(7));
+
+ const snap = await generateChannelSnapshot(paths, "alpha");
+ assert.equal(snap.totalClipsBytes, 57);
+ assert.equal(snap.totalMediaBytes, 100 + meta.length + 57);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+});
+
+test("a channel with no clips/ reports 0 clip bytes, not undefined", async () => {
+ // A fresh snapshot always carries the field; only a snapshot written BEFORE
+ // the field existed lacks it, which is the case readers render as "unknown".
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-snap-noclips-"));
+ try {
+ const paths = { channelsDir: path.join(dir, "channels") } as Paths;
+ const channelDir = path.join(paths.channelsDir, "alpha");
+ const videoDir = path.join(channelDir, "data", "v1");
+ await mkdir(videoDir, { recursive: true });
+ await writeFile(
+ path.join(channelDir, "config.json"),
+ JSON.stringify({ url: "https://example.com/c" }),
+ );
+ await writeFile(path.join(channelDir, "playlist"), "");
+ await writeFile(path.join(videoDir, "audio.m4a"), Buffer.alloc(100));
+
+ const snap = await generateChannelSnapshot(paths, "alpha");
+ assert.equal(snap.totalClipsBytes, 0);
+ assert.equal(snap.totalMediaBytes, 100);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+});
diff --git a/common/controller/channelSnapshot.ts b/common/controller/channelSnapshot.ts
@@ -20,6 +20,7 @@ import {
} from "../lib/availability";
import { resolveCookiePolicy } from "../lib/cookiePolicy";
import { assertChannelMediaReachable } from "../lib/channelMedia";
+import { CLIPS_DIR_NAME } from "../lib/clipWindow";
import { getSettings } from "../lib/settings";
import {
loadAvailability,
@@ -329,6 +330,19 @@ export type ChannelSnapshot = {
// until Refresh report", never as 0 — a zero would rank a 400 GB channel
// bottom of a "free up N GB" list.
totalMediaBytes?: number;
+ // OF WHICH: the bytes held by `data/<id>/clips/` — the clip windows umtool
+ // and the video page fetch a few seconds at a time. A SUBSET of
+ // `totalMediaBytes`, not a sibling of it: a window lives under the video dir,
+ // `rsync -a` carries it with everything else, and a volume is holding it.
+ //
+ // Split out because it is the one part of a channel's bytes that is a CACHE:
+ // nothing prunes a window, so a channel walked by many reports accumulates
+ // them, and an operator looking at a row wants to know which of those two
+ // things the number is. `evictClipWindows` is what acts on it.
+ //
+ // Optional on the same terms as `totalMediaBytes`: absent means "unknown
+ // until Refresh report", never 0.
+ totalClipsBytes?: number;
// WHY the audio that isn't reclaimable isn't reclaimable, in the sweep's own
// order (cleanAudioFromTranscribed's discover loop). A video leaves at the
// FIRST gate it hits, so these are an attribution and never overlapping sets:
@@ -629,6 +643,29 @@ export { SNAPSHOT_FILENAME, snapshotPath, readChannelSnapshot };
const SNAPSHOT_VIDEO_CONCURRENCY = 16;
+// ONE LEVEL of a video dir's `clips/` — the files in it, and nothing deeper.
+// Deliberately not recursive: `clipWindow-server.ts` writes `<from>-<to>.<ext>`
+// and `<from>-<to>.json` flat into it and nothing else does, so a recursion
+// would be machinery for a case that cannot happen. A missing dir is 0.
+async function dirFileBytes(dir: string): Promise<number> {
+ let total = 0;
+ let entries: string[];
+ try {
+ entries = await readdir(dir);
+ } catch {
+ return 0;
+ }
+ for (const name of entries) {
+ try {
+ const st = await stat(path.join(dir, name));
+ if (st.isFile()) total += st.size;
+ } catch {
+ // ignore — vanished mid-walk
+ }
+ }
+ return total;
+}
+
async function readPlaylistUrls(file: string): Promise<string[]> {
try {
const raw = await readFile(file, "utf8");
@@ -766,15 +803,34 @@ export async function generateChannelSnapshot(
// and the audio stats this loop replaces were being paid anyway. What
// it adds is a stat per NON-audio entry — six to ten per video, warm
// inode cache, on a pass that already reads several sidecars per video.
- // Sub-directories are counted as nothing rather than recursed: a video
- // dir is flat, and a walk here would be the second walk this avoids.
+ //
+ // A VIDEO DIR IS NO LONGER FLAT, and exactly one subdirectory is the
+ // reason: `clips/` (CLIPS_DIR_NAME), the fetched clip windows. It is
+ // recursed ONE LEVEL — the windows and their `.json` sidecars are files,
+ // and nothing writes a directory under it — and its bytes are counted
+ // BOTH into `clipsBytes` and into `mediaBytes`. Every OTHER directory
+ // still counts nothing: there is no other one today, and a blanket
+ // recursion here would be the second walk this loop exists to avoid.
+ //
+ // Why clips count toward `mediaBytes` at all: `mediaBytes` is every
+ // byte under `data/<id>/`, which is what the volume is holding and what
+ // `rsync -a` carries in a relocation. A window was being left out of
+ // both numbers while sitting on the platter.
const audioSet = new Set(files.audioFiles);
const audioSizes: Record<string, number> = {};
let mediaBytes = 0;
+ let clipsBytes = 0;
for (const name of files.entries) {
try {
const st = await stat(path.join(dir, name));
- if (!st.isFile()) continue;
+ if (!st.isFile()) {
+ if (st.isDirectory() && name === CLIPS_DIR_NAME) {
+ const bytes = await dirFileBytes(path.join(dir, name));
+ clipsBytes += bytes;
+ mediaBytes += bytes;
+ }
+ continue;
+ }
mediaBytes += st.size;
if (audioSet.has(name)) audioSizes[name] = st.size;
} catch {
@@ -850,6 +906,7 @@ export async function generateChannelSnapshot(
backfill,
audioSizes,
mediaBytes,
+ clipsBytes,
nativeId,
availability,
effectiveAvailability,
@@ -965,6 +1022,9 @@ export async function generateChannelSnapshot(
// surfaces are priced in: how much a volume is holding for this channel, and
// how much a move would carry.
let totalMediaBytes = 0;
+ // The `clips/` share of the above. Counted in BOTH, deliberately: see the
+ // field comment on `totalClipsBytes`.
+ let totalClipsBytes = 0;
const heldAudioBytes = emptyHeldAudio();
const heldAudioCounts = emptyHeldAudio();
let reclaimableAtRiskBytes = 0;
@@ -981,6 +1041,7 @@ export async function generateChannelSnapshot(
files,
audioSizes,
mediaBytes,
+ clipsBytes,
backfill,
effectiveAvailability,
outcome,
@@ -991,6 +1052,7 @@ export async function generateChannelSnapshot(
if (isVideoTranscribed(files)) transcribed++;
if (isVideoDownloaded(files)) downloaded++;
totalMediaBytes += mediaBytes;
+ totalClipsBytes += clipsBytes;
// --- Hold attribution ---------------------------------------------------
// FIRST, before the short-circuits below: they `continue` past videos that
@@ -1447,6 +1509,7 @@ export async function generateChannelSnapshot(
},
totalAudioBytes,
totalMediaBytes,
+ totalClipsBytes,
heldAudioBytes,
heldAudioCounts,
reclaimableAtRiskBytes,
