import { getPaths } from "yt-dlp-transcript-common/lib/paths"; import { getSettings } from "yt-dlp-transcript-common/lib/settings"; import { getRegistry } from "yt-dlp-transcript-common/jobs/registry"; import { getFreeBytes } from "yt-dlp-transcript-common/lib/diskSpace"; import { udisksctlAvailable } from "yt-dlp-transcript-common/lib/storageVolumes"; import { healthTimings, isDriveNotAnswering, notAnsweringText, onDrive, stalledLocation, } from "yt-dlp-transcript-common/lib/storageHealth"; import { clearRuleText } from "yt-dlp-transcript-common/lib/storageHealthTimings"; import { listChannelBriefs } from "yt-dlp-transcript-common/controller/channels"; import { channelsOnLocation, probeAllLocations, } from "yt-dlp-transcript-common/controller/storageLocations"; import { measureTreeCached } from "./lib/measureStore"; import { inspectSavedVideosStore, savedVideosMarkerPath, } from "yt-dlp-transcript-common/controller/relocateSavedVideos"; import { pathExists } from "yt-dlp-transcript-common/controller/relocateDir"; import { buildStorageRows, type StorageRowsPayload, } from "yt-dlp-transcript-common/views/storage"; // THE SHELL. `common/views/storage.ts` is pure and takes every live fact as an // argument; this is the file that actually goes and gets them — settings off // disk, one probe per location (memoised for 10 s, so a double render is one // set of subprocesses), the per-channel roll-up, the registry, and the clock. // // SERVER ONLY. It reaches `storageVolumes.ts` and the controller, both of which // import execa. Nothing `"use client"` may import this file; `next build` is // what proves it. // NO `refresh` PARAMETER. The page never wants one: a render answers from the // 10 s memo on purpose, and the Refresh button is a server ACTION that calls // `probeLocationMemo(..., { refresh: true })` itself and then revalidates. An // option nothing passes is a claim about a path that does not exist. export async function buildStorage(): Promise { const paths = getPaths(); const settings = getSettings(); const locations = settings.storage.locations; // THE SIZES COME OFF THE REPORTS, not off a walk. `listChannelBriefs` is one // config + one snapshot per channel (6.5 MB of JSON across 71 channels on the // production corpus) and it is what /channels already pays; walking 523 GB of // media to size this page would be the opposite trade. A channel whose // snapshot predates `totalMediaBytes` contributes to `unknownBytes` and the // row says so rather than under-reporting. const briefs = await listChannelBriefs(paths); // THE THREE TIERS OFF EACH REPORT (release 17): the media tier (what a // location holds), the text tier and the clip windows (on the corpus volume // whatever the media's location). A report written before release 17 has no // text figure: unknown, never 0. const mediaBytes: Record = {}; const clipsBytes: Record = {}; const textBytes: Record = {}; const corpus = { textBytes: 0, clipsBytes: 0, unknown: 0, legacy: 0 }; for (const b of briefs) { mediaBytes[b.slug] = b.snapshot?.totalMediaBytes; clipsBytes[b.slug] = b.snapshot?.totalClipsBytes; textBytes[b.slug] = b.snapshot?.totalTextBytes; // A LEGACY channel's text and clips are not on the corpus volume at all // (they are on its retired `dataDir`, and its report cannot be refreshed // until it is migrated): counted as "to migrate", never added here and // never "unmeasured" (review N6). if (b.config.dataDir?.trim()) { corpus.legacy += 1; continue; } const text = b.snapshot?.totalTextBytes; const clips = b.snapshot?.totalClipsBytes; if (typeof text === "number") corpus.textBytes += text; if (typeof clips === "number") corpus.clipsBytes += clips; // Unmeasured on the corpus volume: a report with no text figure — except // an in-place channel whose MEDIA is unmeasured too, which the internal // rollup already counts (one channel, one "unmeasured"). const inPlace = !b.config.mediaDir?.trim(); if ( typeof text !== "number" && !(inPlace && typeof b.snapshot?.totalMediaBytes !== "number") ) { corpus.unknown += 1; } } const configs = briefs.map((b) => ({ slug: b.slug, config: b.config })); const [probes, rollups, udisksctl, freeOnCorpus] = await Promise.all([ probeAllLocations(locations, paths), channelsOnLocation({ paths, locations, configs, mediaBytes, clipsBytes, textBytes, includeInternal: true, }), // Memoised per binary path inside storageVolumes, so this is one // `--version` for the life of the process. locations.length > 0 ? udisksctlAvailable(paths) : Promise.resolve(false), getFreeBytes(paths.channelsDir), ]); // THE STORE IS WALKED, and that is affordable because of what it holds: one // persisted container per PINNED or kept-latest video, not one per video. It // is a few dozen files on the production corpus, against 523 GB of channel // media that is never walked here (that comes off the reports). If the store // ever grows to corpus scale, this is the line that has to change — and // `listSavedVideos` (which reads the pointers, each carrying its own `bytes`) // is the cheaper answer waiting. // // MEMOIZED FOR 60 SECONDS (see lib/measureStore.ts). This page is // `force-dynamic` and the global AutoRefresh re-renders it on a timer, so an // un-cached walk got slower exactly as the store grew — which is the // situation the operator opened the page to understand. The store's SIZE is // the only thing cached; its location, status and marker are read fresh every // render, because those are the safety facts. // THE DRIVES THAT ARE NOT ANSWERING, in words. From memory: the health pass // (the block device's counters, every `storage.health.passIntervalMs`) or the // watchdog on a page's read is what found it (lib/storageHealth.ts). const now = Date.now(); const clears = clearRuleText(healthTimings().clearAfterCleanPasses); const notAnswering: Record = {}; for (const loc of locations) { const stall = stalledLocation(loc); if (stall) { const watched = stall.detector === "counters" ? " Watched through its disk's request counters." : stall.detector === "stat" ? " Watched with a stat of its root (no disk could be named here)." : ""; notAnswering[loc.id] = `${notAnsweringText(stall, now)} — ${stall.cause ?? "its root did not answer"}. ` + `Pages and polls skip this drive until it answers ${clears}.${watched}`; } } const store = await inspectSavedVideosStore(paths, settings); // A store on a location walks that location's drive: through the watchdog, // so a drive that stops answering mid-walk leaves the size at 0 (the store's // status line says why on the next render) instead of holding the page. const storeLocation = locations.find((l) => l.id === store.locationId); let storeMeasured = { bytes: 0, files: 0 }; if (store.status !== "unreachable" && store.status !== "in-transition") { try { storeMeasured = await (storeLocation ? onDrive(storeLocation, () => measureTreeCached(paths.savedVideosDir)) : measureTreeCached(paths.savedVideosDir)); } catch (err) { if (!isDriveNotAnswering(err)) throw err; } } return buildStorageRows({ locations, savedVideos: { dir: store.dir, at: store.target ?? store.dir, locationId: store.locationId, status: store.status, ...(store.detail ? { detail: store.detail } : {}), bytes: storeMeasured.bytes, files: storeMeasured.files, hasMarker: await pathExists(savedVideosMarkerPath(paths)), }, // The corpus volume, always: it is where every unrelocated channel's media // is, and it is the row the operator is actually trying to empty. internal: { root: paths.channelsDir, ...(Number.isFinite(freeOnCorpus) ? { freeBytes: freeOnCorpus } : {}), corpus, }, defaultLocationId: settings.storage.defaultLocationId, probes, notAnswering, rollups, registry: getRegistry(), udisksctlAvailable: udisksctl, now, }); }