import path from "node:path"; import { statfs } from "node:fs/promises"; import type { Paths } from "./paths"; import type { SiteSettings } from "./settings"; import { formatBytes } from "./format"; const BYTES_PER_GB = 1024 ** 3; // Free space (bytes) available to an unprivileged process on the filesystem // holding `dir`. statfs reports blocks in `bsize`-sized units; `bavail` excludes // blocks reserved for root, which is what a normal write can actually use. // // A DIRECTORY THAT DOES NOT EXIST YET IS MEASURED ON ITS NEAREST EXISTING // ANCESTOR, and that is the difference between this gate working and this gate // being decorative. Since the gate became per-volume, six callers pass // `channels//data` — the volume the bytes are ABOUT to land on — and that // directory does not exist until the channel's first download creates it // (neither createChannel nor syncPaged makes it, deliberately). A bare statfs // there is ENOENT, the old fail-open answered Infinity, and the gate then waved // through exactly the first download onto a disk it had been asked to protect. // The ancestor is the right answer and not an approximation: a directory about // to be created lands on the filesystem its parent is on. // // Everything else still FAILS OPEN (Infinity): statfs unsupported, permissions, // a nonsense path (ENOTDIR). A measurement glitch must never block a download — // that is the original contract and it is unchanged. Only ENOENT walks, because // only ENOENT means "not there YET"; the walk terminates at the filesystem root, // where dirname is a fixed point. export async function getFreeBytes(dir: string): Promise { let current = path.resolve(dir); for (;;) { try { const stats = await statfs(current); return stats.bsize * stats.bavail; } catch (err) { if ((err as NodeJS.ErrnoException).code !== "ENOENT") { return Number.POSITIVE_INFINITY; } const parent = path.dirname(current); if (parent === current) return Number.POSITIVE_INFINITY; current = parent; } } } export type DiskSpaceStatus = { // Whether the low-disk gate is configured at all (minFreeDiskGB > 0). enabled: boolean; freeBytes: number; thresholdBytes: number; // True when there is enough free space (or the gate is disabled). When false, // downloads should be prevented from starting and a running batch should stop // starting new videos. ok: boolean; }; // Compare free space on the filesystem holding `dir` against the configured // floor. When the gate is disabled (minFreeDiskGB === 0) this always reports ok // and skips the statfs syscall entirely (it runs before every video in a batch // and on every active-jobs poll, and a disabled gate hides the indicator anyway). export async function checkDiskSpaceFor( dir: string, settings: SiteSettings, ): Promise { const enabled = settings.minFreeDiskGB > 0; const thresholdBytes = settings.minFreeDiskGB * BYTES_PER_GB; if (!enabled) { return { enabled, freeBytes: Number.POSITIVE_INFINITY, thresholdBytes, ok: true, }; } const freeBytes = await getFreeBytes(dir); return { enabled, freeBytes, thresholdBytes, ok: freeBytes >= thresholdBytes }; } // Measure free space on the transcripts data directory (where all downloads // land) and compare it against the configured floor. export async function checkDiskSpace( paths: Paths, settings: SiteSettings, ): Promise { return checkDiskSpaceFor(paths.transcriptsDir, settings); } // --------------------------------------------------------------------------- // THE GATE: one answer to "may byte-writing work start right now?", shared by // every path that writes media. // // checkDiskSpace above answers the instantaneous question. Two things it does // not have are exactly what an unattended runner asking every few seconds // needs: // // 1. HYSTERESIS. Resuming at the same number we stopped at flaps. The first // video to restart writing pushes free space back under the floor, we // stop, a temp file gets cleaned up, we start again — a runner that // oscillates on every deleted file, logging a stop each time. So a stop // LATCHES, and clears only once free space has climbed back to // minFreeDiskGB + resumeMarginGB. The margin is what makes "resumed" mean // the operator freed something, not that a scratch file went away. // // 2. A REASON. "downloads paused" has meant exactly one thing for as long as // it has existed — the manual toggle. A disk stop that renders identically // gets read as manual and left alone, which is how a full disk goes // unnoticed for a week. Every result here carries a reason and a // preformatted message so no caller has to invent its own wording. // // The latch is module-level on purpose: every caller in the process shares one // rule. It is self-healing — any check that sees enough headroom (or a disabled // gate) clears it — so no caller has to remember to reset it, and a crashed or // cancelled job cannot leave the pipeline wedged. // // IT IS KEYED BY DIRECTORY, because there is no longer one disk. A channel whose // media has been relocated to another drive writes its downloads THERE // (common/lib/channelMedia.ts), so a full SSD must not pause work landing on the // platter and a full platter must not pause everything else. One shared boolean // would have done exactly that, in both directions, and would have read as the // manual pause while it did it. Callers that write media for a KNOWN channel // pass that channel's data dir; the rest keep the default, paths.transcriptsDir. // // DELIBERATELY NOT GATED: derived sidecars (digest.json, diarization.json, // attribution.json). Those are kilobytes. Stopping them frees nothing and costs // days of a multi-week sweep, so they run regardless of this gate. Only paths // that write MEDIA — containers, audio, re-fetched source — ask. export type DiskGateReason = // Enough headroom, or the gate is switched off (minFreeDiskGB === 0). | "ok" // Free space is under the floor. A fresh stop. | "below-floor" // Latched: back above the floor, but not yet above floor + resume margin, so // we keep holding rather than flap. | "below-resume-margin"; export type DiskGateStatus = { // True when byte-writing work may start. Callers idle or refuse when false — // this is a refusal to START, never a reason to kill work already running. ok: boolean; // Whether the gate is configured at all. When false the UI hides it. enabled: boolean; freeBytes: number; // The floor: minFreeDiskGB. thresholdBytes: number; // The higher bar a latched gate must clear to reopen: floor + resume margin. // Equal to thresholdBytes when the margin is 0. resumeBytes: number; reason: DiskGateReason; // Preformatted, log-ready, and empty when ok. Callers append their own // subject ("Not re-acquiring X: " + message). message: string; }; // The pure core, so hysteresis is testable without a filesystem or a process. // `latched` is the previous decision; the returned `latched` is the next one. export function evaluateDiskGate(opts: { freeBytes: number; minFreeDiskGB: number; resumeMarginGB: number; latched: boolean; }): DiskGateStatus & { latched: boolean } { const { freeBytes, minFreeDiskGB, resumeMarginGB } = opts; const enabled = minFreeDiskGB > 0; const thresholdBytes = minFreeDiskGB * BYTES_PER_GB; const resumeBytes = (minFreeDiskGB + Math.max(0, resumeMarginGB)) * BYTES_PER_GB; if (!enabled) { return { ok: true, enabled, freeBytes, thresholdBytes, resumeBytes, reason: "ok", latched: false, message: "", }; } // A latched gate is held to the HIGHER bar; an open one only has to clear the // floor. That asymmetry is the whole of the hysteresis. const bar = opts.latched ? resumeBytes : thresholdBytes; if (freeBytes >= bar) { return { ok: true, enabled, freeBytes, thresholdBytes, resumeBytes, reason: "ok", latched: false, message: "", }; } const belowFloor = freeBytes < thresholdBytes; return { ok: false, enabled, freeBytes, thresholdBytes, resumeBytes, reason: belowFloor ? "below-floor" : "below-resume-margin", latched: true, message: belowFloor ? `only ${formatBytes(freeBytes)} free, below the ` + `${formatBytes(thresholdBytes)} floor` : `only ${formatBytes(freeBytes)} free — above the ` + `${formatBytes(thresholdBytes)} floor but waiting for ` + `${formatBytes(resumeBytes)} before resuming`, }; } // dir -> latched. One entry per volume anything has actually asked about, which // is the corpus plus however many media roots the operator is using — single // digits, and it never grows on its own. const gateLatched = new Map(); // Which of the three kinds of caller is asking. The distinctions are the whole // reason this is one shared function rather than three private checks: // // "enforce" (default) — an unattended loop deciding whether to take more // work. Reads AND writes the shared latch, so once it has stopped it is // held to the higher resume bar. This is what the hysteresis is for. // // "observe" — a UI poll. Reads the latch but never writes it, so a dashboard // refreshing every few seconds reports exactly the state the runners are // in — INCLUDING "stopped, waiting for the resume mark" — without being the // thing that changes it. Reporting only the floor here would put the // dashboard back to showing green while the pipeline sits idle, which is // the invisible-stop defect this whole gate exists to remove. // // "manual" — a one-shot the operator just triggered by clicking something. // Only the floor applies, and the latch is neither read nor written. The // hysteresis exists to stop an automatic loop flapping, not to argue with a // person who is standing right there; equally, one manual download must not // silently reopen the gate for the unattended runner. export type DiskGateMode = "enforce" | "observe" | "manual"; // Measure a filesystem and apply the gate. `dir` defaults to // paths.transcriptsDir — pass the channel's data dir when the caller knows which // volume the bytes are about to land on. Skips the statfs syscall entirely when // the gate is disabled, like checkDiskSpaceFor — this runs before every item of // every byte-writing batch. export async function diskGate( paths: Paths, settings: SiteSettings, opts?: { mode?: DiskGateMode; dir?: string }, ): Promise { const mode = opts?.mode ?? "enforce"; const dir = opts?.dir ?? paths.transcriptsDir; if (settings.minFreeDiskGB <= 0) { // A disabled gate clears EVERY volume's latch, not just this one: the // operator turning the floor off means nothing is held anywhere, and a // surviving entry would keep isDiskGateLatched() answering yes forever. if (mode === "enforce") gateLatched.clear(); return { ok: true, enabled: false, freeBytes: Number.POSITIVE_INFINITY, thresholdBytes: 0, resumeBytes: 0, reason: "ok", message: "", }; } const freeBytes = await getFreeBytes(dir); const { latched, ...status } = evaluateDiskGate({ freeBytes, minFreeDiskGB: settings.minFreeDiskGB, resumeMarginGB: settings.resumeMarginGB, latched: mode === "manual" ? false : (gateLatched.get(dir) ?? false), }); if (mode === "enforce") { // Delete rather than store false: the map is "what is currently held", so // an unlatched volume leaves no trace and the map stays the size of the // problem. if (latched) gateLatched.set(dir, true); else gateLatched.delete(dir); } return status; } // Whether the gate is currently holding. Read-only; for surfaces that want to // say "stopped by disk" without taking another measurement. With no `dir` this // answers for ANY volume — which is what a single global "downloads are stopped // by disk" indicator wants. export function isDiskGateLatched(dir?: string): boolean { if (dir === undefined) return gateLatched.size > 0; return gateLatched.get(dir) ?? false; } // Drop the latch. For tests and for an operator action that means "try again // now" — nothing in normal operation needs it, since the gate self-heals. With // no `dir`, drops every volume's. export function resetDiskGate(dir?: string): void { if (dir === undefined) gateLatched.clear(); else gateLatched.delete(dir); }