commit c661c087f9a5b5ee832c873653fad74952ea9fc2
parent 3d4991c941b01725a60581780a4cbc7ca5144ccf
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 11 Sep 2026 11:19:29 -0400
common: the disk gate measures the volume the bytes are going to
There is no longer one disk. A relocated channel writes its downloads to another
drive through the data/ symlink, and the gate's latch was one module-level
boolean — so a full SSD would have paused work landing on the platter, a healthy
platter would have reopened the gate the SSD closed, and both would have
rendered as the manual pause while they did it.
`diskGate(paths, settings, { mode, dir })` — `dir` defaults to
`paths.transcriptsDir`, so every caller that does not pass one is byte-identical
to before. The latch is a `Map<dir, boolean>` holding only what is currently
held: an unlatched volume leaves no entry, a disabled gate (floor 0) clears
every entry, and `isDiskGateLatched()` with no argument still answers the global
"is anything stopped by disk" a single indicator wants.
Threaded from the callers that write media for a KNOWN channel: `runYtdlp`'s
per-video batch check, `backfillReacquire`, and the editor's `pipelineActions`,
`videoActions` and `fixIncompleteTranscript` preflights. `persistKept` and
`buildActiveJobs` (observe) keep the corpus volume.
The auto-runner's pre-pick gate deliberately keeps the corpus volume too, and
the comment says why: it runs BEFORE a channel is picked, across 68 of them on
however many volumes, so there is no per-channel answer to give there. The
per-channel one is taken in runYtdlp, where the slug is known and the bytes are
about to be written.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
8 files changed, 181 insertions(+), 32 deletions(-)
diff --git a/common/controller/autoRunner.ts b/common/controller/autoRunner.ts
@@ -1109,6 +1109,12 @@ async function runLoop(
// only — transcription writes a transcript.json next to audio it already
// has, so stopping it frees nothing.
if (kind === "download") {
+ // The CORPUS volume, deliberately, and not a channel's. This check runs
+ // before the pick, so there is no channel yet — and the lane spans 68 of
+ // them, on however many volumes. The per-channel answer is taken further
+ // down, in runYtdlp, where the slug is known and the bytes are about to
+ // be written; a full SSD idling the lane here is the conservative half of
+ // the pair, not the whole of it.
const gate = await diskGate(paths, settings);
if (!gate.ok) {
if (!diskIdle) {
diff --git a/common/controller/backfillReacquire.ts b/common/controller/backfillReacquire.ts
@@ -163,7 +163,10 @@ export async function reacquireMediaFor(opts: {
}
const settings = getSettings();
- const gate = await diskGate(opts.paths, settings);
+ // The channel's own volume: a relocated channel re-acquires onto the platter.
+ const gate = await diskGate(opts.paths, settings, {
+ dir: path.join(opts.paths.channelsDir, opts.channelSlug, "data"),
+ });
if (!gate.ok) {
log(`Not re-acquiring ${opts.videoId}: ${gate.message}.`);
return { status: "disk-floor", cleanup: NOTHING_TO_CLEAN };
diff --git a/common/lib/diskSpace.test.ts b/common/lib/diskSpace.test.ts
@@ -1,6 +1,13 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { evaluateDiskGate } from "./diskSpace";
+import {
+ diskGate,
+ evaluateDiskGate,
+ isDiskGateLatched,
+ resetDiskGate,
+} from "./diskSpace";
+import type { Paths } from "./paths";
+import type { SiteSettings } from "./settings";
// The gate's whole job is the pair (floor, hysteresis). evaluateDiskGate is the
// pure core precisely so both can be pinned without a filesystem: the flapping
@@ -106,3 +113,86 @@ test("a statfs failure fails open", () => {
assert.equal(g.ok, true);
assert.equal(g.latched, false);
});
+
+// --- THE LATCH IS PER VOLUME ----------------------------------------------
+//
+// There is no longer one disk. A channel whose media has been relocated writes
+// its downloads to another drive through the data/ symlink
+// (common/lib/channelMedia.ts), so one shared boolean would have let a full SSD
+// pause work landing on the platter and a healthy platter reopen the gate the
+// SSD had closed — in both directions, and reading as the manual pause while it
+// did it.
+//
+// These take a real measurement, which is what makes them worth having next to
+// the pure cases above: getFreeBytes fails OPEN (Infinity) on a path that does
+// not exist, so a missing dir stands in for "a volume with room" and a real one
+// under an absurd floor stands in for "a volume that is full". No filesystem is
+// written to.
+
+const FULL = "/"; // a real path, measured, and always under the floor below
+const ROOMY = "/definitely-not-a-mountpoint-ttb-test"; // statfs fails -> Infinity
+
+const settingsWithFloor = (minFreeDiskGB: number) =>
+ ({ minFreeDiskGB, resumeMarginGB: 1 }) as SiteSettings;
+const somePaths = { transcriptsDir: FULL } as Paths;
+
+test("a latch on one volume does not hold another", async () => {
+ resetDiskGate();
+ // An absurd floor no real filesystem clears.
+ const full = await diskGate(somePaths, settingsWithFloor(1e9), { dir: FULL });
+ assert.equal(full.ok, false);
+ assert.equal(isDiskGateLatched(FULL), true);
+ assert.equal(isDiskGateLatched(ROOMY), false);
+
+ // The other volume is unaffected — and, crucially, asking about it does not
+ // clear the first one's latch. With one shared boolean it would have.
+ const roomy = await diskGate(somePaths, settingsWithFloor(1e9), {
+ dir: ROOMY,
+ });
+ assert.equal(roomy.ok, true);
+ assert.equal(isDiskGateLatched(FULL), true);
+ assert.equal(isDiskGateLatched(ROOMY), false);
+ resetDiskGate();
+});
+
+test("the hysteresis is still per volume: a latched dir is held to the higher bar", async () => {
+ resetDiskGate();
+ await diskGate(somePaths, settingsWithFloor(1e9), { dir: FULL });
+ assert.equal(isDiskGateLatched(FULL), true);
+ // A floor of 0 disables the gate entirely, which clears every volume: the
+ // operator switching the floor off means nothing is held anywhere.
+ const off = await diskGate(somePaths, settingsWithFloor(0), { dir: FULL });
+ assert.equal(off.ok, true);
+ assert.equal(off.enabled, false);
+ assert.equal(isDiskGateLatched(), false);
+});
+
+test("observe reads a volume's latch without writing it", async () => {
+ resetDiskGate();
+ const observed = await diskGate(somePaths, settingsWithFloor(1e9), {
+ dir: FULL,
+ mode: "observe",
+ });
+ assert.equal(observed.ok, false);
+ // Nothing was recorded: a dashboard poll is not the thing that latches.
+ assert.equal(isDiskGateLatched(FULL), false);
+ resetDiskGate();
+});
+
+test("resetDiskGate drops one volume or all of them", async () => {
+ resetDiskGate();
+ await diskGate(somePaths, settingsWithFloor(1e9), { dir: FULL });
+ assert.equal(isDiskGateLatched(), true);
+ resetDiskGate(ROOMY);
+ assert.equal(isDiskGateLatched(FULL), true);
+ resetDiskGate(FULL);
+ assert.equal(isDiskGateLatched(FULL), false);
+ assert.equal(isDiskGateLatched(), false);
+});
+
+test("with no dir the gate measures paths.transcriptsDir, as it always has", async () => {
+ resetDiskGate();
+ await diskGate(somePaths, settingsWithFloor(1e9));
+ assert.equal(isDiskGateLatched(somePaths.transcriptsDir), true);
+ resetDiskGate();
+});
diff --git a/common/lib/diskSpace.ts b/common/lib/diskSpace.ts
@@ -83,11 +83,18 @@ export async function checkDiskSpace(
// unnoticed for a week. Every result here carries a reason and a
// preformatted message so no caller has to invent its own wording.
//
-// The latch is module-level on purpose: there is one disk, so there is one
-// rule, and every caller in the process shares it. It is self-healing — any
-// check that sees enough headroom (or a disabled gate) clears it — so no caller
-// has to remember to reset it, and a crashed or cancelled job cannot leave the
-// pipeline wedged.
+// The latch is module-level on purpose: every caller in the process shares one
+// rule. It is self-healing — any check that sees enough headroom (or a disabled
+// gate) clears it — so no caller has to remember to reset it, and a crashed or
+// cancelled job cannot leave the pipeline wedged.
+//
+// IT IS KEYED BY DIRECTORY, because there is no longer one disk. A channel whose
+// media has been relocated to another drive writes its downloads THERE
+// (common/lib/channelMedia.ts), so a full SSD must not pause work landing on the
+// platter and a full platter must not pause everything else. One shared boolean
+// would have done exactly that, in both directions, and would have read as the
+// manual pause while it did it. Callers that write media for a KNOWN channel
+// pass that channel's data dir; the rest keep the default, paths.transcriptsDir.
//
// DELIBERATELY NOT GATED: derived sidecars (digest.json, diarization.json,
// attribution.json). Those are kilobytes. Stopping them frees nothing and costs
@@ -178,7 +185,10 @@ export function evaluateDiskGate(opts: {
};
}
-let gateLatched = false;
+// dir -> latched. One entry per volume anything has actually asked about, which
+// is the corpus plus however many media roots the operator is using — single
+// digits, and it never grows on its own.
+const gateLatched = new Map<string, boolean>();
// Which of the three kinds of caller is asking. The distinctions are the whole
// reason this is one shared function rather than three private checks:
@@ -201,17 +211,23 @@ let gateLatched = false;
// silently reopen the gate for the unattended runner.
export type DiskGateMode = "enforce" | "observe" | "manual";
-// Measure the transcripts filesystem and apply the gate. Skips the statfs
-// syscall entirely when the gate is disabled, like checkDiskSpaceFor — this
-// runs before every item of every byte-writing batch.
+// Measure a filesystem and apply the gate. `dir` defaults to
+// paths.transcriptsDir — pass the channel's data dir when the caller knows which
+// volume the bytes are about to land on. Skips the statfs syscall entirely when
+// the gate is disabled, like checkDiskSpaceFor — this runs before every item of
+// every byte-writing batch.
export async function diskGate(
paths: Paths,
settings: SiteSettings,
- opts?: { mode?: DiskGateMode },
+ opts?: { mode?: DiskGateMode; dir?: string },
): Promise<DiskGateStatus> {
const mode = opts?.mode ?? "enforce";
+ const dir = opts?.dir ?? paths.transcriptsDir;
if (settings.minFreeDiskGB <= 0) {
- if (mode === "enforce") gateLatched = false;
+ // A disabled gate clears EVERY volume's latch, not just this one: the
+ // operator turning the floor off means nothing is held anywhere, and a
+ // surviving entry would keep isDiskGateLatched() answering yes forever.
+ if (mode === "enforce") gateLatched.clear();
return {
ok: true,
enabled: false,
@@ -222,25 +238,36 @@ export async function diskGate(
message: "",
};
}
- const freeBytes = await getFreeBytes(paths.transcriptsDir);
+ const freeBytes = await getFreeBytes(dir);
const { latched, ...status } = evaluateDiskGate({
freeBytes,
minFreeDiskGB: settings.minFreeDiskGB,
resumeMarginGB: settings.resumeMarginGB,
- latched: mode === "manual" ? false : gateLatched,
+ latched: mode === "manual" ? false : (gateLatched.get(dir) ?? false),
});
- if (mode === "enforce") gateLatched = latched;
+ if (mode === "enforce") {
+ // Delete rather than store false: the map is "what is currently held", so
+ // an unlatched volume leaves no trace and the map stays the size of the
+ // problem.
+ if (latched) gateLatched.set(dir, true);
+ else gateLatched.delete(dir);
+ }
return status;
}
// Whether the gate is currently holding. Read-only; for surfaces that want to
-// say "stopped by disk" without taking another measurement.
-export function isDiskGateLatched(): boolean {
- return gateLatched;
+// say "stopped by disk" without taking another measurement. With no `dir` this
+// answers for ANY volume — which is what a single global "downloads are stopped
+// by disk" indicator wants.
+export function isDiskGateLatched(dir?: string): boolean {
+ if (dir === undefined) return gateLatched.size > 0;
+ return gateLatched.get(dir) ?? false;
}
// Drop the latch. For tests and for an operator action that means "try again
-// now" — nothing in normal operation needs it, since the gate self-heals.
-export function resetDiskGate(): void {
- gateLatched = false;
+// now" — nothing in normal operation needs it, since the gate self-heals. With
+// no `dir`, drops every volume's.
+export function resetDiskGate(dir?: string): void {
+ if (dir === undefined) gateLatched.clear();
+ else gateLatched.delete(dir);
}
diff --git a/common/ytdlp/runYtdlp.ts b/common/ytdlp/runYtdlp.ts
@@ -873,7 +873,12 @@ async function runManagedDownloads(
if (opts.drainSignal?.aborted) return;
if (firstFailure && abortOnError) return;
if (lowDiskStopped) return;
- const gate = await diskGate(opts.paths, settings);
+ // THIS channel's volume, not the corpus's: a relocated channel's
+ // downloads land on the platter through the data/ symlink, so the SSD
+ // being full is not a reason to stop them (and vice versa).
+ const gate = await diskGate(opts.paths, settings, {
+ dir: path.join(channelRoot(opts), "data"),
+ });
if (!gate.ok) {
lowDiskStopped = true;
opts.onLog(
diff --git a/editor/app/channels/[slug]/lib/fixIncompleteTranscript.ts b/editor/app/channels/[slug]/lib/fixIncompleteTranscript.ts
@@ -75,7 +75,9 @@ export async function fixIncompleteTranscriptOne(opts: {
// full disk once that audio is gone would leave the video with neither the
// stub nor a replacement. Latching, because the channel-level batch calls this
// in a loop unattended.
- const gate = await diskGate(paths, getSettings());
+ const gate = await diskGate(paths, getSettings(), {
+ dir: path.join(paths.channelsDir, slug, "data"),
+ });
if (!gate.ok) {
throw new Error(
`Cannot re-download audio for ${videoId}: ${gate.message}. ` +
diff --git a/editor/app/channels/[slug]/pipelineActions.ts b/editor/app/channels/[slug]/pipelineActions.ts
@@ -1,5 +1,6 @@
"use server";
+import path from "node:path";
import { revalidatePath } from "next/cache";
import {
HANDLING_VALUES,
@@ -43,10 +44,16 @@ import type { Paths } from "yt-dlp-transcript-common/lib/paths";
// between videos via the gate in runManagedDownloads (common/ytdlp/runYtdlp.ts).
async function lowDiskError(
paths: Paths,
+ slug: string,
): Promise<{ ok: false; error: string } | null> {
// The operator clicked this: only the floor applies and the shared hysteresis
- // latch is left alone (see diskGate). runYtdlp re-checks per video mid-batch.
- const disk = await diskGate(paths, getSettings(), { mode: "manual" });
+ // latch is left alone (see diskGate). runYtdlp re-checks per video mid-batch,
+ // against this same volume — the channel's own, which for a relocated channel
+ // is the platter and not the corpus disk.
+ const disk = await diskGate(paths, getSettings(), {
+ mode: "manual",
+ dir: path.join(paths.channelsDir, slug, "data"),
+ });
if (disk.ok) return null;
return {
ok: false,
@@ -139,7 +146,7 @@ async function runPipelineAction(
// store-playlist only fetches the video list (no media written), so it is not
// gated; every other mode downloads audio/subtitles into transcriptsDir.
if (mode !== "store-playlist") {
- const err = await lowDiskError(paths);
+ const err = await lowDiskError(paths, slug);
if (err) return err;
}
return runManagedFunction({
@@ -415,7 +422,7 @@ export async function importVideoAction(
// downloadOneManaged falls back to %(id)s and the reconcile pass repairs the
// dir, so we don't hard-fail here.
const videoId = extractVideoId(videoUrl) ?? undefined;
- const err = await lowDiskError(paths);
+ const err = await lowDiskError(paths, slug);
if (err) return err;
const settings = getSettings();
return runManagedFunction({
diff --git a/editor/app/channels/[slug]/videos/[id]/videoActions.ts b/editor/app/channels/[slug]/videos/[id]/videoActions.ts
@@ -142,7 +142,7 @@ export async function downloadVideoPipelineAction(
const paths = getPaths();
// This action fetches a full container. Its sibling redownloadToArchiveAction
// has always preflighted the disk; this one never did.
- const lowDisk = await lowDiskError(paths);
+ const lowDisk = await lowDiskError(paths, slug);
if (lowDisk) return lowDisk;
const url = await findVideoSourceUrl(paths, slug, videoId, r.config);
if (!url) {
@@ -218,7 +218,7 @@ export async function redownloadToArchiveAction(
"Could not determine the video URL: no metadata.info.json and the playlist does not contain a matching entry.",
};
}
- const err = await lowDiskError(paths);
+ const err = await lowDiskError(paths, slug);
if (err) return err;
const settings = getSettings();
return runManagedFunction({
@@ -262,8 +262,14 @@ export async function redownloadToArchiveAction(
// applies and the shared hysteresis latch is left alone (see diskGate).
async function lowDiskError(
paths: Paths,
+ slug: string,
): Promise<{ ok: false; error: string } | null> {
- const gate = await diskGate(paths, getSettings(), { mode: "manual" });
+ // The channel's own volume. A relocated channel's bytes land on the platter
+ // through the data/ symlink, so a full SSD is not a reason to refuse them.
+ const gate = await diskGate(paths, getSettings(), {
+ mode: "manual",
+ dir: path.join(paths.channelsDir, slug, "data"),
+ });
if (gate.ok) return null;
return {
ok: false,
@@ -298,7 +304,10 @@ export async function whisperVideoAction(
// disk produces a transcript.json measured in kilobytes, so a low disk
// is no reason to refuse it — the gate belongs on the fetch, not on the
// whole action.
- const gate = await diskGate(paths, getSettings(), { mode: "manual" });
+ const gate = await diskGate(paths, getSettings(), {
+ mode: "manual",
+ dir: path.join(paths.channelsDir, slug, "data"),
+ });
if (!gate.ok) {
throw new Error(
`Cannot download audio for ${videoId}: ${gate.message}. ` +
