commit 8315b8cb4b83ecb9d89a2e5470b6c80de9f3068b
parent 7497bfbaa8898ad9f8de5823b13694025716812d
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 29 Sep 2026 22:52:03 -0400
editor: the pages' reads of a relocated channel's drive go through the watchdog
The videos list (its data/ listing and titles, then the selected video's
files), the video page (its whole directory read, as one unit) and its title,
the media file route's stat, the Cleanup stage's saved-video totals and the
/storage store walk go through `onDrive`: a drive that has not answered in 3 s
is marked stalled and the page shows the not-answering notice (a 503 for the
file route; a size of 0 for the store walk). MediaNotAnswering takes a stall
with no known location.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
6 files changed, 219 insertions(+), 88 deletions(-)
diff --git a/editor/app/api/channels/[slug]/videos/[id]/files/[name]/route.ts b/editor/app/api/channels/[slug]/videos/[id]/files/[name]/route.ts
@@ -6,7 +6,11 @@ import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import { makeSafeController } from "yt-dlp-transcript-common/lib/safeStreamController";
import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels";
import { channelMediaStall } from "yt-dlp-transcript-common/lib/channelMedia";
-import { NOT_ANSWERING } from "yt-dlp-transcript-common/lib/storageHealth";
+import {
+ NOT_ANSWERING,
+ isDriveNotAnswering,
+ onDrive,
+} from "yt-dlp-transcript-common/lib/storageHealth";
export const dynamic = "force-dynamic";
@@ -127,18 +131,23 @@ export async function GET(
}
// A drive that is not answering is not asked: the stat and the stream would
- // each wait on it. 503, because it is a state that passes.
- if (channelMediaStall(await readChannelConfig(paths, slug))) {
- return NextResponse.json(
+ // each wait on it. 503, because it is a state that passes. The stat goes
+ // through the watchdog, so a drive that stops answering now is a 503 too;
+ // the stream that follows a stat that answered is not raced.
+ const notAnswering = () =>
+ NextResponse.json(
{ error: `Media not read: ${NOT_ANSWERING}.` },
{ status: 503, headers: { "retry-after": "15" } },
);
- }
+ const channelConfig = await readChannelConfig(paths, slug);
+ if (channelMediaStall(channelConfig)) return notAnswering();
+ const drive = channelConfig?.dataDir?.trim();
let stats;
try {
- stats = await stat(fullPath);
- } catch {
+ stats = await (drive ? onDrive(drive, () => stat(fullPath)) : stat(fullPath));
+ } catch (err) {
+ if (isDriveNotAnswering(err)) return notAnswering();
return NextResponse.json({ error: "Not found" }, { status: 404 });
}
if (!stats.isFile()) {
diff --git a/editor/app/channels/[slug]/components/MediaNotAnswering.tsx b/editor/app/channels/[slug]/components/MediaNotAnswering.tsx
@@ -17,14 +17,17 @@ import {
// renders.
//
// Server-only: it takes the health entry from the page, which read it from
-// memory.
+// memory — or from the DriveNotAnsweringError a page's read got from `onDrive`'s
+// watchdog.
export function MediaNotAnswering({
slug,
stall,
what,
}: {
slug: string;
- stall: LocationHealth;
+ // The stalled location, or null when the drive is on no location the health
+ // state knows and a read of it did not answer within 3 s.
+ stall: LocationHealth | null;
// What would have been shown: "The video list", "This video".
what: string;
}) {
@@ -35,9 +38,19 @@ export function MediaNotAnswering({
aria-label="media not answering"
className="rounded border border-destructive/50 bg-destructive/5 px-3 py-2 text-sm text-destructive"
>
- {what} reads this channel's media, which is on “{stall.label}
- ” — a drive that is {notAnsweringText(stall)}. Nothing is read
- from it until it answers again (it is checked every 15 s).
+ {stall ? (
+ <>
+ {what} reads this channel's media, which is on “
+ {stall.label}” — a drive that is {notAnsweringText(stall)}.
+ Nothing is read from it until it answers again (it is checked every
+ 15 s).
+ </>
+ ) : (
+ <>
+ {what} reads this channel's media, and a read of its drive did
+ not answer within 3 s. Nothing more is read from it on this page.
+ </>
+ )}
</p>
<p className="text-sm text-muted-foreground">
<Link href={`/channels/${slug}`} className="underline hover:text-foreground">
diff --git a/editor/app/channels/[slug]/page.tsx b/editor/app/channels/[slug]/page.tsx
@@ -529,11 +529,24 @@ export default async function ChannelDetailPage({
);
}
// Saved-video store summary for this channel + whether backups are
- // configured, for the Retention & persistence section.
+ // configured, for the Retention & persistence section. Its reads go
+ // through the watchdog (`notAnswering`): a drive that stops answering
+ // on the way is named there, and the stage says so.
+ const notAnswering: string[] = [];
const savedTotals = await savedVideoTotals({
paths,
channelSlug: slug,
+ notAnswering,
});
+ if (notAnswering.length > 0) {
+ return (
+ <MediaNotAnswering
+ slug={slug}
+ stall={channelMediaStall(config)}
+ what="The Cleanup stage"
+ />
+ );
+ }
return (
<CleanupStage
slug={slug}
diff --git a/editor/app/channels/[slug]/videos/[id]/page.tsx b/editor/app/channels/[slug]/videos/[id]/page.tsx
@@ -43,6 +43,10 @@ import { MetadataHistoryDetails } from "./components/MetadataHistoryDetails";
import { loadVideoTags } from "./lib/videoTags";
import { loadVideoOperationPanels } from "./lib/videoOperationPanels";
import { channelMediaStall } from "yt-dlp-transcript-common/lib/channelMedia";
+import {
+ isDriveNotAnswering,
+ onDrive,
+} from "yt-dlp-transcript-common/lib/storageHealth";
import { MediaNotAnswering } from "../../components/MediaNotAnswering";
export const dynamic = "force-dynamic";
@@ -82,12 +86,18 @@ export async function generateMetadata({
params: Promise<{ slug: string; id: string }>;
}): Promise<Metadata> {
const { slug, id } = await params;
- // The title is read off the drive; a drive that is not answering is not asked.
- if (channelMediaStall(await readChannelConfig(getPaths(), slug))) {
- return { title: `${id} — Video — ${slug}` };
+ // The title is read off the drive; a drive that is not answering is not
+ // asked, and one that does not answer in 3 s is given up on.
+ const drive = (await readChannelConfig(getPaths(), slug))?.dataDir?.trim();
+ let subject = id;
+ try {
+ const meta = await (drive
+ ? onDrive(drive, () => loadMeta(slug, id))
+ : loadMeta(slug, id));
+ subject = meta.title ?? id;
+ } catch (err) {
+ if (!isDriveNotAnswering(err)) throw err;
}
- const meta = await loadMeta(slug, id);
- const subject = meta.title ?? id;
return { title: `${subject} — Video — ${slug}` };
}
@@ -105,63 +115,110 @@ export default async function VideoDetailPage({
if (stall) {
return <MediaNotAnswering slug={slug} stall={stall} what="This video's page" />;
}
- const dirData = await loadVideoDir(slug, id);
- const meta = await loadMeta(slug, id);
- const videoDir = path.join(getPaths().channelsDir, slug, "data", id);
- const downloadOutcome = await loadDownloadOutcome(videoDir);
- const availabilityRecord = await loadAvailability(videoDir);
- const availabilityHistory = availabilityRecord?.history ?? [];
- // Every rewrite of metadata.info.json that changed its bytes (release 10
- // slice N). One small bounded file, read once; null → nothing drawn.
- const metadataHistory = metadataHistoryView(
- await loadMetadataHistory(videoDir),
- Date.now(),
- );
- const doNotClean = await isDoNotClean(videoDir);
- const excludedFromTruncatedCheck =
- await isExcludedFromTruncatedCheck(videoDir);
- const savedVideo = await loadSavedVideo(videoDir);
- // The windows another tool asked this editor to fetch. One readdir of
- // data/<id>/clips/ plus a stat per file — and no per-CHANNEL count anywhere,
- // because that would be a walk of every video dir to draw one number.
- const clipWindows = await listClipWindows(videoDir);
- // Where each subtitle track came from, so the panel can say "YouTube
- // auto-captions" vs "manual captions" — and offer to replace the former with a
- // transcript of our own. One 4 KB head read per VTT (see subtitleProvenance).
- const vttProvenance: Record<string, SubtitleProvenance> = {};
- for (const f of dirData.files) {
- if (!isTranscriptVtt(f.name)) continue;
- vttProvenance[f.name] = await resolveVttProvenance(videoDir, f.name);
- }
- const cov = await readTranscriptCoverage(videoDir);
- const coverage = cov
- ? {
- lastCueEnd: cov.cov.lastCueEnd,
- duration: cov.cov.duration,
- coverage: cov.cov.coverage,
- incomplete:
- !excludedFromTruncatedCheck &&
- isIncompleteTranscript(cov.cov, {
- isLivestream: cov.isLivestream,
- }),
- }
- : null;
+ // EVERY READ BELOW IS OF THIS VIDEO'S DIRECTORY, on the channel's drive when
+ // it is relocated, so they go through the watchdog as one unit: a drive that
+ // has not answered them in 3 s is marked stalled and the page says so.
+ const loadAll = async () => {
+ const dirData = await loadVideoDir(slug, id);
+ const meta = await loadMeta(slug, id);
+ const videoDir = path.join(getPaths().channelsDir, slug, "data", id);
+ const downloadOutcome = await loadDownloadOutcome(videoDir);
+ const availabilityRecord = await loadAvailability(videoDir);
+ const availabilityHistory = availabilityRecord?.history ?? [];
+ // Every rewrite of metadata.info.json that changed its bytes (release 10
+ // slice N). One small bounded file, read once; null → nothing drawn.
+ const metadataHistory = metadataHistoryView(
+ await loadMetadataHistory(videoDir),
+ Date.now(),
+ );
+ const doNotClean = await isDoNotClean(videoDir);
+ const excludedFromTruncatedCheck =
+ await isExcludedFromTruncatedCheck(videoDir);
+ const savedVideo = await loadSavedVideo(videoDir);
+ // The windows another tool asked this editor to fetch. One readdir of
+ // data/<id>/clips/ plus a stat per file — and no per-CHANNEL count anywhere,
+ // because that would be a walk of every video dir to draw one number.
+ const clipWindows = await listClipWindows(videoDir);
+ // Where each subtitle track came from, so the panel can say "YouTube
+ // auto-captions" vs "manual captions" — and offer to replace the former with a
+ // transcript of our own. One 4 KB head read per VTT (see subtitleProvenance).
+ const vttProvenance: Record<string, SubtitleProvenance> = {};
+ for (const f of dirData.files) {
+ if (!isTranscriptVtt(f.name)) continue;
+ vttProvenance[f.name] = await resolveVttProvenance(videoDir, f.name);
+ }
+ const cov = await readTranscriptCoverage(videoDir);
+ const coverage = cov
+ ? {
+ lastCueEnd: cov.cov.lastCueEnd,
+ duration: cov.cov.duration,
+ coverage: cov.cov.coverage,
+ incomplete:
+ !excludedFromTruncatedCheck &&
+ isIncompleteTranscript(cov.cov, {
+ isLivestream: cov.isLivestream,
+ }),
+ }
+ : null;
- // ONE PANEL PER REGISTRY OPERATION, each carrying the state that entry's own
- // state() reports. The page used to re-derive the digest's freshness here —
- // its own target resolution and its own per-section fold beside the
- // registry's — which is how a page and a work list end up describing the same
- // disk differently.
- const panels = await loadVideoOperationPanels({
- paths: getPaths(),
- channelSlug: slug,
- videoId: id,
- settings: getSettings(),
- });
+ // ONE PANEL PER REGISTRY OPERATION, each carrying the state that entry's own
+ // state() reports. The page used to re-derive the digest's freshness here —
+ // its own target resolution and its own per-section fold beside the
+ // registry's — which is how a page and a work list end up describing the same
+ // disk differently.
+ const panels = await loadVideoOperationPanels({
+ paths: getPaths(),
+ channelSlug: slug,
+ videoId: id,
+ settings: getSettings(),
+ });
- // The curated tags on this video, with each one's provenance. Reads tags.json
- // plus (for rule hits) ONE key out of the transcript index — not a scan.
- const tagsView = await loadVideoTags(slug, id);
+ // The curated tags on this video, with each one's provenance. Reads tags.json
+ // plus (for rule hits) ONE key out of the transcript index — not a scan.
+ const tagsView = await loadVideoTags(slug, id);
+ return {
+ dirData,
+ meta,
+ videoDir,
+ downloadOutcome,
+ availabilityHistory,
+ metadataHistory,
+ doNotClean,
+ excludedFromTruncatedCheck,
+ savedVideo,
+ clipWindows,
+ vttProvenance,
+ coverage,
+ panels,
+ tagsView,
+ };
+ };
+ const drive = config.dataDir?.trim();
+ let loaded: Awaited<ReturnType<typeof loadAll>>;
+ try {
+ loaded = await (drive ? onDrive(drive, loadAll) : loadAll());
+ } catch (err) {
+ if (!isDriveNotAnswering(err)) throw err;
+ return (
+ <MediaNotAnswering slug={slug} stall={err.health} what="This video's page" />
+ );
+ }
+ const {
+ dirData,
+ meta,
+ videoDir,
+ downloadOutcome,
+ availabilityHistory,
+ metadataHistory,
+ doNotClean,
+ excludedFromTruncatedCheck,
+ savedVideo,
+ clipWindows,
+ vttProvenance,
+ coverage,
+ panels,
+ tagsView,
+ } = loaded;
const registry = getRegistry();
const existingQueues = registry.activeQueueNames();
diff --git a/editor/app/channels/[slug]/videos/page.tsx b/editor/app/channels/[slug]/videos/page.tsx
@@ -37,6 +37,10 @@ import { normalizeBuckets } from "yt-dlp-transcript-common/views/pipeline/stageS
import { attachCuratedTags } from "../lib/videoTagRows";
import { readChannelVideoTitles } from "yt-dlp-transcript-common/controller/videoTitles";
import { channelMediaStall } from "yt-dlp-transcript-common/lib/channelMedia";
+import {
+ isDriveNotAnswering,
+ onDrive,
+} from "yt-dlp-transcript-common/lib/storageHealth";
import { MediaNotAnswering } from "../components/MediaNotAnswering";
export const dynamic = "force-dynamic";
@@ -140,14 +144,30 @@ export default async function ChannelVideosPage({
);
const channelDataDir = path.join(paths.channelsDir, slug, "data");
- const channelDataDirIds = await readDataDirVideoIds(channelDataDir);
- // What each video is called: one key range over the transcript index, one
- // read of the channel's metadata-scan.json, and a head read of
- // metadata.info.json only for what those two did not name. Measured at
- // ~80 ms for a synthetic 5,000-id channel (plans/release-8.md, slice V).
- const titles = await readChannelVideoTitles(paths, slug, [
- ...new Set([...channelDataDirIds, ...(snapshot.undownloadedIds ?? [])]),
- ]);
+ // THE READS OF THE DRIVE go through the watchdog when the channel is
+ // relocated: a drive that has not answered them in 3 s is marked stalled and
+ // the page says so instead (see MediaNotAnswering).
+ const drive = config.dataDir?.trim();
+ const onMedia = <T,>(call: () => Promise<T>): Promise<T> =>
+ drive ? onDrive(drive, call) : call();
+ let channelDataDirIds: string[];
+ let titles: Awaited<ReturnType<typeof readChannelVideoTitles>>;
+ try {
+ [channelDataDirIds, titles] = await onMedia(async () => {
+ const ids = await readDataDirVideoIds(channelDataDir);
+ // What each video is called: one key range over the transcript index,
+ // one read of the channel's metadata-scan.json, and a head read of
+ // metadata.info.json only for what those two did not name. Measured at
+ // ~80 ms for a synthetic 5,000-id channel (plans/release-8.md, slice V).
+ const named = await readChannelVideoTitles(paths, slug, [
+ ...new Set([...ids, ...(snapshot.undownloadedIds ?? [])]),
+ ]);
+ return [ids, named] as const;
+ });
+ } catch (err) {
+ if (!isDriveNotAnswering(err)) throw err;
+ return <MediaNotAnswering slug={slug} stall={err.health} what="The video list" />;
+ }
const rows = computeVideoRows({
channelDataDirIds,
snapshot,
@@ -197,8 +217,9 @@ export default async function ChannelVideosPage({
);
if (selectedVideoId) {
const videoDir = path.join(channelDataDir, selectedVideoId);
- const [dirData, title, outcome, availabilityRecord, cov, excludedTrunc] =
- await Promise.all([
+ let loadedVideo;
+ try {
+ loadedVideo = await onMedia(() => Promise.all([
loadVideoDir(channelDataDir, selectedVideoId),
// Already read for the list — the title map covers every row.
Promise.resolve(titles.get(selectedVideoId)?.title ?? null),
@@ -206,7 +227,13 @@ export default async function ChannelVideosPage({
loadAvailability(videoDir),
readTranscriptCoverage(videoDir),
isExcludedFromTruncatedCheck(videoDir),
- ]);
+ ]));
+ } catch (err) {
+ if (!isDriveNotAnswering(err)) throw err;
+ return <MediaNotAnswering slug={slug} stall={err.health} what="The video list" />;
+ }
+ const [dirData, title, outcome, availabilityRecord, cov, excludedTrunc] =
+ loadedVideo;
const coverage = cov
? {
lastCueEnd: cov.cov.lastCueEnd,
diff --git a/editor/app/storage/buildStorage.ts b/editor/app/storage/buildStorage.ts
@@ -4,7 +4,9 @@ import { getRegistry } from "yt-dlp-transcript-common/jobs/registry";
import { getFreeBytes } from "yt-dlp-transcript-common/lib/diskSpace";
import { udisksctlAvailable } from "yt-dlp-transcript-common/lib/storageVolumes";
import {
+ isDriveNotAnswering,
notAnsweringText,
+ onDrive,
stalledLocation,
} from "yt-dlp-transcript-common/lib/storageHealth";
import { listChannelBriefs } from "yt-dlp-transcript-common/controller/channels";
@@ -98,10 +100,20 @@ export async function buildStorage(): Promise<StorageRowsPayload> {
}
}
const store = await inspectSavedVideosStore(paths, settings);
- const storeMeasured =
- store.status === "unreachable" || store.status === "in-transition"
- ? { bytes: 0, files: 0 }
- : await measureTreeCached(paths.savedVideosDir);
+ // A store on a location walks that location's drive: through the watchdog,
+ // so a drive that stops answering mid-walk leaves the size at 0 (the store's
+ // status line says why on the next render) instead of holding the page.
+ const storeLocation = locations.find((l) => l.id === store.locationId);
+ let storeMeasured = { bytes: 0, files: 0 };
+ if (store.status !== "unreachable" && store.status !== "in-transition") {
+ try {
+ storeMeasured = await (storeLocation
+ ? onDrive(storeLocation, () => measureTreeCached(paths.savedVideosDir))
+ : measureTreeCached(paths.savedVideosDir));
+ } catch (err) {
+ if (!isDriveNotAnswering(err)) throw err;
+ }
+ }
return buildStorageRows({
locations,
savedVideos: {
