commit 20466b29e57eab7e474707981b5cb0e68493ef5f parent 519cfcedac86a3fcc61af4fd4f8e5e0a7810d948 Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st> Date: Fri, 2 Oct 2026 00:47:21 -0400 editor: the media tier on the Storage panel, the video pages and the file route The Storage panel shows two rows under one heading — the media path (the target, or <channel>/media in place) with the media tier's bytes, and the text path with the text tier's — the preview's "files tiered first", the hold sentence's "its text stays readable", and a legacy channel's refusal; the badge's legacy entry is "Media layout retired" (danger). Resume refuses a tier migration's marker; the bulk move skips a legacy channel naming migrate-tier; the job's done line says how many files were tiered first. The video page and the videos list read the text on the corpus disk (held only for a legacy channel), list tiered links, and ask the media drive for their sizes through one watchdog call (size unknown when it is away). The file route picks the drive by the file's kind and answers 503 with retry-after for a link whose target is not there. A file delete refuses a tiered file whose drive is away. The Configure form shows mediaDir. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Diffstat:
14 files changed, 318 insertions(+), 127 deletions(-)
diff --git a/editor/app/api/channels/[slug]/videos/[id]/files/[name]/route.ts b/editor/app/api/channels/[slug]/videos/[id]/files/[name]/route.ts @@ -1,11 +1,15 @@ import path from "node:path"; import { createReadStream, type ReadStream } from "node:fs"; -import { stat } from "node:fs/promises"; +import { lstat, stat } from "node:fs/promises"; import { NextResponse } from "next/server"; import { getPaths } from "yt-dlp-transcript-common/lib/paths"; import { makeSafeController } from "yt-dlp-transcript-common/lib/safeStreamController"; import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels"; -import { channelMediaStall } from "yt-dlp-transcript-common/lib/channelMedia"; +import { + channelMediaStall, + channelTextStall, +} from "yt-dlp-transcript-common/lib/channelMedia"; +import { classifyEntry } from "yt-dlp-transcript-common/lib/mediaTier"; import { NOT_ANSWERING, isDriveNotAnswering, @@ -139,15 +143,44 @@ export async function GET( { error: `Media not read: ${NOT_ANSWERING}.` }, { status: 503, headers: { "retry-after": "15" } }, ); + // THE DRIVE IS CHOSEN BY THE FILE'S KIND (release 17). A media file (the + // audio, the raw live chat, a container) is on the media tier's drive — + // `mediaDir`, reached through its relative link — so its stat goes through + // that drive's watchdog and a stalled media drive refuses it. A text file is + // on the corpus disk and is read directly: a stalled media drive never holds + // a transcript. (A legacy channel's text is on its retired `dataDir`.) const channelConfig = await readChannelConfig(paths, slug); - if (channelMediaStall(channelConfig)) return notAnswering(); - const drive = channelConfig?.dataDir?.trim(); + const isMedia = classifyEntry(name) === "media"; + if (isMedia ? channelMediaStall(channelConfig) : channelTextStall(channelConfig)) { + return notAnswering(); + } + const legacyDrive = channelConfig?.dataDir?.trim() || undefined; + const drive = isMedia + ? channelConfig?.mediaDir?.trim() || legacyDrive + : legacyDrive; let stats; try { stats = await (drive ? onDrive(drive, () => stat(fullPath)) : stat(fullPath)); } catch (err) { if (isDriveNotAnswering(err)) return notAnswering(); + // A LINK WHOSE TARGET IS NOT THERE is a tiered file on a media drive that + // is unmounted (or renamed away): the file exists, its bytes are not + // reachable now. 503 with a retry, never 404 — a 404 tells a player, and + // an operator, that there is no such file. + const isLink = await lstat(fullPath) + .then((l) => l.isSymbolicLink()) + .catch(() => false); + if (isLink) { + return NextResponse.json( + { + error: + `Media not read: ${name} is on this channel's media drive, which ` + + `is not mounted or not answering.`, + }, + { status: 503, headers: { "retry-after": "15" } }, + ); + } return NextResponse.json({ error: "Not found" }, { status: 404 }); } if (!stats.isFile()) { diff --git a/editor/app/channels/[slug]/components/stages/StorageStage.tsx b/editor/app/channels/[slug]/components/stages/StorageStage.tsx @@ -84,10 +84,14 @@ type Props = { // the badge. Undefined when it is on none — in place, or on a root nobody // named. See common/lib/storageLocations.ts. locationLabel?: string; - // Audio bytes this channel holds, from the loaded snapshot — NOT a walk. Null - // when the snapshot predates the field (or there is no snapshot), and rendered - // as "—" rather than "0": a zero here would claim a measurement nobody took. + // THE TWO TIERS (release 17), from the loaded snapshot — NOT a walk. The + // MEDIA tier (`totalMediaBytes`: the audio and the raw live chat, what a move + // carries) and the TEXT tier (`totalTextBytes`: everything under `data/` on + // the corpus disk but the tier and the clips). Null when the snapshot + // predates the field (or there is no snapshot), and rendered as "—" rather + // than "0": a zero here would claim a measurement nobody took. mediaBytes: number | null; + textBytes: number | null; // Bytes this channel's `data/<id>/clips/` windows occupy, from the same // snapshot. Null when the snapshot predates the field, for the same reason // mediaBytes is: "0 B of clips" is a measurement nobody took. @@ -124,6 +128,7 @@ export function StorageStage({ location, locationLabel, mediaBytes, + textBytes, clipsBytes, freeBytes, volumeDir, @@ -140,22 +145,31 @@ export function StorageStage({ <h3 className="text-base font-semibold">Location</h3> <MediaLocationBadge media={location} locationLabel={locationLabel} /> </div> + {/* TWO ROWS UNDER ONE HEADING (release 17): the MEDIA tier — the big + files, `channels/<slug>/media`, which is what moves — and the TEXT + tier, `data/`, which never leaves the corpus disk. */} <dl className="grid grid-cols-[max-content_1fr] gap-x-4 gap-y-1 text-sm"> <dt className="text-muted-foreground">Media path</dt> <dd className="font-mono text-xs break-all" aria-label="media path"> {location.relocated && location.target ? location.target - : location.dataDir} + : `${location.mediaLink} (in place)`} </dd> - <dt className="text-muted-foreground">Read as</dt> - <dd className="font-mono text-xs break-all"> - {location.dataDir} - {location.relocated ? " (symlink)" : ""} - </dd> - <dt className="text-muted-foreground">Audio on disk</dt> + <dt className="text-muted-foreground">Media on disk</dt> <dd aria-label="media bytes"> {mediaBytes === null ? "—" : formatBytes(mediaBytes)} </dd> + <dt className="text-muted-foreground">Text path</dt> + <dd className="font-mono text-xs break-all" aria-label="text path"> + {location.dataDir} + {location.status === "legacy" + ? " (a symlink — the retired whole-directory layout)" + : " (in place)"} + </dd> + <dt className="text-muted-foreground">Text on disk</dt> + <dd aria-label="text bytes"> + {textBytes === null ? "—" : formatBytes(textBytes)} + </dd> <dt className="text-muted-foreground">Free on that volume</dt> <dd aria-label="free on media volume"> {freeBytes === null ? "—" : formatBytes(freeBytes)}{" "} @@ -183,11 +197,14 @@ export function StorageStage({ {location.status === "in-transition" ? " — when the move completes, or its marker is cleared below." : "."} + {location.text.readable + ? " Its text stays readable: the video pages, the index and the digests go on." + : " Its text is held too until the channel is migrated."} </p> )} {mediaBytes === null && ( <p className="text-xs text-muted-foreground"> - No audio total in this channel’s report yet — refresh the + No media-tier total in this channel’s report yet — refresh the report for a figure. The preview below measures the real tree regardless, and it is the number the move acts on. </p> @@ -238,13 +255,17 @@ export function StorageStage({ // is swept too. The controller refuses it; this is the same refusal with // a reason, one click earlier. unvouched={ - location.status === "inconsistent" || - location.status === "unreachable" || - location.status === "stalled" - ? `This channel's media location is ${location.status}: ${ - location.detail ?? "disk and config do not agree" - } Moving back would delete the relocated copy, so it is refused until the location reads "relocated · reachable".` - : null + location.status === "legacy" + ? `This channel's media cannot be moved: ${ + location.detail ?? "its media layout is the retired whole-directory one" + }.` + : location.status === "inconsistent" || + location.status === "unreachable" || + location.status === "stalled" + ? `This channel's media location is ${location.status}: ${ + location.detail ?? "disk and config do not agree" + } Moving back would delete the relocated copy, so it is refused until the location reads "relocated · reachable".` + : null } /> @@ -343,7 +364,7 @@ function MoveMedia({ // NO PREVIEW GATE ON THE WAY BACK, and it is not an oversight: the preview // answers "is there room on the destination", and the controller's move-back // asks the corpus volume that itself, charging only the bytes still missing - // from a resumed `data.incoming`. There is no root for the operator to get + // from a resumed `media.incoming`. There is no root for the operator to get // wrong either — it is the channel's own directory. const confirmed = movingBack || (named && key === previewedKey); const disabled = movingBack @@ -373,15 +394,19 @@ function MoveMedia({ {movingBack ? ( <> Copies <code>{target ?? "the target"}</code> back into the channel - dir, verifies it, replaces the symlink with a real directory and - clears the recorded location. + dir, verifies it, replaces the <code>media</code> symlink with a + real directory and clears the recorded location. The text never + moved. </> ) : ( <> - Copies <code>data/</code> to{" "} - <code><root>/{slug}/data</code>, verifies it, and leaves a - symlink behind so every reader, yt-dlp and the index keep working - unchanged. The source is not touched until the copy verifies. + Copies the media tier — <code>media/</code>, the audio and the raw + live chat — to <code><root>/{slug}/media</code>, verifies + it, and leaves a symlink behind so every reader, yt-dlp and the + index keep working unchanged. A channel whose big files are still + in <code>data/</code> is tiered into <code>media/</code> first, on + this disk. The text stays here, and the source is not touched + until the copy verifies. </> )} </p> @@ -477,7 +502,7 @@ function MoveMedia({ /> <span className="text-xs text-muted-foreground"> An absolute directory that already exists. One root holds many - channels; each gets its own <code><slug>/data</code> under it. + channels; each gets its own <code><slug>/media</code> under it. A root you expect to use again belongs on{" "} <Link href="/storage" className="underline"> Storage @@ -523,6 +548,13 @@ function MoveMedia({ <dd>{formatBytes(preview.freeOnSource)}</dd> </dl> )} + {preview && preview.tieredFirst > 0 && ( + <p aria-label="tiered first" className="text-sm text-muted-foreground"> + {preview.tieredFirst.toLocaleString()} file(s) tiered first — moved + into <code>media/</code> on this disk and linked from{" "} + <code>data/</code>, so the figures above are what the move carries. + </p> + )} {preview && preview.freeOnRoot < preview.bytesToMove && ( <p role="alert" className="text-sm text-destructive"> The destination has less free space than the media needs. The job will @@ -625,8 +657,8 @@ function StaleMarker({ {marker ? ` at phase "${marker.phase}" (${marker.direction === "out" ? "to" : "from"} ${marker.target})` : ""} - . While the marker stands this channel is skipped by every lane - and its media jobs are refused. + . While the marker stands this channel is skipped by the media + lanes and its media jobs are refused; its text stays readable. </p> <p className="text-sm text-muted-foreground"> <strong>Resume move</strong> runs the same move again from where diff --git a/editor/app/channels/[slug]/lib/videoDirFiles.ts b/editor/app/channels/[slug]/lib/videoDirFiles.ts @@ -0,0 +1,81 @@ +import path from "node:path"; +import { lstat, readdir, stat } from "node:fs/promises"; +import type { Dirent } from "node:fs"; +import { classifyEntry } from "yt-dlp-transcript-common/lib/mediaTier"; +import { onDrive } from "yt-dlp-transcript-common/lib/storageHealth"; +import type { VideoFile } from "../videos/[id]/components/cards/videoFiles"; + +// ONE VIDEO DIRECTORY'S FILES, for the video page and the videos list's detail +// pane (server only). +// +// RELEASE 17: a tiered file is a RELATIVE SYMLINK in `data/<id>/` into +// `channels/<slug>/media/`, and a dirent `isFile()` filter hides it — on a +// tiered channel the audio would vanish from both pages. So links are listed +// too, and BY FILE KIND: +// +// - a real file is statted directly (the corpus disk); +// - a link is `lstat`ed for its time (the hook gives the link the file's own +// times, so the corpus disk answers it) and its SIZE is asked of the media +// drive — every media link of the directory in ONE `onDrive(mediaDir)` call +// when the channel's media is relocated, so a drive that does not answer +// costs one budget, not one per file, and a stalled one none at all. A +// link whose target is not there (an unmounted or renamed-away drive) or +// whose drive does not answer is listed with `size: null` — the file exists; +// its bytes are not reachable now. +// +// `measureTree` (the mover's walk) keeps its own `isFile()`: over `data/` it +// measures the corpus disk's text, over `media/` the real bytes. +export async function loadVideoDirFiles( + videoDir: string, + // `config.mediaDir`, when the channel's media is relocated. Absent: an + // in-place `media/` is on the corpus disk and needs no watchdog. + mediaDir: string | undefined, +): Promise<{ files: VideoFile[] }> { + let entries: Dirent[]; + try { + entries = await readdir(videoDir, { withFileTypes: true }); + } catch { + return { files: [] }; + } + const files: VideoFile[] = []; + const links: Dirent[] = []; + for (const e of entries) { + if (e.name.includes(".tmp-")) continue; + if (e.isFile()) { + const s = await stat(path.join(videoDir, e.name)).catch(() => null); + if (s) files.push({ name: e.name, size: s.size, mtime: s.mtimeMs }); + } else if (e.isSymbolicLink()) { + links.push(e); + } + } + if (links.length > 0) { + const sizes = new Map<string, number>(); + const statLinks = async (kind: "media" | "other") => { + for (const e of links) { + const isMedia = classifyEntry(e.name) === "media"; + if ((kind === "media") !== isMedia) continue; + const s = await stat(path.join(videoDir, e.name)).catch(() => null); + if (s?.isFile()) sizes.set(e.name, s.size); + } + }; + // A link that is not a media name is not the tier's (nothing makes one); + // it is statted directly, like the files. + await statLinks("other"); + try { + await (mediaDir ? onDrive(mediaDir, () => statLinks("media")) : statLinks("media")); + } catch { + /* the drive did not answer: the sizes stay unknown */ + } + for (const e of links) { + const l = await lstat(path.join(videoDir, e.name)).catch(() => null); + if (!l) continue; + files.push({ + name: e.name, + size: sizes.get(e.name) ?? null, + mtime: l.mtimeMs, + }); + } + } + files.sort((a, b) => a.name.localeCompare(b.name)); + return { files }; +} diff --git a/editor/app/channels/[slug]/page.tsx b/editor/app/channels/[slug]/page.tsx @@ -658,7 +658,10 @@ export default async function ChannelDetailPage({ // From the loaded snapshot, not a walk. Null (rendered "—") when the // snapshot predates the field or does not exist: a 0 would claim a // measurement nobody took. - mediaBytes={snapshot.totalAudioBytes ?? null} + mediaBytes={snapshot.totalMediaBytes ?? null} + // The text tier (release 17). Null on a report written before + // it: unknown, never 0. + textBytes={snapshot.totalTextBytes ?? null} // The clips/ share of the same snapshot — the number the evict // card acts on. Null, not 0, when the field is absent: the // snapshot may predate it. diff --git a/editor/app/channels/[slug]/storageActions.ts b/editor/app/channels/[slug]/storageActions.ts @@ -16,7 +16,7 @@ // because the in-memory registry keys by slug and those jobs would be orphaned // by the move. A relocation has the same hazard with a sharper edge — a download // or a transcribe running against `data/` WHILE its bytes are being copied out -// would write into the directory the swap is about to replace, and the verify +// would write into the `media/` the swap is about to replace, and the verify // would then fail (which is the safe outcome) or the write would be lost (which // is not). The check is cheap and refuses early, before any bytes move. // @@ -32,8 +32,8 @@ import type { StreamActionResult } from "yt-dlp-transcript-common/jobs/streamCom import { clearRelocationMarker, readRelocationMarker, - relocatedDataDir, } from "yt-dlp-transcript-common/lib/channelMedia"; +import { relocatedMediaDir } from "yt-dlp-transcript-common/lib/mediaTier-server"; import { previewRelocation, relocationRootProblem, @@ -49,10 +49,12 @@ import { channelMediaBusyReason } from "../lib/mediaBusy"; export type PreviewRelocationResult = { ok: true; preview: RelocationPreview } | { ok: false; error: string }; -// Read-only: one tree walk of the channel's data dir plus two statfs calls. It -// runs INLINE rather than as a job because its whole purpose is to answer a -// question the operator is holding a form open for; a queued job with a log -// would be a worse way to show two numbers. +// One tree walk of the channel's `media/` plus two statfs calls — and, the +// first time for a classic channel, the tiering the job's preflight would do +// (same-disk renames, idempotent: release 17's ruling). It runs INLINE rather +// than as a job because its whole purpose is to answer a question the operator +// is holding a form open for; a queued job with a log would be a worse way to +// show two numbers. export async function previewRelocationAction( slug: string, dest: MoveDestination, @@ -112,7 +114,7 @@ export async function relocateChannelMediaAction( // // So this enqueues the SAME relocation job with the SAME direction, and the // controller does the resuming. What it adds is the root, which the marker does -// not carry: a marker records `<root>/<slug>/data`, so the root is its +// not carry: a marker records `<root>/<slug>/media`, so the root is its // grandparent. That inversion is checked rather than assumed — if rebuilding // the target from the derived root does not give back the marker's own target // (a hand-edited marker, a slug with a separator in it, a future layout), this @@ -162,16 +164,27 @@ async function resumeRelocation( reconcile ? "reconciling its move" : "resuming its move", ); if (refusal) return { ok: false, error: refusal }; + // A TIER MIGRATION'S MARKER (release 17) is not this panel's to resume: + // the migration runs with the editor stopped and resumes itself. + if (marker.scope === "tier-migration") { + return { + ok: false, + error: + `Channel "${slug}" has a media-tier migration in flight or ` + + `interrupted (phase "${marker.phase}") — finish it with ` + + `archilyzer storage migrate-tier ${slug}, with the editor stopped.`, + }; + } if (marker.direction === "back") { return enqueueRelocation({ slug, direction: "back", reconcile }); } const root = path.dirname(path.dirname(marker.target)); - if (marker.target !== relocatedDataDir(root, slug)) { + if (marker.target !== relocatedMediaDir(root, slug)) { return { ok: false, error: `The relocation marker points at ${marker.target}, which is not ` + - `<root>/${slug}/data — the destination root cannot be recovered from ` + + `<root>/${slug}/media — the destination root cannot be recovered from ` + `it. Clear the marker and start the move again.`, }; } diff --git a/editor/app/channels/[slug]/videos/[id]/components/cards/FilesList.tsx b/editor/app/channels/[slug]/videos/[id]/components/cards/FilesList.tsx @@ -41,7 +41,8 @@ export function FilesList({ {f.name} </span> <span className="text-xs text-muted-foreground"> - {formatBytes(f.size)} · {new Date(f.mtime).toLocaleString()} + {f.size === null ? "— (media drive not reachable)" : formatBytes(f.size)}{" "} + · {new Date(f.mtime).toLocaleString()} </span> </div> {kind === "audio" || kind === "video" ? ( diff --git a/editor/app/channels/[slug]/videos/[id]/components/cards/videoFiles.ts b/editor/app/channels/[slug]/videos/[id]/components/cards/videoFiles.ts @@ -5,7 +5,10 @@ import { export type VideoFile = { name: string; - size: number; + // NULL for a tiered file (a link into `channels/<slug>/media/`, release 17) + // whose media drive is unmounted or not answering: the file exists, its size + // is not knowable now. Rendered "—", never 0. + size: number | null; mtime: number; }; diff --git a/editor/app/channels/[slug]/videos/[id]/page.tsx b/editor/app/channels/[slug]/videos/[id]/page.tsx @@ -2,8 +2,6 @@ import type { Metadata } from "next"; import Link from "next/link"; import { notFound } from "next/navigation"; import path from "node:path"; -import { readdir, stat } from "node:fs/promises"; -import type { Dirent } from "node:fs"; import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels"; import { loadDownloadOutcome } from "yt-dlp-transcript-common/lib/downloadOutcome-server"; import { loadAvailability } from "yt-dlp-transcript-common/lib/availability-server"; @@ -42,7 +40,8 @@ import { TagsPanel } from "./components/TagsPanel"; import { MetadataHistoryDetails } from "./components/MetadataHistoryDetails"; import { loadVideoTags } from "./lib/videoTags"; import { loadVideoOperationPanels } from "./lib/videoOperationPanels"; -import { channelMediaStall } from "yt-dlp-transcript-common/lib/channelMedia"; +import { channelTextStall } from "yt-dlp-transcript-common/lib/channelMedia"; +import { loadVideoDirFiles } from "../../lib/videoDirFiles"; import { isDriveNotAnswering, onDrive, @@ -51,26 +50,15 @@ import { MediaNotAnswering } from "../../components/MediaNotAnswering"; export const dynamic = "force-dynamic"; +// The directory's files, tiered links included (release 17), each media +// link's size asked of the media drive. See lib/videoDirFiles.ts. async function loadVideoDir( slug: string, videoId: string, + mediaDir: string | undefined, ): Promise<{ files: VideoFile[] }> { const dir = path.join(getPaths().channelsDir, slug, "data", videoId); - let entries: Dirent[]; - try { - entries = await readdir(dir, { withFileTypes: true }); - } catch { - return { files: [] }; - } - const files: VideoFile[] = []; - for (const e of entries) { - if (!e.isFile()) continue; - if (e.name.includes(".tmp-")) continue; - const s = await stat(path.join(dir, e.name)); - files.push({ name: e.name, size: s.size, mtime: s.mtimeMs }); - } - files.sort((a, b) => a.name.localeCompare(b.name)); - return { files }; + return loadVideoDirFiles(dir, mediaDir); } // metadata.info.json when the video was downloaded, else the channel's @@ -86,9 +74,11 @@ export async function generateMetadata({ params: Promise<{ slug: string; id: string }>; }): Promise<Metadata> { const { slug, id } = await params; - // The title is read off the drive; a drive that is not answering is not - // asked, and one that does not answer within the budget - // (`storage.health.budgetMs`, 3 s by default) is given up on. + // The title is TEXT, on the corpus disk (release 17). Only a channel still on + // the retired whole-directory layout has its text on another drive: that + // drive is not asked while it is not answering, and one that does not answer + // within the budget (`storage.health.budgetMs`, 3 s by default) is given up + // on. const drive = (await readChannelConfig(getPaths(), slug))?.dataDir?.trim(); let subject = id; try { @@ -110,18 +100,24 @@ export default async function VideoDetailPage({ const { slug, id } = await params; const config = await readChannelConfig(getPaths(), slug); if (!config) notFound(); - // Everything below reads the video's directory on the channel's drive; a - // drive that is not answering is not read. See MediaNotAnswering. - const stall = channelMediaStall(config); + // Everything below reads the video's TEXT, which is on the corpus disk + // (release 17): a stalled MEDIA drive does not hold this page — its audio + // is listed with its size unknown, and the file route answers 503 for it. + // Only a legacy channel (text on its retired `dataDir`) is held here. See + // MediaNotAnswering. + const stall = channelTextStall(config); if (stall) { return <MediaNotAnswering slug={slug} stall={stall} what="This video's page" />; } - // EVERY READ BELOW IS OF THIS VIDEO'S DIRECTORY, on the channel's drive when - // it is relocated, so they go through the watchdog as one unit: a drive that - // has not answered them within the budget (3 s by default) is marked stalled - // and the page says so. + const mediaDir = config.mediaDir?.trim() || undefined; + // On a legacy channel EVERY READ BELOW is of a directory on its retired + // drive, so they go through the watchdog as one unit: a drive that has not + // answered them within the budget (3 s by default) is marked stalled and the + // page says so. On every other channel they read the corpus disk directly; + // the media links' sizes go through the media drive's watchdog on their own + // (lib/videoDirFiles.ts). const loadAll = async () => { - const dirData = await loadVideoDir(slug, id); + const dirData = await loadVideoDir(slug, id, mediaDir); const meta = await loadMeta(slug, id); const videoDir = path.join(getPaths().channelsDir, slug, "data", id); const downloadOutcome = await loadDownloadOutcome(videoDir); diff --git a/editor/app/channels/[slug]/videos/[id]/videoActions.ts b/editor/app/channels/[slug]/videos/[id]/videoActions.ts @@ -1,7 +1,7 @@ "use server"; import path from "node:path"; -import { readdir, readFile, rm, stat } from "node:fs/promises"; +import { lstat, readdir, readFile, rm, stat } from "node:fs/promises"; import { revalidatePath } from "next/cache"; import { safeRevalidate } from "../../../../lib/safeRevalidate"; import { redirect } from "next/navigation"; @@ -41,6 +41,7 @@ import { inspectChannelMedia, } from "yt-dlp-transcript-common/lib/channelMedia"; import { onDrive } from "yt-dlp-transcript-common/lib/storageHealth"; +import { classifyEntry } from "yt-dlp-transcript-common/lib/mediaTier"; import { setExcludedFromTruncatedCheck } from "yt-dlp-transcript-common/lib/excludeTruncatedCheck-server"; import { pruneFailedTranscriptions } from "yt-dlp-transcript-common/controller/failedTranscriptions"; import { transcodeAudio } from "yt-dlp-transcript-common/controller/transcode"; @@ -498,10 +499,31 @@ export async function deleteVideoFileAction( if (!target) { return { ok: false, error: `Refusing to delete suspicious filename "${filename}"` }; } + // A media file's stat is asked of the media tier's drive, through its + // watchdog (release 17: the file may be a link onto a relocated drive). + const mediaDrive = + classifyEntry(path.basename(target)) === "media" + ? (await readChannelConfig(getPaths(), slug))?.mediaDir?.trim() + : undefined; let s; try { - s = await stat(target); + s = await (mediaDrive + ? onDrive(mediaDrive, () => stat(target)) + : stat(target)); } catch { + // A TIERED FILE WHOSE DRIVE IS NOT THERE: removing the link alone would + // orphan its bytes on that drive, so nothing is removed. + const isLink = await lstat(target) + .then((l) => l.isSymbolicLink()) + .catch(() => false); + if (isLink) { + return { + ok: false, + error: + `${filename} is on this channel's media drive, which is not mounted ` + + `or not answering — nothing was deleted.`, + }; + } return { ok: false, error: `File not found: ${filename}` }; } if (!s.isFile()) { diff --git a/editor/app/channels/[slug]/videos/page.tsx b/editor/app/channels/[slug]/videos/page.tsx @@ -1,6 +1,4 @@ import path from "node:path"; -import { readdir, stat } from "node:fs/promises"; -import type { Dirent } from "node:fs"; import type { Metadata } from "next"; import { notFound } from "next/navigation"; import { isSocialChannel } from "yt-dlp-transcript-common/lib/channelConfig"; @@ -36,7 +34,8 @@ import { computeVideoRows, readDataDirVideoIds } from "../lib/videoRowsServer"; import { normalizeBuckets } from "yt-dlp-transcript-common/views/pipeline/stageStatus"; import { attachCuratedTags } from "../lib/videoTagRows"; import { readChannelVideoTitles } from "yt-dlp-transcript-common/controller/videoTitles"; -import { channelMediaStall } from "yt-dlp-transcript-common/lib/channelMedia"; +import { channelTextStall } from "yt-dlp-transcript-common/lib/channelMedia"; +import { loadVideoDirFiles } from "../lib/videoDirFiles"; import { isDriveNotAnswering, onDrive, @@ -45,26 +44,14 @@ import { MediaNotAnswering } from "../components/MediaNotAnswering"; export const dynamic = "force-dynamic"; +// The selected video's files, tiered links included (release 17): a dirent +// `isFile()` filter hides them. See lib/videoDirFiles.ts. async function loadVideoDir( channelDataDir: string, videoId: string, + mediaDir: string | undefined, ): Promise<{ files: VideoFile[] }> { - const dir = path.join(channelDataDir, videoId); - let entries: Dirent[]; - try { - entries = await readdir(dir, { withFileTypes: true }); - } catch { - return { files: [] }; - } - const files: VideoFile[] = []; - for (const e of entries) { - if (!e.isFile()) continue; - if (e.name.includes(".tmp-")) continue; - const s = await stat(path.join(dir, e.name)); - files.push({ name: e.name, size: s.size, mtime: s.mtimeMs }); - } - files.sort((a, b) => a.name.localeCompare(b.name)); - return { files }; + return loadVideoDirFiles(path.join(channelDataDir, videoId), mediaDir); } export async function generateMetadata({ @@ -106,10 +93,12 @@ export default async function ChannelVideosPage({ // A social channel has posts, not videos — there is no data directory to list // and nothing here would render. 404 rather than an empty workspace. if (isSocialChannel(config)) notFound(); - // THE LIST IS READ OFF THE DRIVE (a readdir of data/, a head read per title, - // the selected video's files), so a drive that is not answering is not read: - // the page says so instead. See MediaNotAnswering. - const stall = channelMediaStall(config); + // THE LIST IS TEXT (a readdir of data/, a head read per title, the selected + // video's files), on the corpus disk since release 17: a stalled MEDIA drive + // does not hold it. Only a legacy channel's text is on another drive, and a + // drive that is not answering is not read: the page says so instead. See + // MediaNotAnswering. + const stall = channelTextStall(config); if (stall) { return <MediaNotAnswering slug={slug} stall={stall} what="The video list" />; } @@ -144,11 +133,12 @@ export default async function ChannelVideosPage({ ); const channelDataDir = path.join(paths.channelsDir, slug, "data"); - // THE READS OF THE DRIVE go through the watchdog when the channel is - // relocated: a drive that has not answered them within the budget - // (`storage.health.budgetMs`, 3 s by default) is marked stalled and the page - // says so instead (see MediaNotAnswering). + // THE READS OF A LEGACY CHANNEL'S DRIVE go through the watchdog: a drive + // that has not answered them within the budget (`storage.health.budgetMs`, + // 3 s by default) is marked stalled and the page says so instead (see + // MediaNotAnswering). Every other channel's text is on the corpus disk. const drive = config.dataDir?.trim(); + const mediaDir = config.mediaDir?.trim() || undefined; const onMedia = <T,>(call: () => Promise<T>): Promise<T> => drive ? onDrive(drive, call) : call(); let channelDataDirIds: string[]; @@ -221,7 +211,7 @@ export default async function ChannelVideosPage({ let loadedVideo; try { loadedVideo = await onMedia(() => Promise.all([ - loadVideoDir(channelDataDir, selectedVideoId), + loadVideoDir(channelDataDir, selectedVideoId, mediaDir), // Already read for the list — the title map covers every row. Promise.resolve(titles.get(selectedVideoId)?.title ?? null), loadDownloadOutcome(videoDir), diff --git a/editor/app/channels/bulkStorageActions.ts b/editor/app/channels/bulkStorageActions.ts @@ -47,10 +47,9 @@ import { // so the bar renders it the same way they do. export type BulkRelocateResult = QueueOutcome; -// Follows the link, deliberately: for a relocated channel `data/` is a symlink -// and what matters is whether the thing it points at is there. (This path only -// reaches it for a channel inspect() already called in-place, so in practice it -// is a real directory or nothing.) +// Follows a link, deliberately. This path only reaches it for a channel +// inspect() already called in-place, whose `data/` is a real directory on the +// corpus disk or nothing. async function isDirectory(p: string): Promise<boolean> { try { return (await stat(p)).isDirectory(); @@ -116,6 +115,11 @@ export async function bulkRelocateChannelMediaAction( if (media.marker) { return `a relocation (${media.marker.direction}) to ${media.marker.target} is already in flight`; } + // THE RETIRED LAYOUT (release 17) is migrated, never moved: the skip + // names the command. + if (media.status === "legacy") { + return `cannot be moved — ${media.detail ?? "its media layout is the retired whole-directory one"}`; + } // ALREADY RELOCATED IS A SKIP, NOT A FAILURE — including to a DIFFERENT // root. Selecting the whole page and pressing Move is the ordinary way // this gets used, and the channels already on the platter are exactly the @@ -123,7 +127,7 @@ export async function bulkRelocateChannelMediaAction( if (media.relocated) { return `already relocated to ${media.target ?? "another root"}`; } - // Containment is per-channel because the target is: `<root>/<slug>/data` + // Containment is per-channel because the target is: `<root>/<slug>/media` // can resolve into one channel's directory and not another's. return relocationRootProblem({ paths, slug, root: chosen }); }, diff --git a/editor/app/channels/components/ChannelForm.tsx b/editor/app/channels/components/ChannelForm.tsx @@ -717,11 +717,11 @@ export function ChannelForm({ /> {/* READ-ONLY, AND NOT AN INPUT — the one field on this page that is a record of the disk rather than an instruction to it. - `config.dataDir` is written ONLY by the relocate job, on success, + `config.mediaDir` is written ONLY by the relocate job, on success, after the bytes are copied, verified and the symlink is in place. An editable text box here would let the two disagree with a keystroke: type a path nothing was moved to and every reader - follows a `data/` link that still points somewhere else, which + follows a `media` link that still points somewhere else, which inspectChannelMedia reports as `inconsistent` and refuses to guess past. So the move is the only writer, and this line just says what it wrote. (It is also not in CHANNEL_FORM_FIELDS, so saving this @@ -732,14 +732,17 @@ export function ChannelForm({ aria-label="media location" className="font-mono text-xs break-all rounded border border-border bg-muted px-2 py-1" > - {c?.dataDir?.trim() - ? c.dataDir.trim() - : "In the channel directory (data/)"} + {c?.mediaDir?.trim() + ? c.mediaDir.trim() + : c?.dataDir?.trim() + ? `${c.dataDir.trim()} (the retired whole-directory layout — run archilyzer storage migrate-tier)` + : "In the channel directory (media/)"} </span> <span className="text-xs text-muted-foreground"> - Where this channel’s downloaded media actually lives. Change it - from the Storage panel, which copies and verifies the bytes before - recording anything here. + Where this channel’s big files — the audio and the raw live + chat — actually live; its text is always in the channel directory. + Change it from the Storage panel, which copies and verifies the + bytes before recording anything here. </span> </div> </Section> diff --git a/editor/app/channels/lib/relocationJob.ts b/editor/app/channels/lib/relocationJob.ts @@ -109,7 +109,10 @@ export async function enqueueRelocation(opts: { // fail the verify with 131 GB correctly on the far side. (result.retried ? " (a change made during the copy was settled by a second mirror pass)" - : ""), + : "") + + // A classic channel's first move tiers it in place first (release + // 17): its big files renamed into `media/` and linked from `data/`. + (result.tiered > 0 ? ` (${result.tiered} file(s) tiered first)` : ""), ); // No snapshot regen — deliberately, and `relocate-channel-media` is in // NO_REGEN_KINDS so the central hook does not arm one either. The move diff --git a/editor/app/components/MediaLocationBadge.tsx b/editor/app/components/MediaLocationBadge.tsx @@ -10,19 +10,25 @@ import type { ChannelRowMedia } from "yt-dlp-transcript-common/views/channelRow" // the filesystem; the inspect() call that produces the location happens on the // server, once per row, and only its result travels. // -// WHAT THE SIX STATUSES LOOK LIKE, and why there are only three appearances: +// WHAT THE SEVEN STATUSES LOOK LIKE, and why there are only three appearances: // // in-place → NOTHING. The overwhelming majority of channels are in place, // and a badge on every row saying "normal" is noise that makes // the two that matter harder to see, not easier. // ok → neutral. Relocated and reachable is a fact worth stating (the // bytes are not on the corpus disk) but it is not a problem. -// everything → red. unreachable, in-transition, inconsistent and stalled are -// else all "do not trust what this channel's dirs say right now": the -// first because the drive is not mounted, the second because a -// move is half-done, the third because disk and config disagree -// and nothing here is willing to guess which one is right, the -// fourth because the drive is mounted and not answering. +// everything → red. unreachable, in-transition, inconsistent, stalled and +// else legacy are all "do not trust what this channel's media says +// right now": the first because the drive is not mounted, the +// second because a move is half-done, the third because disk and +// config disagree and nothing here is willing to guess which one +// is right, the fourth because the drive is mounted and not +// answering, the fifth because the channel is still on the +// retired whole-directory layout and is held until +// `archilyzer storage migrate-tier` runs (release 17). +// +// Release 17: "media" is the media TIER — `channels/<slug>/media`, the big +// files. A channel's text is on the corpus disk whatever this badge says. // // The `detail` string is the operator's prose from inspect() — the drive path, // the phase, the disagreement — and it goes on `title` so a row badge carries @@ -54,8 +60,9 @@ const LABELS: Record<ChannelMediaStatus, string | null> = { "in-transition": "Media moving", inconsistent: "Media inconsistent", stalled: "Media not answering", - // Release 17: the retired whole-directory layout (`archilyzer storage - // migrate-tier`). Added by slice T1 so the status union stays exhaustive. + // Release 17: the retired whole-directory layout, held until + // `archilyzer storage migrate-tier` runs. Danger, like every non-ok status; + // the detail on the title names the command. legacy: "Media layout retired", }; @@ -76,7 +83,7 @@ const SHORT_STATUS: Record<ChannelMediaStatus, string | null> = { // caller that could not inspect). Both are the same instruction to a renderer. // // `locationLabel` IS THE NAME OF THE STORAGE LOCATION the channel's media sits -// on — `locationOfDataDir(config.dataDir, settings.storage.locations)`, resolved +// on — `locationOfDataDir(config.mediaDir, settings.storage.locations)`, resolved // on the SERVER where the settings are, and passed down as a string. Undefined // when it sits on none: a root the operator typed by hand, or a corpus with no // locations configured.