import type { ChannelConfig } from "../../lib/channelConfig"; import { digestWorkOf, excludedDownloadIdSet, type ChannelSnapshot, } from "../../controller/channelSnapshot"; import { backfillLaneEntriesOf, operationsGroupLabel, reachableOperationWork, type OperationGroup, } from "../../lib/operations"; // TYPE ONLY. channelMedia.ts imports node:fs, and this module is imported by // three client components (AttentionStrip, NextAction, OverviewPanel) for its // types. A type import is erased, a value import would not be. import type { ChannelMediaLocation } from "../../lib/channelMedia"; export type SnapshotBuckets = ChannelSnapshot["buckets"]; // Older snapshots on disk may pre-date some bucket fields. Normalize to // always-present arrays so callers can read .length without guards. export function normalizeBuckets( raw: Partial | undefined, ): SnapshotBuckets { return { noTranscript: raw?.noTranscript ?? [], downloadedNoTranscript: raw?.downloadedNoTranscript ?? [], wrongFormatAudio: raw?.wrongFormatAudio ?? [], multipleAudioFormats: raw?.multipleAudioFormats ?? [], transcribedWithAudio: raw?.transcribedWithAudio ?? [], untranscribable: raw?.untranscribable ?? [], noMetadata: raw?.noMetadata ?? [], failedListed: raw?.failedListed ?? [], missingFromArchive: raw?.missingFromArchive ?? [], duplicateDirs: raw?.duplicateDirs ?? [], partialDownloads: raw?.partialDownloads ?? [], corruptSource: raw?.corruptSource ?? [], corruptFullSource: raw?.corruptFullSource ?? [], nonStandardVtt: raw?.nonStandardVtt ?? [], skippedByFilter: raw?.skippedByFilter ?? [], skippedByTitleFilter: raw?.skippedByTitleFilter ?? [], chatOnly: raw?.chatOnly ?? [], chatOnlyPending: raw?.chatOnlyPending ?? [], incompleteTranscript: raw?.incompleteTranscript ?? [], shortAudio: raw?.shortAudio ?? [], autoSubsOnly: raw?.autoSubsOnly ?? [], downloadedAutoSubsOnly: raw?.downloadedAutoSubsOnly ?? [], supersededAutoSubs: raw?.supersededAutoSubs ?? [], needsCookies: raw?.needsCookies ?? [], }; } export type StageId = | "configure" | "playlist" | "download" | "transcribe" | "digest" // THE OPERATIONS, NOT THE QUEUE. This card was called "backfill" — a queue key // wearing a stage's name. Nobody can arm, pause or run "a backfill"; what the // card actually holds is the speaker work (diarization and the two attribution // kinds), which is a thing an operator recognises. The queue key BACKFILL_QUEUE // is untouched: it is a scheduler key and correctly named as one. // // ?stage=backfill still resolves here — see STAGE_ALIASES in page.tsx. | "speakers" | "cleanup" | "diagnostics" // WHERE THE MEDIA PHYSICALLY IS (plans/relocate-channel-media.md). A channel // CHORE like cleanup and diagnostics, not an operation — nothing registers it // and no group owns it — so it is hand-listed at the call site alongside them // and is deliberately absent from GROUP_STAGES below. | "storage" | "danger"; // THE STAGES EACH OPERATION GROUP OWNS, in the group's own order. // // Record<> is EXHAUSTIVE, which is the whole point: a new OperationGroup does // not compile until it names its stage(s). Before this, the channel page's stage // list was a hand-written literal, so a registered operation in a new group got // a card only if someone remembered to add one — and the failure mode was a // silent absence, not an error. // // NOT 1:1 with the groups, and no honest derivation makes it so: `sync` owns // NO stage (below), and a group may own more than one — `media` did while the // transcode stage existed (retired 2026-08-30). So this is a Record of ARRAYS, // spread in OPERATION_GROUP_ORDER — which yields exactly today's order and // today's cardinality. It is a compile-time membership check, not a re-shaping // of the page. // // DELIBERATELY ONLY THE MIDDLE. configure/playlist and cleanup/diagnostics/ // danger are channel CHORES, not operations — nothing registers them and no // group owns them — so they stay hand-listed at the call site. Do not "finish" // this derivation by inventing groups for them. export const GROUP_STAGES: Record = { media: ["download"], transcript: ["transcribe"], digest: ["digest"], speakers: ["speakers"], // Empty on purpose, and this is the paragraph above in practice: sync's // channel surface is the hand-listed `playlist` bookend (JOB_KIND_TO_STAGE // maps the sync job kind to it), which is a channel chore, not a stage this // Record owns. Do not "finish" the derivation by giving sync a stage here — // the channel page would grow a second, duplicate playlist card. sync: [], }; export type StageTone = "neutral" | "attention" | "danger" | "running" | "ok"; export type StageStatus = { id: StageId; title: string; pending: number; failed: number; running: boolean; defaultOpen: boolean; summary: string; tone: StageTone; }; const JOB_KIND_TO_STAGE: Record = { "store-playlist": "playlist", sync: "playlist", "metadata-scan": "playlist", "download-from-playlist": "download", "download-missing": "download", "whisper-all": "transcribe", "whisper-retry": "transcribe", "transcribe-one": "transcribe", "whisper-video": "transcribe", "whisper-bucket-auto-subs": "transcribe", "digest-channel-local": "digest", "digest-channel-remote": "digest", "digest-share-cluster": "digest", "backfill-channel": "speakers", // The pre-registry per-channel diarization button lands on the channel queue // but is the same work this card is about, so it lights this card too. "diarize-channel": "speakers", "clean-audio-transcribed": "cleanup", "purge-superseded-auto-subs": "cleanup", "clean-extra-audio-formats": "cleanup", "remove-wrong-format-audio": "cleanup", "check-availability": "diagnostics", }; function pluralize(n: number, singular: string, plural?: string): string { return `${n} ${n === 1 ? singular : plural ?? `${singular}s`}`; } function pickTone(args: { running: boolean; pending: number; failed: number; fallback?: StageTone; }): StageTone { if (args.running) return "running"; if (args.failed > 0) return "danger"; if (args.pending > 0) return "attention"; return args.fallback ?? "neutral"; } export type ComputeStageStatusesInput = { snapshot: ChannelSnapshot; failedVideoIds: string[]; config: ChannelConfig; // Only `status` and `kind` are read (which stage has work in flight), so this // takes the SHAPE rather than the record: the channel page now gets its rows // from the one job-row builder (liveJobRows) and no longer holds JobRecords. runningJobs: ReadonlyArray<{ status: string; kind: string }>; // Whether ANY backfill kind is switched on. // // The snapshot's per-kind counts are a record of what was true when it was // written, and they survive the operator turning the feature off. Without // this flag the card reports "7 videos need derived data" — amber, with a // count — directly above its own body copy saying "No backfill is enabled", // and nothing would ever run the work it is advertising. Defaults to true so // an omitted flag behaves as it always did. backfillEnabled?: boolean; // The ids of the lane's enabled kinds, so the stage can be titled after what // it HOLDS rather than after its queue key. Optional and defaulting to the // generic name, because a caller that only needs tone and counts should not // have to resolve the registry. backfillKindIds?: ReadonlyArray; // Where this channel's media actually is, from inspectChannelMedia. Optional // because the two stats it costs belong to the caller that already has the // config in hand, and a caller that only wants tone and counts should not // have to do I/O to get them — an omitted location reads as "in place", which // is what every channel was before relocation existed. media?: ChannelMediaLocation | null; }; export function computeStageStatuses( input: ComputeStageStatusesInput, ): Record { const { snapshot, failedVideoIds, config, runningJobs, backfillEnabled = true, backfillKindIds, media, } = input; const buckets = normalizeBuckets(snapshot.buckets); const undownloadedIds = snapshot.undownloadedIds ?? []; const excludedDownloadIds = excludedDownloadIdSet(snapshot); const actionableNoTranscript = buckets.noTranscript.filter( (id) => !excludedDownloadIds.has(id), ); const actionableDownloadedNoTranscript = buckets.downloadedNoTranscript.filter( (id) => !excludedDownloadIds.has(id), ); const runningByStage = new Set(); for (const job of runningJobs) { if (job.status !== "running" && job.status !== "queued") continue; const stage = JOB_KIND_TO_STAGE[job.kind]; if (stage) runningByStage.add(stage); } const downloadPending = undownloadedIds.length + actionableNoTranscript.length + buckets.partialDownloads.length; const transcribePending = actionableDownloadedNoTranscript.length; const transcribeFailed = failedVideoIds.length; const cleanupPending = buckets.multipleAudioFormats.length; // `untranscribable` is deliberately excluded: those videos are an intentional // user decision ("mark untranscribable"), not an anomaly with an action. const diagnosticsPending = buckets.noMetadata.length + buckets.missingFromArchive.length + buckets.duplicateDirs.length; // Cards that carry primary actions stay open by default so the user can // always reach the buttons; the summary line communicates idle/busy state // instead of collapsing the controls out of sight. Configure collapses once // the channel has a URL; Danger zone stays collapsed unless opened. const configure: StageStatus = { id: "configure", title: "Configure", pending: 0, failed: 0, running: false, defaultOpen: true, summary: config.url ? `${config.handling}${config.platform ? ` · ${config.platform}` : ""}` : "Channel has no URL — open to configure.", tone: config.url ? "neutral" : "attention", }; const playlistRunning = runningByStage.has("playlist"); const playlist: StageStatus = { id: "playlist", title: "Playlist", pending: 0, failed: 0, running: playlistRunning, defaultOpen: true, summary: playlistRunning ? "Running…" : config.lastSyncedAt ? `Last sync ${new Date(config.lastSyncedAt).toLocaleString()}` : "Never synced.", tone: pickTone({ running: playlistRunning, pending: 0, failed: 0 }), }; const downloadRunning = runningByStage.has("download"); const downloadParts: string[] = []; if (undownloadedIds.length > 0) { downloadParts.push(pluralize(undownloadedIds.length, "undownloaded")); } if (actionableNoTranscript.length > 0) { downloadParts.push( pluralize( actionableNoTranscript.length, "dir missing transcript & audio", "dirs missing transcript & audio", ), ); } if (buckets.partialDownloads.length > 0) { downloadParts.push( pluralize( buckets.partialDownloads.length, "partial download", "partial downloads", ), ); } if (buckets.corruptSource.length > 0) { downloadParts.push( pluralize( buckets.corruptSource.length, "corrupt source (needs re-download)", "corrupt sources (need re-download)", ), ); } if (buckets.corruptFullSource.length > 0) { downloadParts.push( pluralize( buckets.corruptFullSource.length, "corrupt full source (file kept)", "corrupt full sources (files kept)", ), ); } const download: StageStatus = { id: "download", title: "Download", pending: downloadPending, failed: 0, running: downloadRunning, defaultOpen: true, summary: downloadRunning ? "Running…" : downloadParts.length > 0 ? downloadParts.join(" · ") : "Nothing to download.", tone: pickTone({ running: downloadRunning, pending: downloadPending, failed: 0, }), }; const transcribeRunning = runningByStage.has("transcribe"); const transcribeParts: string[] = []; if (actionableDownloadedNoTranscript.length > 0) { transcribeParts.push( pluralize( actionableDownloadedNoTranscript.length, "video awaiting whisper", "videos awaiting whisper", ), ); } if (failedVideoIds.length > 0) { transcribeParts.push(pluralize(failedVideoIds.length, "failed")); } // Informational only — the replace-auto-captions lane is opt-in, so these are // NOT counted as pending work (that would light every YouTube channel up // amber forever). const autoSubsCandidates = buckets.autoSubsOnly.length + buckets.downloadedAutoSubsOnly.length; if (autoSubsCandidates > 0) { transcribeParts.push( pluralize( autoSubsCandidates, "video with only auto-captions", "videos with only auto-captions", ), ); } const transcribe: StageStatus = { id: "transcribe", title: "Transcribe", pending: transcribePending, failed: transcribeFailed, running: transcribeRunning, defaultOpen: true, summary: transcribeRunning ? "Running…" : transcribeParts.length > 0 ? transcribeParts.join(" · ") : "All transcribed.", tone: pickTone({ running: transcribeRunning, pending: transcribePending, failed: transcribeFailed, }), }; // Videos whose digest is missing, stale or part-done against the local lane's // current identity. Read from the operation registry via digestWorkOf, which // is the one definition: a channel with no entry reports unknown coverage // rather than reading as fully digested. // // Counted as pending work rather than merely informational: unlike the // auto-captions lane, every transcribed video is eventually meant to have one. // Videos BLOCKED on transcription are deliberately not in this number — there // is nothing the digest lane can do about them — and the stage card names them // separately. const digestRunning = runningByStage.has("digest"); const digestWork = digestWorkOf(snapshot); const digestPending = digestWork.reachable; const digest: StageStatus = { id: "digest", title: "Digest", pending: digestPending, failed: 0, running: digestRunning, defaultOpen: true, summary: digestRunning ? "Running…" : digestPending > 0 ? pluralize( digestPending, "transcript needs a digest", "transcripts need a digest", ) : // "All digested" MUST NOT be said over a channel that simply has // nothing to digest yet. Before the registry classified them, videos // with no transcript were absent from every digest bucket, so a // channel of untranscribed videos read as finished — the exact failure // declaring the transcription dependency exists to end. digestWork.blocked > 0 ? pluralize( digestWork.blocked, "video is waiting on a transcript", "videos are waiting on transcripts", ) : "All digested at the current settings.", tone: pickTone({ running: digestRunning, pending: digestPending, failed: 0, fallback: "ok", }), }; // The backfill lane's work list, summed across every registered kind. // // `pending` counts ONLY the reachable half. The needs-re-acquiring population // is reported in the summary line and never folded in: it is 91x larger on the // measured corpus, so counting it would hold every channel permanently amber // for work that cannot be done without an opt-in re-download — precisely the // trap /api/widget/actionable documents for the digest work count. const backfillRunning = runningByStage.has("speakers"); // backfillLaneEntriesOf, not Object.values. The snapshot's per-kind map carries every // operation in the catalog now, including digest — which runs on its own queue // key, has its own stage card directly above, and would otherwise add ~75,000 // videos to this instrument on the measured corpus. The filter is by the kind's // declared lane rather than by its id, so the next operation registered on a // lane of its own does not re-arm the same trap. // A disabled lane has NO work, whatever the snapshot recorded before it was // switched off — see `backfillEnabled` above. const backfillEntries = backfillEnabled ? backfillLaneEntriesOf(snapshot.backfill) : []; const backfillPending = backfillEntries.reduce( (n, e) => n + reachableOperationWork(e), 0, ); const backfillMissingInput = backfillEntries.reduce( (n, e) => n + e.missingInput, 0, ); // Same treatment as missingInput: reported in the summary line, never folded // into `pending`. `?? 0` because snapshots written before the cap existed have // no such field. const backfillDeferred = backfillEntries.reduce( (n, e) => n + (e.deferred ?? 0), 0, ); // Same treatment again, and the wording matters: this number falls on its own // as the prerequisite lane runs, so it must not read as something to fix. const backfillBlocked = backfillEntries.reduce( (n, e) => n + (e.blocked ?? 0), 0, ); const backfillParts: string[] = []; if (backfillPending > 0) { backfillParts.push( pluralize( backfillPending, "video needs derived data", "videos need derived data", ), ); } if (backfillMissingInput > 0) { backfillParts.push( `${backfillMissingInput.toLocaleString()} needing media re-acquired`, ); } if (backfillDeferred > 0) { backfillParts.push( `${backfillDeferred.toLocaleString()} deferred (too long to diarize)`, ); } if (backfillBlocked > 0) { backfillParts.push( `${backfillBlocked.toLocaleString()} waiting on an earlier backfill`, ); } const speakers: StageStatus = { id: "speakers", // NAMED AFTER THE OPERATIONS, NOT THE QUEUE. "Backfill" is a scheduler key // that on this install stands for three different operations; nobody can // arm, pause or run "a backfill". Derived, so a lane that gains a kind from // another group degrades to "Derived data" rather than going stale. title: operationsGroupLabel(backfillKindIds ?? []), pending: backfillPending, failed: 0, running: backfillRunning, defaultOpen: true, summary: backfillRunning ? "Running…" : backfillEntries.length === 0 ? "Nothing here is enabled." : backfillParts.length > 0 ? backfillParts.join(" · ") : "Everything reachable is current.", tone: pickTone({ running: backfillRunning, pending: backfillPending, failed: 0, // Neutral rather than "ok" when nothing is enabled: an empty work list // because a feature is off is not the same as being finished. fallback: backfillEntries.length === 0 ? "neutral" : "ok", }), }; const cleanupRunning = runningByStage.has("cleanup"); const cleanupParts: string[] = []; if (cleanupPending > 0) { cleanupParts.push( pluralize( cleanupPending, "dir has extra audio formats", "dirs have extra audio formats", ), ); } // Kept auto-caption backups. Informational (not folded into `pending`): they // are deliberately retained until purged by hand, so they are inventory, not // a chore. if (buckets.supersededAutoSubs.length > 0) { cleanupParts.push( pluralize( buckets.supersededAutoSubs.length, "superseded auto-caption backup", "superseded auto-caption backups", ), ); } const cleanup: StageStatus = { id: "cleanup", title: "Cleanup", pending: cleanupPending, failed: 0, running: cleanupRunning, defaultOpen: true, summary: cleanupRunning ? "Running…" : cleanupParts.length > 0 ? cleanupParts.join(" · ") : "Nothing to clean.", tone: pickTone({ running: cleanupRunning, pending: cleanupPending, failed: 0, }), }; const diagnosticsRunning = runningByStage.has("diagnostics"); const diagnostics: StageStatus = { id: "diagnostics", title: "Diagnostics", pending: diagnosticsPending, failed: 0, running: diagnosticsRunning, defaultOpen: true, summary: diagnosticsRunning ? "Running…" : diagnosticsPending > 0 ? pluralize(diagnosticsPending, "anomaly", "anomalies") : "All clear.", tone: pickTone({ running: diagnosticsRunning, pending: diagnosticsPending, failed: 0, fallback: "ok", }), }; // WHERE THE MEDIA IS. The only stage whose tone comes from a filesystem fact // rather than from a count: an unreachable channel is a channel whose numbers // everywhere else on this page are about to be wrong (an unmounted drive reads // as "nothing downloaded"), so this card is red the moment inspect() says so // and neutral the rest of the time. "in-place" is not an achievement, so it is // never "ok" — the fallback tone for a healthy relocation is neutral too. const mediaStatus = media?.status ?? "in-place"; const storage: StageStatus = { id: "storage", title: "Storage", pending: 0, failed: 0, running: mediaStatus === "in-transition", defaultOpen: true, summary: mediaStatus === "in-place" ? "Media is in the channel directory." : mediaStatus === "ok" ? `Media relocated to ${media?.target ?? "another drive"}.` : (media?.detail ?? mediaStatus), tone: mediaStatus === "in-transition" ? "running" : mediaStatus === "unreachable" || mediaStatus === "inconsistent" || mediaStatus === "stalled" ? "danger" : "neutral", }; const danger: StageStatus = { id: "danger", title: "Danger zone", pending: 0, failed: 0, running: false, defaultOpen: true, summary: "Delete this channel.", tone: "neutral", }; return { configure, playlist, download, transcribe, digest, speakers, cleanup, diagnostics, storage, danger, }; }