Archilyzer · Source

archilyzer

Archilyzer
git clone https://archilyzer.pages.dev/source/archilyzer.git
Log | Files | Refs | README | LICENSE

commit 6995cc817f98308b72eeb0e6e37ad093cabc122d
parent 4d4aa8b2543272762a152d460be7bbfd9d701fdd
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date:   Tue, 19 May 2026 20:32:33 -0400

more UI improvements

Diffstat:
Mcommon/controller/whisperBatch.ts | 16+++++++++++++++-
Mcommon/lib/availability.ts | 53+++++++++++++++++++++++++++++++++++++++++++++++++++++
Mcommon/ytdlp/runYtdlp.ts | 16++++++++++++++--
Aeditor/app/channels/[slug]/components/VideoListPaneSection.tsx | 76++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Meditor/app/channels/[slug]/components/stages/TranscribeStage.tsx | 336++++++++++++++++++++++++++++++++++++++++++++++++-------------------------------
Meditor/app/channels/[slug]/page.tsx | 44+++++++++++++++++++++++---------------------
Meditor/app/channels/[slug]/whisperActions.ts | 39+++++++++++++++++++++++++++++++++++++++
Meditor/app/layout.tsx | 2+-
8 files changed, 426 insertions(+), 156 deletions(-)

diff --git a/common/controller/whisperBatch.ts b/common/controller/whisperBatch.ts @@ -19,6 +19,10 @@ export type WhisperBatchOptions = { reverse?: boolean; shardTotal?: number; shardIndex?: number; + // When set, only consider these video IDs (intersected with what's on + // disk). Used by bucket-scoped actions like "Transcribe downloaded audio" + // where the caller has already determined the exact set. + ids?: string[]; onLog?: (msg: string) => void; signal?: AbortSignal; }; @@ -40,6 +44,7 @@ export async function runWhisperBatch({ reverse = false, shardTotal, shardIndex, + ids, onLog, signal, }: WhisperBatchOptions): Promise<WhisperBatchResult> { @@ -59,11 +64,20 @@ export async function runWhisperBatch({ .filter(Boolean); const allDirs = await readdir(dataDir); + // When the caller supplies an explicit id set (bucket-scoped run), filter + // to that intersection so we never try to transcribe ids whose dirs aren't + // on disk. Otherwise consider every video dir. + const fullItems = ids + ? (() => { + const onDisk = new Set(allDirs); + return ids.filter((id) => onDisk.has(id)); + })() + : allDirs; const { items: shardItems } = await resolveShardItems({ paths, slug: channelSlug, op: "transcribe-missing", - fullItems: allDirs, + fullItems, totalShards: shardTotal, shardIndex: shardIndex, onLog: log, diff --git a/common/lib/availability.ts b/common/lib/availability.ts @@ -80,6 +80,59 @@ export function parseUnavailableFromStderr(stderr: string): Availability { return "error"; } +// Classify a download failure for the managed batch loop's "abort vs skip" +// decision. Per-video permanent failures (private, members-only, etc.) are +// `per_video` — the rest of the batch can continue. Rate-limit and network +// errors are batch-level signals: continuing would just hammer the source or +// waste cycles. Everything else is `unknown` and treated as fatal to be safe. +export type DownloadFailureClass = + | "per_video" + | "rate_limit" + | "network" + | "unknown"; + +const PER_VIDEO_CLASSES: ReadonlyArray<Availability> = [ + "private", + "members_only", + "needs_auth", + "deleted", +]; + +export function classifyDownloadFailure( + stderrTail: string, + availabilityClass: Availability | undefined, +): DownloadFailureClass { + if ( + availabilityClass !== undefined && + PER_VIDEO_CLASSES.includes(availabilityClass) + ) { + return "per_video"; + } + const s = stderrTail.toLowerCase(); + if ( + /http error 429/.test(s) || + /too many requests/.test(s) || + /rate[- ]?limit/.test(s) || + /throttl/.test(s) + ) { + return "rate_limit"; + } + if ( + /econnrefused/.test(s) || + /etimedout/.test(s) || + /enetunreach/.test(s) || + /getaddrinfo/.test(s) || + /network is unreachable/.test(s) || + /temporary failure in name resolution/.test(s) || + /connection reset/.test(s) || + /tls handshake/.test(s) || + /ssl/.test(s) + ) { + return "network"; + } + return "unknown"; +} + // Map the `availability` field that yt-dlp puts in --dump-json output to our // enum. yt-dlp uses: public, unlisted, needs_auth, premium_only, subscriber_only, // public_unlisted (rare), private, etc. diff --git a/common/ytdlp/runYtdlp.ts b/common/ytdlp/runYtdlp.ts @@ -20,6 +20,7 @@ import { getSettings } from "../lib/settings"; import type { Paths } from "../lib/paths"; import { EXCLUDED_FROM_DOWNLOAD, + classifyDownloadFailure, type Availability, } from "../lib/availability"; import { resolveEffectiveAvailability } from "../lib/availability-server"; @@ -454,9 +455,20 @@ async function downloadPlaylistManaged( if (outcome.status === "failed") { failedCount++; if (abortOnError && !firstFailure) { - firstFailure = new Error( - `Managed download failed for ${url} (status=failed)`, + const lastAttempt = + outcome.attempts[outcome.attempts.length - 1]; + const failureClass = classifyDownloadFailure( + lastAttempt?.error ?? "", + lastAttempt?.availabilityClass, ); + // Per-video failures (private, members-only, age-gated, deleted) + // are properties of that video, not signals the batch should + // stop. Only abort on rate-limit / network / unclassified-fatal. + if (failureClass !== "per_video") { + firstFailure = new Error( + `Managed download aborted (${failureClass}) for ${url}`, + ); + } } } processedCount++; diff --git a/editor/app/channels/[slug]/components/VideoListPaneSection.tsx b/editor/app/channels/[slug]/components/VideoListPaneSection.tsx @@ -0,0 +1,76 @@ +"use client"; + +import { useEffect, useState, type ReactNode } from "react"; + +const STORAGE_KEY = "vlp:collapsed"; + +export function VideoListPaneSection({ + left, + right, +}: { + left: ReactNode; + right: ReactNode; +}) { + const [collapsed, setCollapsed] = useState(false); + const [hydrated, setHydrated] = useState(false); + + useEffect(() => { + try { + setCollapsed(window.localStorage.getItem(STORAGE_KEY) === "1"); + } catch { + // localStorage unavailable — keep default. + } + setHydrated(true); + }, []); + + useEffect(() => { + if (!hydrated) return; + try { + window.localStorage.setItem(STORAGE_KEY, collapsed ? "1" : "0"); + } catch { + // ignore + } + }, [collapsed, hydrated]); + + const gridCols = collapsed + ? "lg:grid-cols-[2.25rem_minmax(0,1fr)]" + : "lg:grid-cols-[24rem_minmax(0,1fr)]"; + + return ( + <section + aria-label="videos" + className={`grid ${gridCols} gap-4 lg:gap-6 min-h-[calc(100vh-22rem)]`} + > + <div className="flex flex-col min-h-0 lg:sticky lg:top-4 lg:self-start lg:max-h-[calc(100vh-2rem)]"> + {collapsed ? ( + <button + type="button" + onClick={() => setCollapsed(false)} + aria-expanded={false} + aria-label="expand video list" + className="hidden lg:flex items-center justify-center w-full h-12 rounded border border-zinc-200 dark:border-zinc-800 bg-white dark:bg-zinc-900 text-zinc-500 hover:text-zinc-900 dark:hover:text-zinc-100 hover:bg-zinc-100 dark:hover:bg-zinc-800" + title="Expand video list" + > + <span aria-hidden="true" className="text-sm">›</span> + </button> + ) : ( + <> + <button + type="button" + onClick={() => setCollapsed(true)} + aria-expanded={true} + aria-label="collapse video list" + className="hidden lg:flex self-end items-center gap-1 text-xs text-zinc-500 hover:text-zinc-900 dark:hover:text-zinc-100 mb-1" + title="Collapse video list" + > + <span aria-hidden="true">‹</span> + <span>Collapse</span> + </button> + {left} + </> + )} + </div> + <div className="min-w-0">{right}</div> + </section> + ); +} diff --git a/editor/app/channels/[slug]/components/stages/TranscribeStage.tsx b/editor/app/channels/[slug]/components/stages/TranscribeStage.tsx @@ -16,9 +16,9 @@ import { import { cancelJobAction } from "../../../../jobs/actions"; import { clearFailedTranscriptionsAction, + transcribeBucketAction, transcribeMissingAction, } from "../../whisperActions"; -import { RetryBucketControl } from "../RetryBucketControl"; import { VideoIdList } from "../VideoIdList"; type AudioFormatChoice = AudioFormat | "any"; @@ -38,11 +38,6 @@ type Props = { downloadedNoTranscriptIds: string[]; defaultConcurrency: number; defaultQueueKey: string; - // Default queue used when the user clicks Retry on the - // downloadedNoTranscript bucket. The retry triggers a download flow, so - // it should run on the platform's download queue (not the transcription - // queue) to avoid hammering the source. - downloadDefaultQueueKey: string; missingShard: ShardConfigSummary | null; }; @@ -53,10 +48,129 @@ export function TranscribeStage({ downloadedNoTranscriptIds, defaultConcurrency, defaultQueueKey, - downloadDefaultQueueKey, missingShard, }: Props) { const channelQueueKey = `channel:${slug}`; + const bucketCount = downloadedNoTranscriptIds.length; + + return ( + <div className="flex flex-col gap-6"> + {bucketCount > 0 && ( + <BucketTranscribeSection + slug={slug} + ids={downloadedNoTranscriptIds} + existingQueues={existingQueues} + defaultConcurrency={defaultConcurrency} + defaultQueueKey={defaultQueueKey} + /> + )} + <TranscribeMissingSection + slug={slug} + existingQueues={existingQueues} + defaultConcurrency={defaultConcurrency} + defaultQueueKey={defaultQueueKey} + missingShard={missingShard} + demoted={bucketCount > 0} + /> + <FailedTranscriptionsSection + slug={slug} + ids={failedVideoIds} + existingQueues={existingQueues} + defaultQueueKey={channelQueueKey} + /> + </div> + ); +} + +function BucketTranscribeSection({ + slug, + ids, + existingQueues, + defaultConcurrency, + defaultQueueKey, +}: { + slug: string; + ids: string[]; + existingQueues: string[]; + defaultConcurrency: number; + defaultQueueKey: string; +}) { + const [queue, setQueue] = useState(defaultQueueKey); + const [concurrency, setConcurrency] = useState(""); + const [audioFormat, setAudioFormat] = useState<AudioFormatChoice>("any"); + const [strictFormat, setStrictFormat] = useState(false); + return ( + <div className="flex flex-col gap-2"> + <Heading + title={`Transcribe downloaded audio (${ids.length})`} + desc="Audio is on disk but no transcript yet. Runs whisper directly on these ids — no re-download." + /> + <VideoIdList + slug={slug} + ids={ids} + ariaLabel="downloaded without transcript list" + emptyAriaLabel="downloaded without transcript empty" + emptyMessage="None" + itemAriaLabel={(id) => `downloaded without transcript ${id}`} + /> + <StreamActionLog + trigger={() => + transcribeBucketAction( + slug, + ids, + queue, + parseConcurrency(concurrency), + audioFormat === "any" ? undefined : audioFormat, + audioFormat !== "any" && strictFormat, + ) + } + cancelAction={cancelJobAction} + buttonLabel={`Transcribe ${ids.length} downloaded`} + runningLabel="Transcribing…" + extraControls={ + <> + <QueueControl + value={queue} + onChange={setQueue} + defaultQueueKey={defaultQueueKey} + existingQueues={existingQueues} + actionLabel="Transcribe downloaded" + /> + <ConcurrencyControl + value={concurrency} + onChange={setConcurrency} + defaultLimit={defaultConcurrency} + actionLabel="Transcribe downloaded" + /> + <AudioFormatControl + actionLabel="Transcribe downloaded" + value={audioFormat} + onChange={setAudioFormat} + strict={strictFormat} + onStrictChange={setStrictFormat} + /> + </> + } + /> + </div> + ); +} + +function TranscribeMissingSection({ + slug, + existingQueues, + defaultConcurrency, + defaultQueueKey, + missingShard, + demoted, +}: { + slug: string; + existingQueues: string[]; + defaultConcurrency: number; + defaultQueueKey: string; + missingShard: ShardConfigSummary | null; + demoted: boolean; +}) { const [missingQueue, setMissingQueue] = useState(defaultQueueKey); const [missingConcurrency, setMissingConcurrency] = useState(""); const [missingReverse, setMissingReverse] = useState(false); @@ -69,86 +183,75 @@ export function TranscribeStage({ const [missingShardIndex, setMissingShardIndex] = useState( missingShard ? String(missingShard.shardIndex) : "", ); - + const wrapperClass = demoted + ? "flex flex-col gap-2 rounded border border-zinc-200 dark:border-zinc-800 p-3" + : "flex flex-col gap-2"; return ( - <div className="flex flex-col gap-6"> - <div className="flex flex-col gap-2"> - <Heading - title="Transcribe missing" - desc="Run whisper-cli over every video that has audio but no transcript.json. Failures append to channels/<slug>/failed-transcriptions and are skipped on the next run until you clear the list." - /> - <StreamActionLog - trigger={() => - transcribeMissingAction( - slug, - missingQueue, - parseConcurrency(missingConcurrency), - missingReverse, - missingAudioFormat === "any" ? undefined : missingAudioFormat, - missingAudioFormat !== "any" && missingStrictFormat, - parseShardField(missingShardTotal), - parseShardField(missingShardIndex), - ) - } - cancelAction={cancelJobAction} - buttonLabel="Transcribe missing" - runningLabel="Transcribing…" - extraControls={ - <> - <QueueControl - value={missingQueue} - onChange={setMissingQueue} - defaultQueueKey={defaultQueueKey} - existingQueues={existingQueues} - actionLabel="Transcribe missing" - /> - <ConcurrencyControl - value={missingConcurrency} - onChange={setMissingConcurrency} - defaultLimit={defaultConcurrency} - actionLabel="Transcribe missing" - /> - <AudioFormatControl - actionLabel="Transcribe missing" - value={missingAudioFormat} - onChange={setMissingAudioFormat} - strict={missingStrictFormat} - onStrictChange={setMissingStrictFormat} - /> - <label className="flex items-center gap-1 text-xs text-zinc-500"> - <input - type="checkbox" - checked={missingReverse} - onChange={(e) => setMissingReverse(e.target.checked)} - aria-label="reverse order for Transcribe missing" - /> - Reverse - </label> - <ShardControl - slug={slug} - op="transcribe-missing" - actionLabel="Transcribe missing" - existing={missingShard} - total={missingShardTotal} - onTotalChange={setMissingShardTotal} - index={missingShardIndex} - onIndexChange={setMissingShardIndex} + <div className={wrapperClass}> + <Heading + title="Transcribe missing" + desc="Run whisper-cli over every video that has audio but no transcript.json. Failures append to channels/<slug>/failed-transcriptions and are skipped on the next run until you clear the list." + small={demoted} + /> + <StreamActionLog + trigger={() => + transcribeMissingAction( + slug, + missingQueue, + parseConcurrency(missingConcurrency), + missingReverse, + missingAudioFormat === "any" ? undefined : missingAudioFormat, + missingAudioFormat !== "any" && missingStrictFormat, + parseShardField(missingShardTotal), + parseShardField(missingShardIndex), + ) + } + cancelAction={cancelJobAction} + buttonLabel="Transcribe missing" + runningLabel="Transcribing…" + extraControls={ + <> + <QueueControl + value={missingQueue} + onChange={setMissingQueue} + defaultQueueKey={defaultQueueKey} + existingQueues={existingQueues} + actionLabel="Transcribe missing" + /> + <ConcurrencyControl + value={missingConcurrency} + onChange={setMissingConcurrency} + defaultLimit={defaultConcurrency} + actionLabel="Transcribe missing" + /> + <AudioFormatControl + actionLabel="Transcribe missing" + value={missingAudioFormat} + onChange={setMissingAudioFormat} + strict={missingStrictFormat} + onStrictChange={setMissingStrictFormat} + /> + <label className="flex items-center gap-1 text-xs text-zinc-500"> + <input + type="checkbox" + checked={missingReverse} + onChange={(e) => setMissingReverse(e.target.checked)} + aria-label="reverse order for Transcribe missing" /> - </> - } - /> - <DownloadedNoTranscriptList - slug={slug} - ids={downloadedNoTranscriptIds} - downloadDefaultQueueKey={downloadDefaultQueueKey} - existingQueues={existingQueues} - /> - </div> - <FailedTranscriptionsSection - slug={slug} - ids={failedVideoIds} - existingQueues={existingQueues} - defaultQueueKey={channelQueueKey} + Reverse + </label> + <ShardControl + slug={slug} + op="transcribe-missing" + actionLabel="Transcribe missing" + existing={missingShard} + total={missingShardTotal} + onTotalChange={setMissingShardTotal} + index={missingShardIndex} + onIndexChange={setMissingShardIndex} + /> + </> + } /> </div> ); @@ -230,49 +333,6 @@ function FailedTranscriptionsSection({ ); } -function DownloadedNoTranscriptList({ - slug, - ids, - downloadDefaultQueueKey, - existingQueues, -}: { - slug: string; - ids: string[]; - downloadDefaultQueueKey: string; - existingQueues: string[]; -}) { - if (ids.length === 0) return null; - return ( - <div className="flex flex-col gap-2 rounded border border-zinc-200 dark:border-zinc-800 p-3"> - <div> - <h4 className="text-sm font-semibold"> - Downloaded but not transcribed ({ids.length}) - </h4> - <p className="text-xs text-zinc-500"> - Audio is on disk but no .vtt or transcript.json yet. The retry runs - a download pass — pick &ldquo;youtube&rdquo; handling to fetch a - missing VTT, or leave default to verify nothing else is needed. - </p> - </div> - <VideoIdList - slug={slug} - ids={ids} - ariaLabel="downloaded without transcript list" - emptyAriaLabel="downloaded without transcript empty" - emptyMessage="None" - itemAriaLabel={(id) => `downloaded without transcript ${id}`} - /> - <RetryBucketControl - slug={slug} - ids={ids} - actionLabel="downloaded but not transcribed" - defaultQueueKey={downloadDefaultQueueKey} - existingQueues={existingQueues} - /> - </div> - ); -} - function AudioFormatControl({ actionLabel, value, @@ -325,11 +385,25 @@ function AudioFormatControl({ ); } -function Heading({ title, desc }: { title: string; desc: string }) { +function Heading({ + title, + desc, + small = false, +}: { + title: string; + desc: string; + small?: boolean; +}) { return ( <div> - <h3 className="text-base font-semibold">{title}</h3> - <p className="text-sm text-zinc-500">{desc}</p> + <h3 + className={small ? "text-sm font-semibold" : "text-base font-semibold"} + > + {title} + </h3> + <p className={small ? "text-xs text-zinc-500" : "text-sm text-zinc-500"}> + {desc} + </p> </div> ); } diff --git a/editor/app/channels/[slug]/page.tsx b/editor/app/channels/[slug]/page.tsx @@ -39,6 +39,7 @@ import { PlaylistStage } from "./components/stages/PlaylistStage"; import { TranscodeStage } from "./components/stages/TranscodeStage"; import { TranscribeStage } from "./components/stages/TranscribeStage"; import { VideoListPane } from "./components/VideoListPane"; +import { VideoListPaneSection } from "./components/VideoListPaneSection"; import { VideoPanel, type VideoFile } from "./videos/[id]/components/VideoPanel"; import { parseFilter, filterRows } from "./lib/videoRows"; import { @@ -245,7 +246,6 @@ export default async function ChannelDetailPage({ downloadedNoTranscriptIds={buckets.downloadedNoTranscript} defaultConcurrency={paths.parallelTranscribeLimit} defaultQueueKey={TRANSCRIPTION_QUEUE} - downloadDefaultQueueKey={platformDefaultQueueKey} missingShard={transcribeMissingShard} /> ), @@ -382,11 +382,8 @@ export default async function ChannelDetailPage({ hideChannelSlug /> - <section - aria-label="videos" - className="grid lg:grid-cols-[24rem_minmax(0,1fr)] gap-4 lg:gap-6 min-h-[calc(100vh-22rem)]" - > - <div className="flex flex-col min-h-0 lg:sticky lg:top-4 lg:self-start lg:max-h-[calc(100vh-2rem)]"> + <VideoListPaneSection + left={ <VideoListPane slug={slug} rows={rows} @@ -394,21 +391,26 @@ export default async function ChannelDetailPage({ initialFilter={filter} initialQuery={queryRaw} /> - </div> - <div className="min-w-0"> - {selectedPanel ?? ( - <EmptyVideoPanel - total={rows.length} - filtered={orderedRows.length} - /> - )} - {selectedVideoId && selectedTitle && ( - <p className="text-sm text-zinc-500 mt-2 truncate" aria-label="video title"> - {selectedTitle} - </p> - )} - </div> - </section> + } + right={ + <> + {selectedPanel ?? ( + <EmptyVideoPanel + total={rows.length} + filtered={orderedRows.length} + /> + )} + {selectedVideoId && selectedTitle && ( + <p + className="text-sm text-zinc-500 mt-2 truncate" + aria-label="video title" + > + {selectedTitle} + </p> + )} + </> + } + /> <details open diff --git a/editor/app/channels/[slug]/whisperActions.ts b/editor/app/channels/[slug]/whisperActions.ts @@ -78,6 +78,45 @@ export async function transcribeMissingAction( }); } +export async function transcribeBucketAction( + slug: string, + ids: string[], + queueKey?: string, + concurrency?: number, + audioFormat?: AudioFormat, + strictAudioFormat?: boolean, +): Promise<StreamActionResult> { + const paths = getPaths(); + const limit = sanitizeConcurrency(concurrency); + const fmt = sanitizeAudioFormat(audioFormat); + const cleaned = Array.from(new Set(ids.map((id) => id.trim()).filter(Boolean))); + if (cleaned.length === 0) { + return { ok: false, error: "No video ids supplied" }; + } + return runManagedFunction({ + kind: "whisper-bucket-downloaded-no-transcript", + queueKey: queueKey === undefined ? TRANSCRIPTION_QUEUE : queueKey.trim(), + paths, + channelSlug: slug, + fn: async (onLog, signal) => { + const result = await runWhisperBatch({ + channelSlug: slug, + paths, + concurrency: limit, + audioFormat: fmt, + strictAudioFormat: fmt !== undefined && strictAudioFormat === true, + ids: cleaned, + onLog, + signal, + }); + onLog( + `Whisper bucket: ${result.succeeded} succeeded, ${result.failed} failed, ${result.skipped} skipped, ${result.attempted} attempted.`, + ); + revalidatePath(`/channels/${slug}`); + }, + }); +} + export async function clearFailedTranscriptionsAction( slug: string, queueKey?: string, diff --git a/editor/app/layout.tsx b/editor/app/layout.tsx @@ -46,7 +46,7 @@ export default async function RootLayout({ return ( <html lang="en" className="h-full antialiased"> <body className="min-h-full flex flex-col md:flex-row bg-zinc-50 text-zinc-900 dark:bg-zinc-950 dark:text-zinc-100"> - <aside className="md:w-56 md:shrink-0 border-b md:border-b-0 md:border-r border-zinc-200 dark:border-zinc-800 bg-white dark:bg-zinc-900 flex flex-col"> + <aside className="md:w-56 md:shrink-0 md:sticky md:top-0 md:self-start md:h-screen md:overflow-y-auto border-b md:border-b-0 md:border-r border-zinc-200 dark:border-zinc-800 bg-white dark:bg-zinc-900 flex flex-col"> <div className="px-4 py-3 md:py-4 md:border-b md:border-zinc-200 dark:md:border-zinc-800"> <Link href="/"