commit 1b24f2d620606b3079c405302d3a583cba1d99a3 parent 711021a8a43aac758234ef52c878889979ae8c89 Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st> Date: Wed, 6 May 2026 16:25:32 -0400 Make page loading more efficient Diffstat:
38 files changed, 1519 insertions(+), 1254 deletions(-)
diff --git a/common/controller/channelHealth.ts b/common/controller/channelHealth.ts @@ -1,92 +0,0 @@ -import path from "node:path"; -import { readdir } from "node:fs/promises"; -import type { Paths } from "../lib/paths"; -import { readArchive } from "../lib/archive"; -import { - isVideoDownloaded, - isVideoTranscribed, - readVideoFiles, -} from "../lib/videoStatus"; -import { loadFailedTranscriptions } from "./failedTranscriptions"; -import { findDuplicateDirs } from "./verifyTranscripts"; - -export type ChannelHealth = { - totals: { - videos: number; - transcribed: number; - downloaded: number; - }; - noTranscript: string[]; - downloadedNoTranscript: string[]; - untranscribable: string[]; - noMetadata: string[]; - failedListed: string[]; - missingFromArchive: string[]; - duplicateDirs: string[]; -}; - -export async function inspectChannelHealth( - paths: Paths, - slug: string, -): Promise<ChannelHealth> { - const channelDir = path.join(paths.channelsDir, slug); - const dataDir = path.join(channelDir, "data"); - const archivePath = path.join(channelDir, "archive"); - - const dirs = await readdir(dataDir, { withFileTypes: true }).catch(() => []); - const videoDirNames = dirs.filter((d) => d.isDirectory()).map((d) => d.name); - - const perVideo = await Promise.all( - videoDirNames.map(async (name) => ({ - id: name, - files: await readVideoFiles(path.join(dataDir, name)), - })), - ); - - const noTranscript: string[] = []; - const downloadedNoTranscript: string[] = []; - const untranscribable: string[] = []; - const noMetadata: string[] = []; - let transcribed = 0; - let downloaded = 0; - for (const v of perVideo) { - if (isVideoTranscribed(v.files)) transcribed++; - if (isVideoDownloaded(v.files)) downloaded++; - if (!v.files.hasMeta) noMetadata.push(v.id); - if (v.files.isUntranscribable) { - untranscribable.push(v.id); - continue; - } - if (!isVideoTranscribed(v.files)) { - if (v.files.audioFiles.length > 0) { - downloadedNoTranscript.push(v.id); - } else { - noTranscript.push(v.id); - } - } - } - - const failedListed = await loadFailedTranscriptions(paths, slug); - - const archive = await readArchive(archivePath); - const { dirMap, duplicates } = await findDuplicateDirs(dataDir); - const missingFromArchive: string[] = []; - for (const id of archive.ids) { - if (!dirMap.has(id)) missingFromArchive.push(id); - } - - return { - totals: { - videos: videoDirNames.length, - transcribed, - downloaded, - }, - noTranscript: noTranscript.sort(), - downloadedNoTranscript: downloadedNoTranscript.sort(), - untranscribable: untranscribable.sort(), - noMetadata: noMetadata.sort(), - failedListed, - missingFromArchive: missingFromArchive.sort(), - duplicateDirs: Array.from(duplicates).sort(), - }; -} diff --git a/common/controller/channelSnapshot.ts b/common/controller/channelSnapshot.ts @@ -0,0 +1,188 @@ +import path from "node:path"; +import type { Dirent } from "node:fs"; +import { readdir, readFile, rename, writeFile } from "node:fs/promises"; +import { readArchive } from "../lib/archive"; +import { + isVideoDownloaded, + isVideoTranscribed, + readVideoFiles, + type VideoFiles, +} from "../lib/videoStatus"; +import type { Paths } from "../lib/paths"; +import { extractVideoId, isRumbleUrl } from "../ytdlp/runYtdlp"; +import { loadFailedTranscriptions } from "./failedTranscriptions"; + +export type ChannelSnapshot = { + generatedAt: string; + totals: { + videos: number; + transcribed: number; + downloaded: number; + }; + buckets: { + noTranscript: string[]; + downloadedNoTranscript: string[]; + untranscribable: string[]; + noMetadata: string[]; + failedListed: string[]; + missingFromArchive: string[]; + duplicateDirs: string[]; + }; + undownloadedIds: string[]; +}; + +export const SNAPSHOT_FILENAME = "snapshot.json"; + +export function snapshotPath(paths: Paths, slug: string): string { + return path.join(paths.channelsDir, slug, SNAPSHOT_FILENAME); +} + +export async function readChannelSnapshot( + paths: Paths, + slug: string, +): Promise<ChannelSnapshot | null> { + try { + const raw = await readFile(snapshotPath(paths, slug), "utf8"); + return JSON.parse(raw) as ChannelSnapshot; + } catch { + return null; + } +} + +async function readPlaylistUrls(file: string): Promise<string[]> { + try { + const raw = await readFile(file, "utf8"); + return raw.split("\n").map((s) => s.trim()).filter(Boolean); + } catch { + return []; + } +} + +function videoHasAnyArtifact(files: VideoFiles): boolean { + return files.hasYtVtt || files.hasWhisper || files.audioFiles.length > 0; +} + +export async function generateChannelSnapshot( + paths: Paths, + slug: string, +): Promise<ChannelSnapshot> { + const channelDir = path.join(paths.channelsDir, slug); + const dataDir = path.join(channelDir, "data"); + const archivePath = path.join(channelDir, "archive"); + const playlistPath = path.join(channelDir, "playlist"); + + const [dirEntries, urls, archive, failedListed] = await Promise.all([ + readdir(dataDir, { withFileTypes: true }).catch(() => [] as Dirent[]), + readPlaylistUrls(playlistPath), + readArchive(archivePath), + loadFailedTranscriptions(paths, slug), + ]); + const videoDirNames = dirEntries + .filter((d) => d.isDirectory()) + .map((d) => d.name); + const wantRumbleIndex = urls.some(isRumbleUrl); + + const perVideo = await Promise.all( + videoDirNames.map(async (id) => { + const dir = path.join(dataDir, id); + const files = await readVideoFiles(dir, { checkUntranscribable: true }); + let webpageUrl: string | null = null; + if (wantRumbleIndex && files.hasMeta) { + try { + const raw = await readFile( + path.join(dir, "metadata.info.json"), + "utf8", + ); + const parsed = JSON.parse(raw); + if (typeof parsed?.webpage_url === "string") { + webpageUrl = parsed.webpage_url; + } + } catch { + // ignore + } + } + return { id, files, webpageUrl }; + }), + ); + + const filesById = new Map<string, VideoFiles>(); + const rumbleIndex = new Map<string, string>(); + for (const v of perVideo) { + filesById.set(v.id, v.files); + if (v.webpageUrl) { + const slugId = extractVideoId(v.webpageUrl); + if (slugId) rumbleIndex.set(slugId, v.id); + } + } + + const noTranscript: string[] = []; + const downloadedNoTranscript: string[] = []; + const untranscribable: string[] = []; + const noMetadata: string[] = []; + let transcribed = 0; + let downloaded = 0; + for (const { id, files } of perVideo) { + if (isVideoTranscribed(files)) transcribed++; + if (isVideoDownloaded(files)) downloaded++; + if (!files.hasMeta) noMetadata.push(id); + if (files.isUntranscribable) { + untranscribable.push(id); + continue; + } + if (!isVideoTranscribed(files)) { + if (files.audioFiles.length > 0) downloadedNoTranscript.push(id); + else noTranscript.push(id); + } + } + + const missingFromArchive: string[] = []; + for (const id of archive.ids) { + if (!filesById.has(id)) missingFromArchive.push(id); + } + + const undownloadedIds: string[] = []; + for (const url of urls) { + const slugId = extractVideoId(url); + if (!slugId) continue; + const dirId = + isRumbleUrl(url) && rumbleIndex.has(slugId) + ? (rumbleIndex.get(slugId) as string) + : slugId; + const f = filesById.get(dirId); + if (f && videoHasAnyArtifact(f)) continue; + undownloadedIds.push(dirId); + } + + const snapshot: ChannelSnapshot = { + generatedAt: new Date().toISOString(), + totals: { + videos: videoDirNames.length, + transcribed, + downloaded, + }, + buckets: { + noTranscript: noTranscript.sort(), + downloadedNoTranscript: downloadedNoTranscript.sort(), + untranscribable: untranscribable.sort(), + noMetadata: noMetadata.sort(), + failedListed, + missingFromArchive: missingFromArchive.sort(), + duplicateDirs: [], + }, + undownloadedIds, + }; + + await writeChannelSnapshot(paths, slug, snapshot); + return snapshot; +} + +async function writeChannelSnapshot( + paths: Paths, + slug: string, + snapshot: ChannelSnapshot, +): Promise<void> { + const file = snapshotPath(paths, slug); + const tmp = `${file}.tmp-${process.pid}`; + await writeFile(tmp, JSON.stringify(snapshot, null, 2) + "\n"); + await rename(tmp, file); +} diff --git a/common/controller/undownloadedVideos.ts b/common/controller/undownloadedVideos.ts @@ -1,6 +1,5 @@ import path from "node:path"; -import { readdir, readFile } from "node:fs/promises"; -import type { ChannelConfig } from "../lib/channelConfig"; +import { readFile } from "node:fs/promises"; import type { Paths } from "../lib/paths"; import { buildRumbleSlugIndex, @@ -8,11 +7,6 @@ import { isRumbleUrl, } from "../ytdlp/runYtdlp"; -export type UndownloadedVideo = { - videoId: string; - url: string; -}; - async function readPlaylistUrls(playlistPath: string): Promise<string[]> { let raw: string; try { @@ -26,52 +20,6 @@ async function readPlaylistUrls(playlistPath: string): Promise<string[]> { .filter(Boolean); } -// Looser than runYtdlp's destinationExists: counts a video as downloaded if it -// has *any* of the artifacts we know about (subs, audio, whisper output). The -// download pipeline uses the strict per-handling check; this list is -// user-facing, so a YouTube video that was whisper-transcribed (no auto-subs) -// should drop off it. -async function videoHasAnyArtifact( - dataDir: string, - id: string, -): Promise<boolean> { - const entries = await readdir(path.join(dataDir, id)).catch( - () => [] as string[], - ); - return entries.some( - (e) => - e === "transcript.en.vtt" || - e === "transcript.json" || - e.startsWith("audio."), - ); -} - -export async function listUndownloadedVideos( - paths: Paths, - slug: string, - _config: ChannelConfig, -): Promise<UndownloadedVideo[]> { - const channelRoot = path.join(paths.channelsDir, slug); - const dataDir = path.join(channelRoot, "data"); - const urls = await readPlaylistUrls(path.join(channelRoot, "playlist")); - if (urls.length === 0) return []; - - const rumbleIndex = urls.some(isRumbleUrl) - ? await buildRumbleSlugIndex(dataDir) - : null; - - const out: UndownloadedVideo[] = []; - for (const url of urls) { - const slugId = extractVideoId(url); - if (!slugId) continue; - const dirId = - rumbleIndex && isRumbleUrl(url) ? rumbleIndex.get(slugId) ?? slugId : slugId; - if (await videoHasAnyArtifact(dataDir, dirId)) continue; - out.push({ videoId: dirId, url }); - } - return out; -} - export async function findVideoSourceUrl( paths: Paths, slug: string, diff --git a/common/controller/whisperBatch.ts b/common/controller/whisperBatch.ts @@ -15,6 +15,7 @@ export type WhisperBatchOptions = { mode: WhisperBatchMode; concurrency?: number; audioFilename?: string; + reverse?: boolean; onLog?: (msg: string) => void; signal?: AbortSignal; }; @@ -32,6 +33,7 @@ export async function runWhisperBatch({ mode, concurrency, audioFilename = "audio.mp3", + reverse = false, onLog, signal, }: WhisperBatchOptions): Promise<WhisperBatchResult> { @@ -53,6 +55,7 @@ export async function runWhisperBatch({ } else { videoDirs = await readdir(dataDir); } + if (reverse) videoDirs.reverse(); const failedSet = new Set<string>(originalFailures); const succeeded = new Set<string>(); diff --git a/common/lib/videoStatus.ts b/common/lib/videoStatus.ts @@ -17,7 +17,10 @@ export const VTT_FILENAME = "transcript.en.vtt"; export const WHISPER_FILENAME = "transcript.json"; export const META_FILENAME = "metadata.info.json"; -export async function readVideoFiles(videoDir: string): Promise<VideoFiles> { +export async function readVideoFiles( + videoDir: string, + opts: { checkUntranscribable?: boolean } = {}, +): Promise<VideoFiles> { const entries = await readdir(videoDir).catch(() => [] as string[]); const hasMeta = entries.includes(META_FILENAME); const hasYtVtt = entries.includes(VTT_FILENAME); @@ -26,7 +29,7 @@ export async function readVideoFiles(videoDir: string): Promise<VideoFiles> { (e) => e.startsWith("audio.") && !e.includes(".tmp-"), ); let isUntranscribable = false; - if (hasWhisper) { + if (hasWhisper && opts.checkUntranscribable) { try { const raw = await readFile(path.join(videoDir, WHISPER_FILENAME), "utf8"); const parsed = JSON.parse(raw) as { transcription?: unknown[] }; diff --git a/editor/app/build/_components/BuildButtons.tsx b/editor/app/build/_components/BuildButtons.tsx @@ -1,74 +0,0 @@ -"use client"; - -import { useState } from "react"; -import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog"; -import { QueueControl } from "../../_components/QueueControl"; -import { cancelJobAction } from "../../jobs/actions"; -import { buildExportAction, buildIndexAction } from "../buildAction"; - -type Props = { - existingQueues: string[]; -}; - -const DEFAULT_BUILD_QUEUE = "build"; - -export function BuildButtons({ existingQueues }: Props) { - const [indexQueue, setIndexQueue] = useState(DEFAULT_BUILD_QUEUE); - const [exportQueue, setExportQueue] = useState(DEFAULT_BUILD_QUEUE); - - return ( - <div className="flex flex-col gap-8"> - <section className="flex flex-col gap-3"> - <div> - <h2 className="text-lg font-semibold">Build index</h2> - <p className="text-sm text-zinc-500"> - Re-scans <code>transcripts/channels/</code> and rewrites paginated - JSON in <code>export/public/</code>. Cheap when nothing changed - (mtime short-circuit). - </p> - </div> - <StreamActionLog - trigger={() => buildIndexAction(indexQueue)} - cancelAction={cancelJobAction} - buttonLabel="Build index" - runningLabel="Building index…" - extraControls={ - <QueueControl - value={indexQueue} - onChange={setIndexQueue} - defaultQueueKey={DEFAULT_BUILD_QUEUE} - existingQueues={existingQueues} - actionLabel="Build index" - /> - } - /> - </section> - - <section className="flex flex-col gap-3 border-t border-zinc-200 dark:border-zinc-800 pt-6"> - <div> - <h2 className="text-lg font-semibold">Build static export</h2> - <p className="text-sm text-zinc-500"> - Spawns <code>pnpm run build</code> in <code>export/</code> — runs - the index build, then <code>next build</code> to produce the - static site at <code>export/out/</code>. - </p> - </div> - <StreamActionLog - trigger={() => buildExportAction(exportQueue)} - cancelAction={cancelJobAction} - buttonLabel="Build static export" - runningLabel="Building static export…" - extraControls={ - <QueueControl - value={exportQueue} - onChange={setExportQueue} - defaultQueueKey={DEFAULT_BUILD_QUEUE} - existingQueues={existingQueues} - actionLabel="Build static export" - /> - } - /> - </section> - </div> - ); -} diff --git a/editor/app/build/components/BuildButtons.tsx b/editor/app/build/components/BuildButtons.tsx @@ -0,0 +1,74 @@ +"use client"; + +import { useState } from "react"; +import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog"; +import { QueueControl } from "../../components/QueueControl"; +import { cancelJobAction } from "../../jobs/actions"; +import { buildExportAction, buildIndexAction } from "../buildAction"; + +type Props = { + existingQueues: string[]; +}; + +const DEFAULT_BUILD_QUEUE = "build"; + +export function BuildButtons({ existingQueues }: Props) { + const [indexQueue, setIndexQueue] = useState(DEFAULT_BUILD_QUEUE); + const [exportQueue, setExportQueue] = useState(DEFAULT_BUILD_QUEUE); + + return ( + <div className="flex flex-col gap-8"> + <section className="flex flex-col gap-3"> + <div> + <h2 className="text-lg font-semibold">Build index</h2> + <p className="text-sm text-zinc-500"> + Re-scans <code>transcripts/channels/</code> and rewrites paginated + JSON in <code>export/public/</code>. Cheap when nothing changed + (mtime short-circuit). + </p> + </div> + <StreamActionLog + trigger={() => buildIndexAction(indexQueue)} + cancelAction={cancelJobAction} + buttonLabel="Build index" + runningLabel="Building index…" + extraControls={ + <QueueControl + value={indexQueue} + onChange={setIndexQueue} + defaultQueueKey={DEFAULT_BUILD_QUEUE} + existingQueues={existingQueues} + actionLabel="Build index" + /> + } + /> + </section> + + <section className="flex flex-col gap-3 border-t border-zinc-200 dark:border-zinc-800 pt-6"> + <div> + <h2 className="text-lg font-semibold">Build static export</h2> + <p className="text-sm text-zinc-500"> + Spawns <code>pnpm run build</code> in <code>export/</code> — runs + the index build, then <code>next build</code> to produce the + static site at <code>export/out/</code>. + </p> + </div> + <StreamActionLog + trigger={() => buildExportAction(exportQueue)} + cancelAction={cancelJobAction} + buttonLabel="Build static export" + runningLabel="Building static export…" + extraControls={ + <QueueControl + value={exportQueue} + onChange={setExportQueue} + defaultQueueKey={DEFAULT_BUILD_QUEUE} + existingQueues={existingQueues} + actionLabel="Build static export" + /> + } + /> + </section> + </div> + ); +} diff --git a/editor/app/build/page.tsx b/editor/app/build/page.tsx @@ -1,6 +1,6 @@ import type { Metadata } from "next"; import { getRegistry } from "yt-dlp-transcript-common/jobs/registry"; -import { BuildButtons } from "./_components/BuildButtons"; +import { BuildButtons } from "./components/BuildButtons"; export const dynamic = "force-dynamic"; diff --git a/editor/app/channels/[slug]/_components/PipelinePanel.tsx b/editor/app/channels/[slug]/_components/PipelinePanel.tsx @@ -1,184 +0,0 @@ -"use client"; - -import { useState } from "react"; -import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog"; -import { QueueControl } from "../../../_components/QueueControl"; -import { cancelJobAction } from "../../../jobs/actions"; -import { - downloadAction, - downloadMissingAction, - storePlaylistAction, - syncAction, -} from "../pipelineActions"; - -type Props = { - slug: string; - hasUrl: boolean; - defaultQueueKey: string; - existingQueues: string[]; -}; - -export function PipelinePanel({ - slug, - hasUrl, - defaultQueueKey, - existingQueues, -}: Props) { - const [storeQueue, setStoreQueue] = useState(defaultQueueKey); - const [downloadQueue, setDownloadQueue] = useState(defaultQueueKey); - const [missingQueue, setMissingQueue] = useState(defaultQueueKey); - const [syncQueue, setSyncQueue] = useState(defaultQueueKey); - const [ignoreArchive, setIgnoreArchive] = useState(false); - const [downloadAbortOnError, setDownloadAbortOnError] = useState(true); - const [missingAbortOnError, setMissingAbortOnError] = useState(true); - - if (!hasUrl) { - return ( - <p className="text-sm text-zinc-500"> - Set a <code>url</code> in the configuration above to enable the - pipeline. - </p> - ); - } - return ( - <div className="flex flex-col gap-6"> - <div className="flex flex-col gap-2"> - <Heading - title="Store playlist" - desc="Fetch the channel's full URL list (--flat-playlist --skip-download --print url) and save it for later runs." - /> - <StreamActionLog - trigger={() => storePlaylistAction(slug, storeQueue)} - cancelAction={cancelJobAction} - buttonLabel="Store playlist" - runningLabel="Storing playlist…" - extraControls={ - <QueueControl - value={storeQueue} - onChange={setStoreQueue} - defaultQueueKey={defaultQueueKey} - existingQueues={existingQueues} - actionLabel="Store playlist" - /> - } - /> - </div> - <div className="flex flex-col gap-2"> - <Heading - title="Download from playlist" - desc="Read the saved playlist, prefilter against the archive, then run yt-dlp on whatever's left. Use for first-time imports and resumes." - /> - <label className="flex items-center gap-2 text-sm"> - <input - type="checkbox" - checked={downloadAbortOnError} - onChange={(e) => setDownloadAbortOnError(e.target.checked)} - aria-label="abort on error for Download from playlist" - /> - Abort on error - <span className="text-xs text-zinc-500"> - (stop the run on the first per-video failure) - </span> - </label> - <StreamActionLog - trigger={() => - downloadAction(slug, downloadQueue, downloadAbortOnError) - } - cancelAction={cancelJobAction} - buttonLabel="Download from playlist" - runningLabel="Downloading…" - extraControls={ - <QueueControl - value={downloadQueue} - onChange={setDownloadQueue} - defaultQueueKey={defaultQueueKey} - existingQueues={existingQueues} - actionLabel="Download from playlist" - /> - } - /> - </div> - <div className="flex flex-col gap-2"> - <Heading - title="Download videos" - desc="Read the saved playlist, then re-download any video whose destination file (transcript.en.vtt for YouTube, or audio.* / transcript.json for transcribe) is missing on disk. Useful when the archive is out of sync with the data directory." - /> - <label className="flex items-center gap-2 text-sm"> - <input - type="checkbox" - checked={ignoreArchive} - onChange={(e) => setIgnoreArchive(e.target.checked)} - /> - Ignore archive - <span className="text-xs text-zinc-500"> - (re-fetch even if listed in the archive file) - </span> - </label> - <label className="flex items-center gap-2 text-sm"> - <input - type="checkbox" - checked={missingAbortOnError} - onChange={(e) => setMissingAbortOnError(e.target.checked)} - aria-label="abort on error for Download videos" - /> - Abort on error - <span className="text-xs text-zinc-500"> - (stop the run on the first per-video failure) - </span> - </label> - <StreamActionLog - trigger={() => - downloadMissingAction( - slug, - missingQueue, - ignoreArchive, - missingAbortOnError, - ) - } - cancelAction={cancelJobAction} - buttonLabel="Download videos" - runningLabel="Downloading missing…" - extraControls={ - <QueueControl - value={missingQueue} - onChange={setMissingQueue} - defaultQueueKey={defaultQueueKey} - existingQueues={existingQueues} - actionLabel="Download videos" - /> - } - /> - </div> - <div className="flex flex-col gap-2"> - <Heading - title="Sync" - desc="Quick incremental fetch using --lazy-playlist + --break-on-existing. Run regularly to grab the newest videos." - /> - <StreamActionLog - trigger={() => syncAction(slug, syncQueue)} - cancelAction={cancelJobAction} - buttonLabel="Sync" - runningLabel="Syncing…" - extraControls={ - <QueueControl - value={syncQueue} - onChange={setSyncQueue} - defaultQueueKey={defaultQueueKey} - existingQueues={existingQueues} - actionLabel="Sync" - /> - } - /> - </div> - </div> - ); -} - -function Heading({ title, desc }: { title: string; desc: string }) { - return ( - <div> - <h3 className="text-base font-semibold">{title}</h3> - <p className="text-sm text-zinc-500">{desc}</p> - </div> - ); -} diff --git a/editor/app/channels/[slug]/_components/WhisperPanel.tsx b/editor/app/channels/[slug]/_components/WhisperPanel.tsx @@ -1,277 +0,0 @@ -"use client"; - -import { useState } from "react"; -import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog"; -import { - AUDIO_FORMAT_VALUES, - type AudioFormat, -} from "yt-dlp-transcript-common/lib/channelConfig"; -import { ConcurrencyControl } from "../../../_components/ConcurrencyControl"; -import { QueueControl } from "../../../_components/QueueControl"; -import { cancelJobAction } from "../../../jobs/actions"; -import { VideoIdList } from "./VideoIdList"; -import { - cleanAudioAction, - retryFailuresAction, - transcodeFailuresAction, - transcribeMissingAction, - verifyAction, - type VerifyResult, -} from "../whisperActions"; - -type Props = { - slug: string; - existingQueues: string[]; - failedVideoIds: string[]; - defaultConcurrency: number; -}; - -function parseConcurrency(s: string): number | undefined { - const trimmed = s.trim(); - if (trimmed === "") return undefined; - const n = Number(trimmed); - if (!Number.isFinite(n) || n < 1) return undefined; - return Math.floor(n); -} - -export function WhisperPanel({ - slug, - existingQueues, - failedVideoIds, - defaultConcurrency, -}: Props) { - const defaultQueueKey = `channel:${slug}`; - const [missingQueue, setMissingQueue] = useState(defaultQueueKey); - const [missingConcurrency, setMissingConcurrency] = useState(""); - const [retryQueue, setRetryQueue] = useState(defaultQueueKey); - const [retryConcurrency, setRetryConcurrency] = useState(""); - const [transcodeFormat, setTranscodeFormat] = useState<AudioFormat>("mp3"); - const [transcodeQueue, setTranscodeQueue] = useState(defaultQueueKey); - const [cleanQueue, setCleanQueue] = useState(defaultQueueKey); - return ( - <div className="flex flex-col gap-6"> - <div className="flex flex-col gap-2"> - <Heading - title="Transcribe missing" - desc="Run whisper-cli over every video that has audio but no transcript.json. Failures append to channels/<slug>/failed-transcriptions." - /> - <StreamActionLog - trigger={() => - transcribeMissingAction( - slug, - missingQueue, - parseConcurrency(missingConcurrency), - ) - } - cancelAction={cancelJobAction} - buttonLabel="Transcribe missing" - runningLabel="Transcribing…" - extraControls={ - <> - <QueueControl - value={missingQueue} - onChange={setMissingQueue} - defaultQueueKey={defaultQueueKey} - existingQueues={existingQueues} - actionLabel="Transcribe missing" - /> - <ConcurrencyControl - value={missingConcurrency} - onChange={setMissingConcurrency} - defaultLimit={defaultConcurrency} - actionLabel="Transcribe missing" - /> - </> - } - /> - </div> - <div className="flex flex-col gap-2"> - <Heading - title="Retry failures" - desc="Re-run whisper for any video listed below; successes will be removed from the list." - /> - <StreamActionLog - trigger={() => - retryFailuresAction( - slug, - retryQueue, - parseConcurrency(retryConcurrency), - ) - } - cancelAction={cancelJobAction} - buttonLabel="Retry failures" - runningLabel="Retrying…" - extraControls={ - <> - <QueueControl - value={retryQueue} - onChange={setRetryQueue} - defaultQueueKey={defaultQueueKey} - existingQueues={existingQueues} - actionLabel="Retry failures" - /> - <ConcurrencyControl - value={retryConcurrency} - onChange={setRetryConcurrency} - defaultLimit={defaultConcurrency} - actionLabel="Retry failures" - /> - </> - } - /> - <VideoIdList - slug={slug} - ids={failedVideoIds} - ariaLabel="failed transcriptions list" - emptyAriaLabel="failed transcriptions empty" - emptyMessage="No failed transcriptions." - itemAriaLabel={(id) => `failed transcription ${id}`} - /> - </div> - <div className="flex flex-col gap-2"> - <Heading - title="Transcode failures" - desc="Re-encode the audio for every video in the failure list to a different format. Useful when a codec is incompatible with whisper. Existing audio files are kept; the new file is written as audio.<format>." - /> - <label className="flex items-center gap-2 text-sm"> - Target format - <select - value={transcodeFormat} - onChange={(e) => - setTranscodeFormat(e.target.value as AudioFormat) - } - aria-label="transcode failures target format" - className="rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 px-2 py-1 text-sm" - > - {AUDIO_FORMAT_VALUES.map((f) => ( - <option key={f} value={f}> - {f} - </option> - ))} - </select> - </label> - <StreamActionLog - trigger={() => - transcodeFailuresAction(slug, transcodeFormat, transcodeQueue) - } - cancelAction={cancelJobAction} - buttonLabel={`Transcode failures → ${transcodeFormat}`} - runningLabel="Transcoding…" - label="Transcode failures" - extraControls={ - <QueueControl - value={transcodeQueue} - onChange={setTranscodeQueue} - defaultQueueKey={defaultQueueKey} - existingQueues={existingQueues} - actionLabel="Transcode failures" - /> - } - /> - </div> - <div className="flex flex-col gap-2"> - <Heading - title="Clean audio for transcribed videos" - desc="Delete audio.* files in any video directory that has a transcript.json (whisper output). YouTube videos with only auto-sub .vtt files have no audio and are skipped automatically." - /> - <StreamActionLog - trigger={() => cleanAudioAction(slug, cleanQueue)} - cancelAction={cancelJobAction} - buttonLabel="Clean audio" - runningLabel="Cleaning…" - label="Clean audio" - extraControls={ - <QueueControl - value={cleanQueue} - onChange={setCleanQueue} - defaultQueueKey={defaultQueueKey} - existingQueues={existingQueues} - actionLabel="Clean audio" - /> - } - /> - </div> - <div className="flex flex-col gap-2"> - <Heading - title="Verify" - desc="Cross-check the archive against transcripts on disk." - /> - <VerifyButton slug={slug} /> - </div> - </div> - ); -} - -function VerifyButton({ slug }: { slug: string }) { - const [running, setRunning] = useState(false); - const [result, setResult] = useState<VerifyResult | null>(null); - - async function handleClick() { - setRunning(true); - setResult(null); - try { - const r = await verifyAction(slug); - setResult(r); - } finally { - setRunning(false); - } - } - - return ( - <div className="flex flex-col gap-2"> - <button - type="button" - onClick={handleClick} - disabled={running} - className="self-start px-3 py-2 rounded-md bg-zinc-900 dark:bg-zinc-100 text-zinc-100 dark:text-zinc-900 text-sm font-medium hover:opacity-90 disabled:opacity-50" - > - {running ? "Verifying…" : "Verify"} - </button> - {result?.ok === false && ( - <div - role="alert" - className="rounded border border-red-300 bg-red-50 dark:border-red-800 dark:bg-red-950 px-3 py-2 text-sm text-red-700 dark:text-red-300" - > - {result.error} - </div> - )} - {result?.ok === true && ( - <div - aria-label="verify result" - className="grid grid-cols-1 sm:grid-cols-2 gap-3 text-sm" - > - <Bucket label="Duplicates" items={result.duplicates} /> - <Bucket label="Missing" items={result.missing} /> - </div> - )} - </div> - ); -} - -function Bucket({ label, items }: { label: string; items: string[] }) { - return ( - <div className="rounded border border-zinc-200 dark:border-zinc-800 p-3"> - <div className="text-xs uppercase tracking-wide text-zinc-500"> - {label} ({items.length}) - </div> - {items.length === 0 ? ( - <p className="text-zinc-500 mt-1">None</p> - ) : ( - <ul className="mt-1 font-mono text-xs space-y-0.5"> - {items.map((id) => ( - <li key={id}>{id}</li> - ))} - </ul> - )} - </div> - ); -} - -function Heading({ title, desc }: { title: string; desc: string }) { - return ( - <div> - <h3 className="text-base font-semibold">{title}</h3> - <p className="text-sm text-zinc-500">{desc}</p> - </div> - ); -} diff --git a/editor/app/channels/[slug]/components/PipelinePanel.tsx b/editor/app/channels/[slug]/components/PipelinePanel.tsx @@ -0,0 +1,184 @@ +"use client"; + +import { useState } from "react"; +import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog"; +import { QueueControl } from "../../../components/QueueControl"; +import { cancelJobAction } from "../../../jobs/actions"; +import { + downloadAction, + downloadMissingAction, + storePlaylistAction, + syncAction, +} from "../pipelineActions"; + +type Props = { + slug: string; + hasUrl: boolean; + defaultQueueKey: string; + existingQueues: string[]; +}; + +export function PipelinePanel({ + slug, + hasUrl, + defaultQueueKey, + existingQueues, +}: Props) { + const [storeQueue, setStoreQueue] = useState(defaultQueueKey); + const [downloadQueue, setDownloadQueue] = useState(defaultQueueKey); + const [missingQueue, setMissingQueue] = useState(defaultQueueKey); + const [syncQueue, setSyncQueue] = useState(defaultQueueKey); + const [ignoreArchive, setIgnoreArchive] = useState(false); + const [downloadAbortOnError, setDownloadAbortOnError] = useState(true); + const [missingAbortOnError, setMissingAbortOnError] = useState(true); + + if (!hasUrl) { + return ( + <p className="text-sm text-zinc-500"> + Set a <code>url</code> in the configuration above to enable the + pipeline. + </p> + ); + } + return ( + <div className="flex flex-col gap-6"> + <div className="flex flex-col gap-2"> + <Heading + title="Store playlist" + desc="Fetch the channel's full URL list (--flat-playlist --skip-download --print url) and save it for later runs." + /> + <StreamActionLog + trigger={() => storePlaylistAction(slug, storeQueue)} + cancelAction={cancelJobAction} + buttonLabel="Store playlist" + runningLabel="Storing playlist…" + extraControls={ + <QueueControl + value={storeQueue} + onChange={setStoreQueue} + defaultQueueKey={defaultQueueKey} + existingQueues={existingQueues} + actionLabel="Store playlist" + /> + } + /> + </div> + <div className="flex flex-col gap-2"> + <Heading + title="Download from playlist" + desc="Read the saved playlist, prefilter against the archive, then run yt-dlp on whatever's left. Use for first-time imports and resumes." + /> + <label className="flex items-center gap-2 text-sm"> + <input + type="checkbox" + checked={downloadAbortOnError} + onChange={(e) => setDownloadAbortOnError(e.target.checked)} + aria-label="abort on error for Download from playlist" + /> + Abort on error + <span className="text-xs text-zinc-500"> + (stop the run on the first per-video failure) + </span> + </label> + <StreamActionLog + trigger={() => + downloadAction(slug, downloadQueue, downloadAbortOnError) + } + cancelAction={cancelJobAction} + buttonLabel="Download from playlist" + runningLabel="Downloading…" + extraControls={ + <QueueControl + value={downloadQueue} + onChange={setDownloadQueue} + defaultQueueKey={defaultQueueKey} + existingQueues={existingQueues} + actionLabel="Download from playlist" + /> + } + /> + </div> + <div className="flex flex-col gap-2"> + <Heading + title="Download videos" + desc="Read the saved playlist, then re-download any video whose destination file (transcript.en.vtt for YouTube, or audio.* / transcript.json for transcribe) is missing on disk. Useful when the archive is out of sync with the data directory." + /> + <label className="flex items-center gap-2 text-sm"> + <input + type="checkbox" + checked={ignoreArchive} + onChange={(e) => setIgnoreArchive(e.target.checked)} + /> + Ignore archive + <span className="text-xs text-zinc-500"> + (re-fetch even if listed in the archive file) + </span> + </label> + <label className="flex items-center gap-2 text-sm"> + <input + type="checkbox" + checked={missingAbortOnError} + onChange={(e) => setMissingAbortOnError(e.target.checked)} + aria-label="abort on error for Download videos" + /> + Abort on error + <span className="text-xs text-zinc-500"> + (stop the run on the first per-video failure) + </span> + </label> + <StreamActionLog + trigger={() => + downloadMissingAction( + slug, + missingQueue, + ignoreArchive, + missingAbortOnError, + ) + } + cancelAction={cancelJobAction} + buttonLabel="Download videos" + runningLabel="Downloading missing…" + extraControls={ + <QueueControl + value={missingQueue} + onChange={setMissingQueue} + defaultQueueKey={defaultQueueKey} + existingQueues={existingQueues} + actionLabel="Download videos" + /> + } + /> + </div> + <div className="flex flex-col gap-2"> + <Heading + title="Sync" + desc="Quick incremental fetch using --lazy-playlist + --break-on-existing. Run regularly to grab the newest videos." + /> + <StreamActionLog + trigger={() => syncAction(slug, syncQueue)} + cancelAction={cancelJobAction} + buttonLabel="Sync" + runningLabel="Syncing…" + extraControls={ + <QueueControl + value={syncQueue} + onChange={setSyncQueue} + defaultQueueKey={defaultQueueKey} + existingQueues={existingQueues} + actionLabel="Sync" + /> + } + /> + </div> + </div> + ); +} + +function Heading({ title, desc }: { title: string; desc: string }) { + return ( + <div> + <h3 className="text-base font-semibold">{title}</h3> + <p className="text-sm text-zinc-500">{desc}</p> + </div> + ); +} diff --git a/editor/app/channels/[slug]/components/RefreshSnapshotButton.tsx b/editor/app/channels/[slug]/components/RefreshSnapshotButton.tsx @@ -0,0 +1,23 @@ +"use client"; + +import { useTransition } from "react"; +import { refreshChannelSnapshotAction } from "../../actions"; + +export function RefreshSnapshotButton({ slug }: { slug: string }) { + const [pending, startTransition] = useTransition(); + return ( + <button + type="button" + onClick={() => + startTransition(async () => { + await refreshChannelSnapshotAction(slug); + }) + } + disabled={pending} + aria-label="refresh channel report" + className="self-start px-3 py-1 rounded-md bg-zinc-900 dark:bg-zinc-100 text-zinc-100 dark:text-zinc-900 text-sm font-medium hover:opacity-90 disabled:opacity-50" + > + {pending ? "Refreshing…" : "Refresh report"} + </button> + ); +} diff --git a/editor/app/channels/[slug]/_components/VideoIdList.tsx b/editor/app/channels/[slug]/components/VideoIdList.tsx diff --git a/editor/app/channels/[slug]/components/WhisperPanel.tsx b/editor/app/channels/[slug]/components/WhisperPanel.tsx @@ -0,0 +1,288 @@ +"use client"; + +import { useState } from "react"; +import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog"; +import { + AUDIO_FORMAT_VALUES, + type AudioFormat, +} from "yt-dlp-transcript-common/lib/channelConfig"; +import { ConcurrencyControl } from "../../../components/ConcurrencyControl"; +import { QueueControl } from "../../../components/QueueControl"; +import { cancelJobAction } from "../../../jobs/actions"; +import { VideoIdList } from "./VideoIdList"; +import { + cleanAudioAction, + retryFailuresAction, + transcodeFailuresAction, + transcribeMissingAction, + verifyAction, + type VerifyResult, +} from "../whisperActions"; + +type Props = { + slug: string; + existingQueues: string[]; + failedVideoIds: string[]; + defaultConcurrency: number; +}; + +function parseConcurrency(s: string): number | undefined { + const trimmed = s.trim(); + if (trimmed === "") return undefined; + const n = Number(trimmed); + if (!Number.isFinite(n) || n < 1) return undefined; + return Math.floor(n); +} + +export function WhisperPanel({ + slug, + existingQueues, + failedVideoIds, + defaultConcurrency, +}: Props) { + const defaultQueueKey = `channel:${slug}`; + const [missingQueue, setMissingQueue] = useState(defaultQueueKey); + const [missingConcurrency, setMissingConcurrency] = useState(""); + const [missingReverse, setMissingReverse] = useState(false); + const [retryQueue, setRetryQueue] = useState(defaultQueueKey); + const [retryConcurrency, setRetryConcurrency] = useState(""); + const [transcodeFormat, setTranscodeFormat] = useState<AudioFormat>("mp3"); + const [transcodeQueue, setTranscodeQueue] = useState(defaultQueueKey); + const [cleanQueue, setCleanQueue] = useState(defaultQueueKey); + return ( + <div className="flex flex-col gap-6"> + <div className="flex flex-col gap-2"> + <Heading + title="Transcribe missing" + desc="Run whisper-cli over every video that has audio but no transcript.json. Failures append to channels/<slug>/failed-transcriptions." + /> + <StreamActionLog + trigger={() => + transcribeMissingAction( + slug, + missingQueue, + parseConcurrency(missingConcurrency), + missingReverse, + ) + } + cancelAction={cancelJobAction} + buttonLabel="Transcribe missing" + runningLabel="Transcribing…" + extraControls={ + <> + <QueueControl + value={missingQueue} + onChange={setMissingQueue} + defaultQueueKey={defaultQueueKey} + existingQueues={existingQueues} + actionLabel="Transcribe missing" + /> + <ConcurrencyControl + value={missingConcurrency} + onChange={setMissingConcurrency} + defaultLimit={defaultConcurrency} + actionLabel="Transcribe missing" + /> + <label className="flex items-center gap-1 text-xs text-zinc-500"> + <input + type="checkbox" + checked={missingReverse} + onChange={(e) => setMissingReverse(e.target.checked)} + aria-label="reverse order for Transcribe missing" + /> + Reverse + </label> + </> + } + /> + </div> + <div className="flex flex-col gap-2"> + <Heading + title="Retry failures" + desc="Re-run whisper for any video listed below; successes will be removed from the list." + /> + <StreamActionLog + trigger={() => + retryFailuresAction( + slug, + retryQueue, + parseConcurrency(retryConcurrency), + ) + } + cancelAction={cancelJobAction} + buttonLabel="Retry failures" + runningLabel="Retrying…" + extraControls={ + <> + <QueueControl + value={retryQueue} + onChange={setRetryQueue} + defaultQueueKey={defaultQueueKey} + existingQueues={existingQueues} + actionLabel="Retry failures" + /> + <ConcurrencyControl + value={retryConcurrency} + onChange={setRetryConcurrency} + defaultLimit={defaultConcurrency} + actionLabel="Retry failures" + /> + </> + } + /> + <VideoIdList + slug={slug} + ids={failedVideoIds} + ariaLabel="failed transcriptions list" + emptyAriaLabel="failed transcriptions empty" + emptyMessage="No failed transcriptions." + itemAriaLabel={(id) => `failed transcription ${id}`} + /> + </div> + <div className="flex flex-col gap-2"> + <Heading + title="Transcode failures" + desc="Re-encode the audio for every video in the failure list to a different format. Useful when a codec is incompatible with whisper. Existing audio files are kept; the new file is written as audio.<format>." + /> + <label className="flex items-center gap-2 text-sm"> + Target format + <select + value={transcodeFormat} + onChange={(e) => + setTranscodeFormat(e.target.value as AudioFormat) + } + aria-label="transcode failures target format" + className="rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 px-2 py-1 text-sm" + > + {AUDIO_FORMAT_VALUES.map((f) => ( + <option key={f} value={f}> + {f} + </option> + ))} + </select> + </label> + <StreamActionLog + trigger={() => + transcodeFailuresAction(slug, transcodeFormat, transcodeQueue) + } + cancelAction={cancelJobAction} + buttonLabel={`Transcode failures → ${transcodeFormat}`} + runningLabel="Transcoding…" + label="Transcode failures" + extraControls={ + <QueueControl + value={transcodeQueue} + onChange={setTranscodeQueue} + defaultQueueKey={defaultQueueKey} + existingQueues={existingQueues} + actionLabel="Transcode failures" + /> + } + /> + </div> + <div className="flex flex-col gap-2"> + <Heading + title="Clean audio for transcribed videos" + desc="Delete audio.* files in any video directory that has a transcript.json (whisper output). YouTube videos with only auto-sub .vtt files have no audio and are skipped automatically." + /> + <StreamActionLog + trigger={() => cleanAudioAction(slug, cleanQueue)} + cancelAction={cancelJobAction} + buttonLabel="Clean audio" + runningLabel="Cleaning…" + label="Clean audio" + extraControls={ + <QueueControl + value={cleanQueue} + onChange={setCleanQueue} + defaultQueueKey={defaultQueueKey} + existingQueues={existingQueues} + actionLabel="Clean audio" + /> + } + /> + </div> + <div className="flex flex-col gap-2"> + <Heading + title="Verify" + desc="Cross-check the archive against transcripts on disk." + /> + <VerifyButton slug={slug} /> + </div> + </div> + ); +} + +function VerifyButton({ slug }: { slug: string }) { + const [running, setRunning] = useState(false); + const [result, setResult] = useState<VerifyResult | null>(null); + + async function handleClick() { + setRunning(true); + setResult(null); + try { + const r = await verifyAction(slug); + setResult(r); + } finally { + setRunning(false); + } + } + + return ( + <div className="flex flex-col gap-2"> + <button + type="button" + onClick={handleClick} + disabled={running} + className="self-start px-3 py-2 rounded-md bg-zinc-900 dark:bg-zinc-100 text-zinc-100 dark:text-zinc-900 text-sm font-medium hover:opacity-90 disabled:opacity-50" + > + {running ? "Verifying…" : "Verify"} + </button> + {result?.ok === false && ( + <div + role="alert" + className="rounded border border-red-300 bg-red-50 dark:border-red-800 dark:bg-red-950 px-3 py-2 text-sm text-red-700 dark:text-red-300" + > + {result.error} + </div> + )} + {result?.ok === true && ( + <div + aria-label="verify result" + className="grid grid-cols-1 sm:grid-cols-2 gap-3 text-sm" + > + <Bucket label="Duplicates" items={result.duplicates} /> + <Bucket label="Missing" items={result.missing} /> + </div> + )} + </div> + ); +} + +function Bucket({ label, items }: { label: string; items: string[] }) { + return ( + <div className="rounded border border-zinc-200 dark:border-zinc-800 p-3"> + <div className="text-xs uppercase tracking-wide text-zinc-500"> + {label} ({items.length}) + </div> + {items.length === 0 ? ( + <p className="text-zinc-500 mt-1">None</p> + ) : ( + <ul className="mt-1 font-mono text-xs space-y-0.5"> + {items.map((id) => ( + <li key={id}>{id}</li> + ))} + </ul> + )} + </div> + ); +} + +function Heading({ title, desc }: { title: string; desc: string }) { + return ( + <div> + <h3 className="text-base font-semibold">{title}</h3> + <p className="text-sm text-zinc-500">{desc}</p> + </div> + ); +} diff --git a/editor/app/channels/[slug]/page.tsx b/editor/app/channels/[slug]/page.tsx @@ -2,20 +2,24 @@ import type { Metadata } from "next"; import Link from "next/link"; import { notFound } from "next/navigation"; import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels"; -import { inspectChannelHealth, type ChannelHealth } from "yt-dlp-transcript-common/controller/channelHealth"; +import { + generateChannelSnapshot, + readChannelSnapshot, + type ChannelSnapshot, +} from "yt-dlp-transcript-common/controller/channelSnapshot"; import { loadFailedTranscriptions } from "yt-dlp-transcript-common/controller/failedTranscriptions"; -import { listUndownloadedVideos } from "yt-dlp-transcript-common/controller/undownloadedVideos"; import { getPaths } from "yt-dlp-transcript-common/lib/paths"; import { detectPlatform, platformQueueKey, } from "yt-dlp-transcript-common/lib/platform"; import { getRegistry } from "yt-dlp-transcript-common/jobs/registry"; -import { ChannelFormClient } from "../_components/ChannelFormClient"; -import { DeleteChannelForm } from "../_components/DeleteChannelForm"; -import { PipelinePanel } from "./_components/PipelinePanel"; -import { VideoIdList } from "./_components/VideoIdList"; -import { WhisperPanel } from "./_components/WhisperPanel"; +import { ChannelFormClient } from "../components/ChannelFormClient"; +import { DeleteChannelForm } from "../components/DeleteChannelForm"; +import { PipelinePanel } from "./components/PipelinePanel"; +import { RefreshSnapshotButton } from "./components/RefreshSnapshotButton"; +import { VideoIdList } from "./components/VideoIdList"; +import { WhisperPanel } from "./components/WhisperPanel"; import { deleteChannelAction, gotoVideoAction, @@ -52,10 +56,12 @@ export default async function ChannelDetailPage({ const defaultQueueKey = platformQueueKey( config.platform ?? detectPlatform(config.url), ); + const existing = await readChannelSnapshot(paths, slug); + const snapshot = existing ?? (await generateChannelSnapshot(paths, slug)); + // The retry-failures panel is interactive (a click mutates the file), so + // read it fresh on every render — the snapshot bucket only reflects state + // at refresh time. const failedVideoIds = await loadFailedTranscriptions(paths, slug); - const undownloadedVideos = await listUndownloadedVideos(paths, slug, config); - const undownloadedIds = undownloadedVideos.map((v) => v.videoId); - const health = await inspectChannelHealth(paths, slug); return ( <div className="flex flex-col gap-6"> @@ -92,21 +98,7 @@ export default async function ChannelDetailPage({ /> </section> - <section className="flex flex-col gap-3 border-t border-zinc-200 dark:border-zinc-800 pt-6"> - <h2 className="text-lg font-semibold"> - Undownloaded videos ({undownloadedIds.length}) - </h2> - <VideoIdList - slug={slug} - ids={undownloadedIds} - ariaLabel="undownloaded videos list" - emptyAriaLabel="undownloaded videos empty" - emptyMessage="All playlist videos are downloaded." - itemAriaLabel={(id) => `undownloaded video ${id}`} - /> - </section> - - <ChannelHealthSection slug={slug} health={health} /> + <ChannelReportSection slug={slug} snapshot={snapshot} fresh={!existing} /> <section className="flex flex-col gap-3 border-t border-zinc-200 dark:border-zinc-800 pt-6"> <h2 className="text-lg font-semibold">Transcription</h2> @@ -164,7 +156,7 @@ export default async function ChannelDetailPage({ } type Bucket = { - key: keyof Omit<ChannelHealth, "totals">; + key: keyof ChannelSnapshot["buckets"]; label: string; description: string; ariaLabel: string; @@ -215,25 +207,86 @@ const BUCKETS: ReadonlyArray<Bucket> = [ }, ]; -function ChannelHealthSection({ +function ChannelReportSection({ slug, - health, + snapshot, + fresh, }: { slug: string; - health: ChannelHealth; + snapshot: ChannelSnapshot; + fresh: boolean; }) { - const populated = BUCKETS.filter((b) => health[b.key].length > 0); - const total = populated.reduce((acc, b) => acc + health[b.key].length, 0); return ( <section - aria-label="channel health" + aria-label="channel report" className="flex flex-col gap-3 border-t border-zinc-200 dark:border-zinc-800 pt-6" > + <div className="flex items-start justify-between gap-3 flex-wrap"> + <div> + <h2 className="text-lg font-semibold">Report</h2> + <p className="text-sm text-zinc-500"> + {fresh ? "Generated just now" : "Generated "} + {fresh ? null : ( + <time dateTime={snapshot.generatedAt}> + {formatTimestamp(snapshot.generatedAt)} + </time> + )} + . Updates only when refreshed. + </p> + </div> + <RefreshSnapshotButton slug={slug} /> + </div> + <UndownloadedSubsection slug={slug} ids={snapshot.undownloadedIds} /> + <ChannelHealthSubsection slug={slug} snapshot={snapshot} /> + </section> + ); +} + +function UndownloadedSubsection({ + slug, + ids, +}: { + slug: string; + ids: string[]; +}) { + return ( + <div className="flex flex-col gap-2"> + <h3 className="text-base font-semibold"> + Undownloaded videos ({ids.length}) + </h3> + <VideoIdList + slug={slug} + ids={ids} + ariaLabel="undownloaded videos list" + emptyAriaLabel="undownloaded videos empty" + emptyMessage="All playlist videos are downloaded." + itemAriaLabel={(id) => `undownloaded video ${id}`} + /> + </div> + ); +} + +function ChannelHealthSubsection({ + slug, + snapshot, +}: { + slug: string; + snapshot: ChannelSnapshot; +}) { + const populated = BUCKETS.filter( + (b) => snapshot.buckets[b.key].length > 0, + ); + const total = populated.reduce( + (acc, b) => acc + snapshot.buckets[b.key].length, + 0, + ); + return ( + <div aria-label="channel health" className="flex flex-col gap-3"> <div> - <h2 className="text-lg font-semibold">Channel health</h2> + <h3 className="text-base font-semibold">Channel health</h3> <p className="text-sm text-zinc-500"> - {health.totals.videos} video dirs · {health.totals.transcribed}{" "} - transcribed · {health.totals.downloaded} downloaded + {snapshot.totals.videos} video dirs · {snapshot.totals.transcribed}{" "} + transcribed · {snapshot.totals.downloaded} downloaded {total > 0 ? ` · ${total} abnormal` : ""} </p> </div> @@ -252,14 +305,14 @@ function ChannelHealthSection({ className="flex flex-col gap-2 rounded border border-zinc-200 dark:border-zinc-800 p-3" > <div> - <h3 className="text-sm font-semibold"> - {b.label} ({health[b.key].length}) - </h3> + <h4 className="text-sm font-semibold"> + {b.label} ({snapshot.buckets[b.key].length}) + </h4> <p className="text-xs text-zinc-500">{b.description}</p> </div> <VideoIdList slug={slug} - ids={health[b.key]} + ids={snapshot.buckets[b.key]} ariaLabel={`${b.ariaLabel} list`} emptyAriaLabel={`${b.ariaLabel} empty`} emptyMessage="None" @@ -269,6 +322,14 @@ function ChannelHealthSection({ ))} </div> )} - </section> + </div> ); } + +function formatTimestamp(iso: string): string { + try { + return new Date(iso).toLocaleString(); + } catch { + return iso; + } +} diff --git a/editor/app/channels/[slug]/videos/[id]/_components/VideoPanel.tsx b/editor/app/channels/[slug]/videos/[id]/_components/VideoPanel.tsx @@ -1,523 +0,0 @@ -"use client"; - -import { useActionState, useState, useTransition } from "react"; -import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog"; -import { - AUDIO_FORMAT_VALUES, - type AudioFormat, - type ChannelHandling, -} from "yt-dlp-transcript-common/lib/channelConfig"; -import { QueueControl } from "../../../../../_components/QueueControl"; -import { cancelJobAction } from "../../../../../jobs/actions"; -import { - deleteVideoDirAction, - deleteVideoFileAction, - markVideoUntranscribableAction, - redownloadVideoAction, - transcodeAudioAction, - transcribeOneAction, - whisperVideoAction, - type DeleteDirActionResult, -} from "../videoActions"; - -export type VideoFile = { - name: string; - size: number; - mtime: number; -}; - -type Props = { - slug: string; - videoId: string; - files: VideoFile[]; - handling: ChannelHandling; - audioFormat: AudioFormat; - defaultQueueKey: string; - existingQueues: string[]; -}; - -function isAudioFile(name: string): boolean { - return name.startsWith("audio."); -} - -function audioExt(name: string): string { - const dot = name.lastIndexOf("."); - return dot >= 0 ? name.slice(dot + 1) : ""; -} - -function formatSize(bytes: number): string { - if (bytes < 1024) return `${bytes} B`; - if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; - if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; - return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; -} - -function parseExtraArgs(raw: string): string[] { - return raw - .split(/\s+/) - .map((s) => s.trim()) - .filter(Boolean); -} - -export function VideoPanel({ - slug, - videoId, - files, - handling, - audioFormat, - defaultQueueKey, - existingQueues, -}: Props) { - const audioFiles = files.filter((f) => isAudioFile(f.name)); - const hasTranscriptJson = files.some((f) => f.name === "transcript.json"); - const hasYtVtt = files.some((f) => f.name === "transcript.en.vtt"); - const hasTranscript = hasTranscriptJson || hasYtVtt; - const noAudio = audioFiles.length === 0; - - return ( - <div className="flex flex-col gap-6"> - <RedownloadSection - slug={slug} - videoId={videoId} - handling={handling} - defaultFormat={audioFormat} - defaultQueueKey={defaultQueueKey} - existingQueues={existingQueues} - noAudio={noAudio} - /> - - {!hasTranscript && ( - <WhisperVideoSection - slug={slug} - videoId={videoId} - existingQueues={existingQueues} - noAudio={noAudio} - /> - )} - - {!hasTranscript && ( - <MarkUntranscribableSection slug={slug} videoId={videoId} /> - )} - - <section className="flex flex-col gap-3"> - <h2 className="text-lg font-semibold">Files</h2> - {files.length === 0 ? ( - <p className="text-sm text-zinc-500">Directory is empty.</p> - ) : ( - <ul - aria-label={`files for ${videoId}`} - className="flex flex-col gap-3" - > - {files.map((f) => ( - <li - key={f.name} - className="border border-zinc-200 dark:border-zinc-800 rounded-md p-3 flex flex-col gap-2" - > - <div className="flex items-baseline justify-between gap-3 flex-wrap"> - <span className="font-mono text-sm" aria-label={`file name ${f.name}`}> - {f.name} - </span> - <span className="text-xs text-zinc-500"> - {formatSize(f.size)} · {new Date(f.mtime).toLocaleString()} - </span> - </div> - {isAudioFile(f.name) && ( - <AudioFileActions - slug={slug} - videoId={videoId} - file={f} - existingQueues={existingQueues} - hasTranscript={hasTranscript} - /> - )} - <DeleteFileButton - slug={slug} - videoId={videoId} - filename={f.name} - /> - </li> - ))} - </ul> - )} - </section> - - <DeleteVideoDirSection slug={slug} videoId={videoId} /> - </div> - ); -} - -function RedownloadSection({ - slug, - videoId, - handling, - defaultFormat, - defaultQueueKey, - existingQueues, - noAudio, -}: { - slug: string; - videoId: string; - handling: ChannelHandling; - defaultFormat: AudioFormat; - defaultQueueKey: string; - existingQueues: string[]; - noAudio: boolean; -}) { - const [format, setFormat] = useState<AudioFormat>(defaultFormat); - const [extraArgsRaw, setExtraArgsRaw] = useState(""); - const [queueKey, setQueueKey] = useState(defaultQueueKey); - const desc = noAudio - ? handling === "youtube" - ? "YouTube channels normally only fetch auto-subs. Use this when the video has no auto-subs and you want to transcribe it manually." - : "Fetch the audio for this video using yt-dlp." - : "Re-fetch the audio for this video. The existing audio file is kept on disk; yt-dlp may overwrite it with the new one if names collide."; - const actionLabel = `${noAudio ? "Download" : "Redownload"} audio for ${videoId}`; - return ( - <section className="flex flex-col gap-3 border border-zinc-200 dark:border-zinc-800 rounded-md p-4"> - <Heading - title={noAudio ? "Download audio" : "Redownload audio"} - desc={desc} - /> - <div className="flex flex-col sm:flex-row sm:flex-wrap sm:items-center gap-3"> - <label className="flex items-center gap-2 text-sm"> - Format - <select - value={format} - onChange={(e) => setFormat(e.target.value as AudioFormat)} - aria-label="redownload audio format" - className="rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 px-2 py-1 text-sm" - > - {AUDIO_FORMAT_VALUES.map((f) => ( - <option key={f} value={f}> - {f} - </option> - ))} - </select> - </label> - <label className="flex flex-col sm:flex-row sm:items-center gap-2 text-sm flex-1 sm:min-w-[16rem]"> - <span>Extra yt-dlp args</span> - <input - type="text" - value={extraArgsRaw} - onChange={(e) => setExtraArgsRaw(e.target.value)} - aria-label="redownload extra yt-dlp args" - placeholder="--cookies-from-browser firefox --quiet" - className="flex-1 rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 px-2 py-1 text-sm font-mono min-w-0" - /> - </label> - </div> - <StreamActionLog - trigger={() => - redownloadVideoAction(slug, videoId, { - audioFormat: format, - extraArgs: parseExtraArgs(extraArgsRaw), - queueKey, - }) - } - cancelAction={cancelJobAction} - buttonLabel={`${noAudio ? "Download" : "Redownload"} audio (${format})`} - runningLabel={noAudio ? "Downloading…" : "Redownloading…"} - label={actionLabel} - extraControls={ - <QueueControl - value={queueKey} - onChange={setQueueKey} - defaultQueueKey={defaultQueueKey} - existingQueues={existingQueues} - actionLabel={actionLabel} - /> - } - /> - </section> - ); -} - -function WhisperVideoSection({ - slug, - videoId, - existingQueues, - noAudio, -}: { - slug: string; - videoId: string; - existingQueues: string[]; - noAudio: boolean; -}) { - // Run immediately in parallel by default — matches the per-audio-file - // Transcribe button's convention. - const [queueKey, setQueueKey] = useState(""); - const desc = noAudio - ? "Download the audio and run whisper-cli, in one job. Useful when a YouTube video has no auto-subs." - : "Run whisper-cli on this video's audio. Skips the download phase since audio is already on disk."; - const actionLabel = `Whisper transcribe ${videoId}`; - return ( - <section className="flex flex-col gap-3 border border-zinc-200 dark:border-zinc-800 rounded-md p-4"> - <Heading title="Whisper transcribe" desc={desc} /> - <StreamActionLog - trigger={() => whisperVideoAction(slug, videoId, queueKey)} - cancelAction={cancelJobAction} - buttonLabel="Whisper transcribe" - runningLabel="Transcribing…" - label={actionLabel} - extraControls={ - <QueueControl - value={queueKey} - onChange={setQueueKey} - defaultQueueKey="" - existingQueues={existingQueues} - actionLabel={actionLabel} - /> - } - /> - </section> - ); -} - -function AudioFileActions({ - slug, - videoId, - file, - existingQueues, - hasTranscript, -}: { - slug: string; - videoId: string; - file: VideoFile; - existingQueues: string[]; - hasTranscript: boolean; -}) { - const ext = audioExt(file.name); - const transcodeTargets = AUDIO_FORMAT_VALUES.filter((f) => f !== ext); - const [target, setTarget] = useState<AudioFormat>(transcodeTargets[0] ?? "opus"); - // Per-file Transcribe and Transcode default to "" — single-file jobs run - // immediately, in parallel with channel-wide batches. - const [transcribeQueue, setTranscribeQueue] = useState(""); - const [transcodeQueue, setTranscodeQueue] = useState(""); - - return ( - <div className="flex flex-col gap-3"> - <div className="flex flex-col gap-1"> - <Heading - title="Transcribe" - desc={ - hasTranscript - ? "A transcript already exists for this video. Delete transcript.json (or transcript.en.vtt) on disk to re-run." - : "Run whisper-cli on this audio file. Writes transcript.json next to it." - } - /> - {hasTranscript ? ( - <p className="text-sm text-zinc-500"> - Transcribe disabled — transcript already present. - </p> - ) : ( - <StreamActionLog - trigger={() => - transcribeOneAction(slug, videoId, file.name, transcribeQueue) - } - cancelAction={cancelJobAction} - buttonLabel={`Transcribe ${file.name}`} - runningLabel="Transcribing…" - label={`Transcribe ${file.name}`} - extraControls={ - <QueueControl - value={transcribeQueue} - onChange={setTranscribeQueue} - defaultQueueKey="" - existingQueues={existingQueues} - actionLabel={`Transcribe ${file.name}`} - /> - } - /> - )} - </div> - {transcodeTargets.length > 0 && ( - <div className="flex flex-col gap-1"> - <Heading - title="Transcode" - desc="Re-encode this audio file with ffmpeg. The source file is kept; the output is named audio.<format>." - /> - <label className="flex items-center gap-2 text-sm"> - Target format - <select - value={target} - onChange={(e) => setTarget(e.target.value as AudioFormat)} - aria-label={`transcode target for ${file.name}`} - className="rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 px-2 py-1 text-sm" - > - {transcodeTargets.map((t) => ( - <option key={t} value={t}> - {t} - </option> - ))} - </select> - </label> - <StreamActionLog - trigger={() => - transcodeAudioAction( - slug, - videoId, - file.name, - target, - transcodeQueue, - ) - } - cancelAction={cancelJobAction} - buttonLabel={`Transcode ${file.name} → ${target}`} - runningLabel="Transcoding…" - label={`Transcode ${file.name} to ${target}`} - extraControls={ - <QueueControl - value={transcodeQueue} - onChange={setTranscodeQueue} - defaultQueueKey="" - existingQueues={existingQueues} - actionLabel={`Transcode ${file.name} to ${target}`} - /> - } - /> - </div> - )} - </div> - ); -} - -function MarkUntranscribableSection({ - slug, - videoId, -}: { - slug: string; - videoId: string; -}) { - const [pending, startTransition] = useTransition(); - const [error, setError] = useState<string | null>(null); - return ( - <section className="flex flex-col gap-2 border border-zinc-200 dark:border-zinc-800 rounded-md p-4"> - <Heading - title="Mark untranscribable" - desc="Writes an empty transcript.json so the pipeline stops retrying. Removes this video from the failed list. Reversible — just delete transcript.json." - /> - <button - type="button" - disabled={pending} - aria-label={`mark video ${videoId} untranscribable`} - onClick={() => { - setError(null); - startTransition(async () => { - const res = await markVideoUntranscribableAction(slug, videoId); - if (!res.ok) setError(res.error); - }); - }} - className="self-start px-3 py-2 rounded-md bg-zinc-900 dark:bg-zinc-100 text-zinc-100 dark:text-zinc-900 text-sm font-medium hover:opacity-90 disabled:opacity-50" - > - {pending ? "Marking…" : "Mark untranscribable"} - </button> - {error && ( - <span - className="text-sm text-red-600 dark:text-red-400" - aria-label="mark untranscribable error" - > - {error} - </span> - )} - </section> - ); -} - -function DeleteFileButton({ - slug, - videoId, - filename, -}: { - slug: string; - videoId: string; - filename: string; -}) { - const [pending, startTransition] = useTransition(); - const [error, setError] = useState<string | null>(null); - return ( - <div className="flex items-center gap-2"> - <button - type="button" - disabled={pending} - aria-label={`delete file ${filename}`} - onClick={() => { - if (!confirm(`Delete ${filename}?`)) return; - setError(null); - startTransition(async () => { - const res = await deleteVideoFileAction(slug, videoId, filename); - if (!res.ok) setError(res.error); - }); - }} - className="self-start px-2 py-1 rounded border border-red-300 dark:border-red-800 text-red-700 dark:text-red-400 text-xs hover:bg-red-50 dark:hover:bg-red-950 disabled:opacity-50" - > - {pending ? "Deleting…" : "Delete file"} - </button> - {error && ( - <span - className="text-xs text-red-600 dark:text-red-400" - aria-label={`delete error ${filename}`} - > - {error} - </span> - )} - </div> - ); -} - -function DeleteVideoDirSection({ - slug, - videoId, -}: { - slug: string; - videoId: string; -}) { - const action = deleteVideoDirAction.bind(null, slug, videoId); - const [state, formAction] = useActionState<DeleteDirActionResult, FormData>( - action, - undefined, - ); - return ( - <section className="flex flex-col gap-3 border-t border-red-200 dark:border-red-900 pt-4"> - <Heading - title="Delete this video directory" - desc={`Removes data/${videoId}/ and all of its contents. Type the video id to confirm.`} - /> - <form - action={formAction} - aria-label={`delete video directory ${videoId}`} - className="flex flex-wrap items-start gap-2" - > - <input - name="confirmId" - required - placeholder={videoId} - aria-label="confirm video id" - className="rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 px-2 py-1 text-sm font-mono" - /> - <button - type="submit" - className="px-3 py-1.5 rounded-md bg-red-600 text-white text-sm font-medium hover:bg-red-700" - > - Delete directory - </button> - {state?.error && ( - <p - className="basis-full text-sm text-red-600 dark:text-red-400" - aria-label="delete directory error" - > - {state.error} - </p> - )} - </form> - </section> - ); -} - -function Heading({ title, desc }: { title: string; desc: string }) { - return ( - <div> - <h3 className="text-base font-semibold">{title}</h3> - <p className="text-sm text-zinc-500">{desc}</p> - </div> - ); -} diff --git a/editor/app/channels/[slug]/videos/[id]/components/VideoPanel.tsx b/editor/app/channels/[slug]/videos/[id]/components/VideoPanel.tsx @@ -0,0 +1,523 @@ +"use client"; + +import { useActionState, useState, useTransition } from "react"; +import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog"; +import { + AUDIO_FORMAT_VALUES, + type AudioFormat, + type ChannelHandling, +} from "yt-dlp-transcript-common/lib/channelConfig"; +import { QueueControl } from "../../../../../components/QueueControl"; +import { cancelJobAction } from "../../../../../jobs/actions"; +import { + deleteVideoDirAction, + deleteVideoFileAction, + markVideoUntranscribableAction, + redownloadVideoAction, + transcodeAudioAction, + transcribeOneAction, + whisperVideoAction, + type DeleteDirActionResult, +} from "../videoActions"; + +export type VideoFile = { + name: string; + size: number; + mtime: number; +}; + +type Props = { + slug: string; + videoId: string; + files: VideoFile[]; + handling: ChannelHandling; + audioFormat: AudioFormat; + defaultQueueKey: string; + existingQueues: string[]; +}; + +function isAudioFile(name: string): boolean { + return name.startsWith("audio."); +} + +function audioExt(name: string): string { + const dot = name.lastIndexOf("."); + return dot >= 0 ? name.slice(dot + 1) : ""; +} + +function formatSize(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; +} + +function parseExtraArgs(raw: string): string[] { + return raw + .split(/\s+/) + .map((s) => s.trim()) + .filter(Boolean); +} + +export function VideoPanel({ + slug, + videoId, + files, + handling, + audioFormat, + defaultQueueKey, + existingQueues, +}: Props) { + const audioFiles = files.filter((f) => isAudioFile(f.name)); + const hasTranscriptJson = files.some((f) => f.name === "transcript.json"); + const hasYtVtt = files.some((f) => f.name === "transcript.en.vtt"); + const hasTranscript = hasTranscriptJson || hasYtVtt; + const noAudio = audioFiles.length === 0; + + return ( + <div className="flex flex-col gap-6"> + <RedownloadSection + slug={slug} + videoId={videoId} + handling={handling} + defaultFormat={audioFormat} + defaultQueueKey={defaultQueueKey} + existingQueues={existingQueues} + noAudio={noAudio} + /> + + {!hasTranscript && ( + <WhisperVideoSection + slug={slug} + videoId={videoId} + existingQueues={existingQueues} + noAudio={noAudio} + /> + )} + + {!hasTranscript && ( + <MarkUntranscribableSection slug={slug} videoId={videoId} /> + )} + + <section className="flex flex-col gap-3"> + <h2 className="text-lg font-semibold">Files</h2> + {files.length === 0 ? ( + <p className="text-sm text-zinc-500">Directory is empty.</p> + ) : ( + <ul + aria-label={`files for ${videoId}`} + className="flex flex-col gap-3" + > + {files.map((f) => ( + <li + key={f.name} + className="border border-zinc-200 dark:border-zinc-800 rounded-md p-3 flex flex-col gap-2" + > + <div className="flex items-baseline justify-between gap-3 flex-wrap"> + <span className="font-mono text-sm" aria-label={`file name ${f.name}`}> + {f.name} + </span> + <span className="text-xs text-zinc-500"> + {formatSize(f.size)} · {new Date(f.mtime).toLocaleString()} + </span> + </div> + {isAudioFile(f.name) && ( + <AudioFileActions + slug={slug} + videoId={videoId} + file={f} + existingQueues={existingQueues} + hasTranscript={hasTranscript} + /> + )} + <DeleteFileButton + slug={slug} + videoId={videoId} + filename={f.name} + /> + </li> + ))} + </ul> + )} + </section> + + <DeleteVideoDirSection slug={slug} videoId={videoId} /> + </div> + ); +} + +function RedownloadSection({ + slug, + videoId, + handling, + defaultFormat, + defaultQueueKey, + existingQueues, + noAudio, +}: { + slug: string; + videoId: string; + handling: ChannelHandling; + defaultFormat: AudioFormat; + defaultQueueKey: string; + existingQueues: string[]; + noAudio: boolean; +}) { + const [format, setFormat] = useState<AudioFormat>(defaultFormat); + const [extraArgsRaw, setExtraArgsRaw] = useState(""); + const [queueKey, setQueueKey] = useState(defaultQueueKey); + const desc = noAudio + ? handling === "youtube" + ? "YouTube channels normally only fetch auto-subs. Use this when the video has no auto-subs and you want to transcribe it manually." + : "Fetch the audio for this video using yt-dlp." + : "Re-fetch the audio for this video. The existing audio file is kept on disk; yt-dlp may overwrite it with the new one if names collide."; + const actionLabel = `${noAudio ? "Download" : "Redownload"} audio for ${videoId}`; + return ( + <section className="flex flex-col gap-3 border border-zinc-200 dark:border-zinc-800 rounded-md p-4"> + <Heading + title={noAudio ? "Download audio" : "Redownload audio"} + desc={desc} + /> + <div className="flex flex-col sm:flex-row sm:flex-wrap sm:items-center gap-3"> + <label className="flex items-center gap-2 text-sm"> + Format + <select + value={format} + onChange={(e) => setFormat(e.target.value as AudioFormat)} + aria-label="redownload audio format" + className="rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 px-2 py-1 text-sm" + > + {AUDIO_FORMAT_VALUES.map((f) => ( + <option key={f} value={f}> + {f} + </option> + ))} + </select> + </label> + <label className="flex flex-col sm:flex-row sm:items-center gap-2 text-sm flex-1 sm:min-w-[16rem]"> + <span>Extra yt-dlp args</span> + <input + type="text" + value={extraArgsRaw} + onChange={(e) => setExtraArgsRaw(e.target.value)} + aria-label="redownload extra yt-dlp args" + placeholder="--cookies-from-browser firefox --quiet" + className="flex-1 rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 px-2 py-1 text-sm font-mono min-w-0" + /> + </label> + </div> + <StreamActionLog + trigger={() => + redownloadVideoAction(slug, videoId, { + audioFormat: format, + extraArgs: parseExtraArgs(extraArgsRaw), + queueKey, + }) + } + cancelAction={cancelJobAction} + buttonLabel={`${noAudio ? "Download" : "Redownload"} audio (${format})`} + runningLabel={noAudio ? "Downloading…" : "Redownloading…"} + label={actionLabel} + extraControls={ + <QueueControl + value={queueKey} + onChange={setQueueKey} + defaultQueueKey={defaultQueueKey} + existingQueues={existingQueues} + actionLabel={actionLabel} + /> + } + /> + </section> + ); +} + +function WhisperVideoSection({ + slug, + videoId, + existingQueues, + noAudio, +}: { + slug: string; + videoId: string; + existingQueues: string[]; + noAudio: boolean; +}) { + // Run immediately in parallel by default — matches the per-audio-file + // Transcribe button's convention. + const [queueKey, setQueueKey] = useState(""); + const desc = noAudio + ? "Download the audio and run whisper-cli, in one job. Useful when a YouTube video has no auto-subs." + : "Run whisper-cli on this video's audio. Skips the download phase since audio is already on disk."; + const actionLabel = `Whisper transcribe ${videoId}`; + return ( + <section className="flex flex-col gap-3 border border-zinc-200 dark:border-zinc-800 rounded-md p-4"> + <Heading title="Whisper transcribe" desc={desc} /> + <StreamActionLog + trigger={() => whisperVideoAction(slug, videoId, queueKey)} + cancelAction={cancelJobAction} + buttonLabel="Whisper transcribe" + runningLabel="Transcribing…" + label={actionLabel} + extraControls={ + <QueueControl + value={queueKey} + onChange={setQueueKey} + defaultQueueKey="" + existingQueues={existingQueues} + actionLabel={actionLabel} + /> + } + /> + </section> + ); +} + +function AudioFileActions({ + slug, + videoId, + file, + existingQueues, + hasTranscript, +}: { + slug: string; + videoId: string; + file: VideoFile; + existingQueues: string[]; + hasTranscript: boolean; +}) { + const ext = audioExt(file.name); + const transcodeTargets = AUDIO_FORMAT_VALUES.filter((f) => f !== ext); + const [target, setTarget] = useState<AudioFormat>(transcodeTargets[0] ?? "opus"); + // Per-file Transcribe and Transcode default to "" — single-file jobs run + // immediately, in parallel with channel-wide batches. + const [transcribeQueue, setTranscribeQueue] = useState(""); + const [transcodeQueue, setTranscodeQueue] = useState(""); + + return ( + <div className="flex flex-col gap-3"> + <div className="flex flex-col gap-1"> + <Heading + title="Transcribe" + desc={ + hasTranscript + ? "A transcript already exists for this video. Delete transcript.json (or transcript.en.vtt) on disk to re-run." + : "Run whisper-cli on this audio file. Writes transcript.json next to it." + } + /> + {hasTranscript ? ( + <p className="text-sm text-zinc-500"> + Transcribe disabled — transcript already present. + </p> + ) : ( + <StreamActionLog + trigger={() => + transcribeOneAction(slug, videoId, file.name, transcribeQueue) + } + cancelAction={cancelJobAction} + buttonLabel={`Transcribe ${file.name}`} + runningLabel="Transcribing…" + label={`Transcribe ${file.name}`} + extraControls={ + <QueueControl + value={transcribeQueue} + onChange={setTranscribeQueue} + defaultQueueKey="" + existingQueues={existingQueues} + actionLabel={`Transcribe ${file.name}`} + /> + } + /> + )} + </div> + {transcodeTargets.length > 0 && ( + <div className="flex flex-col gap-1"> + <Heading + title="Transcode" + desc="Re-encode this audio file with ffmpeg. The source file is kept; the output is named audio.<format>." + /> + <label className="flex items-center gap-2 text-sm"> + Target format + <select + value={target} + onChange={(e) => setTarget(e.target.value as AudioFormat)} + aria-label={`transcode target for ${file.name}`} + className="rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 px-2 py-1 text-sm" + > + {transcodeTargets.map((t) => ( + <option key={t} value={t}> + {t} + </option> + ))} + </select> + </label> + <StreamActionLog + trigger={() => + transcodeAudioAction( + slug, + videoId, + file.name, + target, + transcodeQueue, + ) + } + cancelAction={cancelJobAction} + buttonLabel={`Transcode ${file.name} → ${target}`} + runningLabel="Transcoding…" + label={`Transcode ${file.name} to ${target}`} + extraControls={ + <QueueControl + value={transcodeQueue} + onChange={setTranscodeQueue} + defaultQueueKey="" + existingQueues={existingQueues} + actionLabel={`Transcode ${file.name} to ${target}`} + /> + } + /> + </div> + )} + </div> + ); +} + +function MarkUntranscribableSection({ + slug, + videoId, +}: { + slug: string; + videoId: string; +}) { + const [pending, startTransition] = useTransition(); + const [error, setError] = useState<string | null>(null); + return ( + <section className="flex flex-col gap-2 border border-zinc-200 dark:border-zinc-800 rounded-md p-4"> + <Heading + title="Mark untranscribable" + desc="Writes an empty transcript.json so the pipeline stops retrying. Removes this video from the failed list. Reversible — just delete transcript.json." + /> + <button + type="button" + disabled={pending} + aria-label={`mark video ${videoId} untranscribable`} + onClick={() => { + setError(null); + startTransition(async () => { + const res = await markVideoUntranscribableAction(slug, videoId); + if (!res.ok) setError(res.error); + }); + }} + className="self-start px-3 py-2 rounded-md bg-zinc-900 dark:bg-zinc-100 text-zinc-100 dark:text-zinc-900 text-sm font-medium hover:opacity-90 disabled:opacity-50" + > + {pending ? "Marking…" : "Mark untranscribable"} + </button> + {error && ( + <span + className="text-sm text-red-600 dark:text-red-400" + aria-label="mark untranscribable error" + > + {error} + </span> + )} + </section> + ); +} + +function DeleteFileButton({ + slug, + videoId, + filename, +}: { + slug: string; + videoId: string; + filename: string; +}) { + const [pending, startTransition] = useTransition(); + const [error, setError] = useState<string | null>(null); + return ( + <div className="flex items-center gap-2"> + <button + type="button" + disabled={pending} + aria-label={`delete file ${filename}`} + onClick={() => { + if (!confirm(`Delete ${filename}?`)) return; + setError(null); + startTransition(async () => { + const res = await deleteVideoFileAction(slug, videoId, filename); + if (!res.ok) setError(res.error); + }); + }} + className="self-start px-2 py-1 rounded border border-red-300 dark:border-red-800 text-red-700 dark:text-red-400 text-xs hover:bg-red-50 dark:hover:bg-red-950 disabled:opacity-50" + > + {pending ? "Deleting…" : "Delete file"} + </button> + {error && ( + <span + className="text-xs text-red-600 dark:text-red-400" + aria-label={`delete error ${filename}`} + > + {error} + </span> + )} + </div> + ); +} + +function DeleteVideoDirSection({ + slug, + videoId, +}: { + slug: string; + videoId: string; +}) { + const action = deleteVideoDirAction.bind(null, slug, videoId); + const [state, formAction] = useActionState<DeleteDirActionResult, FormData>( + action, + undefined, + ); + return ( + <section className="flex flex-col gap-3 border-t border-red-200 dark:border-red-900 pt-4"> + <Heading + title="Delete this video directory" + desc={`Removes data/${videoId}/ and all of its contents. Type the video id to confirm.`} + /> + <form + action={formAction} + aria-label={`delete video directory ${videoId}`} + className="flex flex-wrap items-start gap-2" + > + <input + name="confirmId" + required + placeholder={videoId} + aria-label="confirm video id" + className="rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 px-2 py-1 text-sm font-mono" + /> + <button + type="submit" + className="px-3 py-1.5 rounded-md bg-red-600 text-white text-sm font-medium hover:bg-red-700" + > + Delete directory + </button> + {state?.error && ( + <p + className="basis-full text-sm text-red-600 dark:text-red-400" + aria-label="delete directory error" + > + {state.error} + </p> + )} + </form> + </section> + ); +} + +function Heading({ title, desc }: { title: string; desc: string }) { + return ( + <div> + <h3 className="text-base font-semibold">{title}</h3> + <p className="text-sm text-zinc-500">{desc}</p> + </div> + ); +} diff --git a/editor/app/channels/[slug]/videos/[id]/page.tsx b/editor/app/channels/[slug]/videos/[id]/page.tsx @@ -11,7 +11,7 @@ import { platformQueueKey, } from "yt-dlp-transcript-common/lib/platform"; import { getRegistry } from "yt-dlp-transcript-common/jobs/registry"; -import { VideoPanel, type VideoFile } from "./_components/VideoPanel"; +import { VideoPanel, type VideoFile } from "./components/VideoPanel"; export const dynamic = "force-dynamic"; diff --git a/editor/app/channels/[slug]/whisperActions.ts b/editor/app/channels/[slug]/whisperActions.ts @@ -29,6 +29,7 @@ export async function transcribeMissingAction( slug: string, queueKey?: string, concurrency?: number, + reverse?: boolean, ): Promise<StreamActionResult> { const paths = getPaths(); const limit = sanitizeConcurrency(concurrency); @@ -43,6 +44,7 @@ export async function transcribeMissingAction( paths, mode: "all", concurrency: limit, + reverse: reverse === true, onLog, signal, }); diff --git a/editor/app/channels/actions.ts b/editor/app/channels/actions.ts @@ -11,7 +11,8 @@ import { readChannelConfig, writeChannelConfig, } from "yt-dlp-transcript-common/controller/channels"; -import { parseChannelForm } from "./_components/parseChannelForm"; +import { generateChannelSnapshot } from "yt-dlp-transcript-common/controller/channelSnapshot"; +import { parseChannelForm } from "./components/parseChannelForm"; export type ActionResult = { error: string } | undefined; @@ -78,6 +79,18 @@ export async function gotoVideoAction( redirect(`/channels/${slug}/videos/${encodeURIComponent(id)}`); } +export async function refreshChannelSnapshotAction( + slug: string, +): Promise<ActionResult> { + const paths = getPaths(); + if (!(await channelExists(paths, slug))) { + return { error: `Channel "${slug}" not found` }; + } + await generateChannelSnapshot(paths, slug); + revalidatePath(`/channels/${slug}`); + return undefined; +} + export async function deleteChannelAction( slug: string, _prev: ActionResult, diff --git a/editor/app/channels/_components/ChannelForm.tsx b/editor/app/channels/components/ChannelForm.tsx diff --git a/editor/app/channels/_components/ChannelFormClient.tsx b/editor/app/channels/components/ChannelFormClient.tsx diff --git a/editor/app/channels/components/ChannelSyncButton.tsx b/editor/app/channels/components/ChannelSyncButton.tsx @@ -0,0 +1,82 @@ +"use client"; + +import { useState } from "react"; +import { syncAction } from "../[slug]/pipelineActions"; + +type Status = + | { kind: "idle" } + | { kind: "running"; jobId?: string } + | { kind: "done"; jobId: string } + | { kind: "error"; message: string }; + +export function ChannelSyncButton({ + slug, + disabled, +}: { + slug: string; + disabled?: boolean; +}) { + const [status, setStatus] = useState<Status>({ kind: "idle" }); + + async function handleClick() { + setStatus({ kind: "running" }); + try { + // No queueKey override → server picks platformQueueKey(channel) by default, + // matching the channel-detail page's Sync button. + const result = await syncAction(slug); + if (!result.ok) { + setStatus({ kind: "error", message: result.error }); + return; + } + setStatus({ kind: "running", jobId: result.jobId }); + const reader = result.stream.getReader(); + while (true) { + const { done } = await reader.read(); + if (done) break; + } + setStatus({ kind: "done", jobId: result.jobId }); + } catch (e) { + setStatus({ kind: "error", message: (e as Error).message }); + } + } + + const running = status.kind === "running"; + return ( + <div className="flex items-center gap-2"> + <button + type="button" + onClick={handleClick} + disabled={running || disabled} + aria-label={`sync ${slug}`} + className="px-2 py-1 rounded-md bg-zinc-900 dark:bg-zinc-100 text-zinc-100 dark:text-zinc-900 text-xs font-medium hover:opacity-90 disabled:opacity-50" + > + {running ? "Syncing…" : "Sync"} + </button> + {status.kind === "error" && ( + <span + role="alert" + aria-label={`sync ${slug} error`} + className="text-xs text-red-700 dark:text-red-400" + > + {status.message} + </span> + )} + {status.kind === "running" && status.jobId && ( + <span + aria-label={`sync ${slug} job`} + className="text-xs text-zinc-500 font-mono" + > + job {status.jobId} + </span> + )} + {status.kind === "done" && ( + <span + aria-label={`sync ${slug} done`} + className="text-xs text-zinc-500" + > + done (job {status.jobId}) + </span> + )} + </div> + ); +} diff --git a/editor/app/channels/_components/DeleteChannelForm.tsx b/editor/app/channels/components/DeleteChannelForm.tsx diff --git a/editor/app/channels/_components/parseChannelForm.ts b/editor/app/channels/components/parseChannelForm.ts diff --git a/editor/app/channels/new/page.tsx b/editor/app/channels/new/page.tsx @@ -1,6 +1,6 @@ import type { Metadata } from "next"; import Link from "next/link"; -import { ChannelFormClient } from "../_components/ChannelFormClient"; +import { ChannelFormClient } from "../components/ChannelFormClient"; import { createChannelAction } from "../actions"; export const metadata: Metadata = { title: "New channel — Channels" }; diff --git a/editor/app/channels/page.tsx b/editor/app/channels/page.tsx @@ -2,6 +2,7 @@ import type { Metadata } from "next"; import Link from "next/link"; import { listChannels } from "yt-dlp-transcript-common/controller/channels"; import { getPaths } from "yt-dlp-transcript-common/lib/paths"; +import { ChannelSyncButton } from "./components/ChannelSyncButton"; export const dynamic = "force-dynamic"; @@ -36,6 +37,7 @@ export default async function ChannelsPage() { <Th className="text-right">Downloads</Th> <Th className="text-right">Playlist</Th> <Th className="whitespace-nowrap">Last sync</Th> + <Th className="whitespace-nowrap">Actions</Th> </tr> </thead> <tbody> @@ -77,6 +79,12 @@ export default async function ChannelsPage() { ? new Date(c.config.lastSyncedAt).toLocaleString() : ""} </Td> + <Td className="whitespace-nowrap"> + <ChannelSyncButton + slug={c.slug} + disabled={!c.config.url} + /> + </Td> </tr> ))} </tbody> diff --git a/editor/app/_components/ConcurrencyControl.tsx b/editor/app/components/ConcurrencyControl.tsx diff --git a/editor/app/_components/QueueControl.tsx b/editor/app/components/QueueControl.tsx diff --git a/editor/app/jobs/[id]/_components/JobLogTail.tsx b/editor/app/jobs/[id]/components/JobLogTail.tsx diff --git a/editor/app/jobs/[id]/page.tsx b/editor/app/jobs/[id]/page.tsx @@ -4,7 +4,7 @@ import path from "node:path"; import { notFound } from "next/navigation"; import { listAllJobs } from "yt-dlp-transcript-common/jobs/listJobs"; import { getPaths } from "yt-dlp-transcript-common/lib/paths"; -import { JobLogTail } from "./_components/JobLogTail"; +import { JobLogTail } from "./components/JobLogTail"; export const dynamic = "force-dynamic"; diff --git a/editor/app/jobs/_components/CancelJobButton.tsx b/editor/app/jobs/components/CancelJobButton.tsx diff --git a/editor/app/jobs/_components/ClearArchivedButton.tsx b/editor/app/jobs/components/ClearArchivedButton.tsx diff --git a/editor/app/jobs/_components/JobsAutoRefresh.tsx b/editor/app/jobs/components/JobsAutoRefresh.tsx diff --git a/editor/app/jobs/page.tsx b/editor/app/jobs/page.tsx @@ -2,9 +2,9 @@ import type { Metadata } from "next"; import Link from "next/link"; import { listAllJobs } from "yt-dlp-transcript-common/jobs/listJobs"; import { getPaths } from "yt-dlp-transcript-common/lib/paths"; -import { ClearArchivedButton } from "./_components/ClearArchivedButton"; -import { CancelJobButton } from "./_components/CancelJobButton"; -import { JobsAutoRefresh } from "./_components/JobsAutoRefresh"; +import { ClearArchivedButton } from "./components/ClearArchivedButton"; +import { CancelJobButton } from "./components/CancelJobButton"; +import { JobsAutoRefresh } from "./components/JobsAutoRefresh"; export const dynamic = "force-dynamic"; diff --git a/editor/app/settings/_components/SettingsForm.tsx b/editor/app/settings/components/SettingsForm.tsx diff --git a/editor/app/settings/page.tsx b/editor/app/settings/page.tsx @@ -1,7 +1,7 @@ import type { Metadata } from "next"; import { getPaths } from "yt-dlp-transcript-common/lib/paths"; import { getSettings } from "yt-dlp-transcript-common/lib/settings"; -import { SettingsForm } from "./_components/SettingsForm"; +import { SettingsForm } from "./components/SettingsForm"; export const dynamic = "force-dynamic"; diff --git a/editor/e2e/pipeline.spec.ts b/editor/e2e/pipeline.spec.ts @@ -128,6 +128,21 @@ test("download videos skips a transcribed video even when its audio was cleaned expect(await pathExists(`${channelRoot}/data/vidA/audio.m4a`)).toBe(false); }); +test("channel list sync button runs the platform-queue sync", async ({ + page, +}) => { + await resetData("test-pipeline"); + await page.goto("/channels"); + await page.getByLabel("sync test-pipeline").click(); + await expect(page.getByLabel("sync test-pipeline done")).toBeVisible({ + timeout: 30_000, + }); + const config = await readJson<{ lastSyncedAt?: string }>( + "test-transcripts/channels/test-pipeline/config.json", + ); + expect(typeof config.lastSyncedAt).toBe("string"); +}); + test("blocks pipeline buttons when channel has no url", async ({ page }) => { await resetData("empty"); await page.goto("/channels/new");