commit 4057f76683f5508b6703374e6204973d28b176e2
parent eea11c4af0014e37375265bcff44ce0254833696
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sun, 26 Apr 2026 21:57:46 -0400
feat(editor): jobs viewer, settings UI, dashboard polish
Closes the loop on the editor: every long-running operation gets a
viewable history with log tailing, and site settings can be edited from
the UI without touching settings.json by hand.
Jobs:
- common/jobs/listJobs.ts: merges in-memory registry with leftover
<jobsDir>/*.log files (status: "archived") so jobs that ran in a
previous server lifetime still show up. clearArchivedLogs deletes
files for jobs not currently in the registry.
- editor/app/jobs/page.tsx: table of jobs with status badge, kind,
start/duration, log size, sorted newest-first. "Clear archived
logs" button.
- editor/app/jobs/[id]/page.tsx + JobLogTail.tsx: detail page that
reads the .log file via /api/jobs/[id]/log?from=<bytes>, polls every
600ms while the job is running, shows current status alongside the
pre.
- /api/jobs/[id]/log: byte-offset polling endpoint.
- editor/app/jobs/actions.ts: cancelJobAction (calls registry.cancel)
and clearArchivedAction.
Settings:
- common/lib/paths.ts: adds settingsFile, overridable via SETTINGS_FILE
(used by dev:test).
- common/lib/settings.ts: writeSettings(next) persists merged settings
through getPaths().settingsFile and busts the cache.
- editor/app/settings/page.tsx: new SettingsForm above the existing
read-only paths table. Form covers siteTitle, headerTitle,
siteDescription, homeTagline, maxTranscriptPageBytes (validated
against the existing min/max).
Dashboard:
- editor/app/page.tsx: 4 stat tiles (channels, videos, last build,
running jobs → links to /jobs) plus a per-channel table showing
handling, video count, last sync.
Test harness:
- /api/test/invalidate-cache also clears globalThis.__yttJobRegistry__
so each spec starts with a clean registry.
- editor/cypress/e2e/jobs.cy.ts (2 specs): empty state after clear,
full run-through of build-index → /jobs row → /jobs/<id> log tail.
- editor/cypress/e2e/settings.cy.ts (3 specs): paths visible, save
round-trips through SETTINGS_FILE, range validation rejects 42 B.
- dashboard.cy.ts: scope nav-link assertions to the sidebar so the new
/jobs stat-link doesn't clash.
README: rewritten for the monorepo layout with env-var table, pipeline
mode descriptions, and CLI-shim entry points.
Verified: 28/28 cypress specs pass; pnpm --filter export run build still
produces the existing static output.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Diffstat:
19 files changed, 898 insertions(+), 113 deletions(-)
diff --git a/README.md b/README.md
@@ -1,83 +1,70 @@
-# Jeralyzer
+# yt-dlp transcript browser
-This is [Next.js](https://nextjs.org) vibe-coded slop built to make it easy to browse auto-generated YouTube transcripts downloaded with yt-dlp.
+A pnpm-workspace monorepo for archiving and browsing video transcripts. The project is split into three packages:
-There's basic white-labeling functionality with the `settings.json` file if you feel like using this code for another purpose.
+- **`common/`** — shared library (data layer, controllers, components, types). Consumed by the other two via `workspace:*`.
+- **`editor/`** — dynamic Next.js app on port 3001 with admin UIs for channels, the yt-dlp pipeline, the whisper queue, and the build trigger.
+- **`export/`** — static Next.js app (`output: "export"`) that produces the read-only public site.
-## Adding Transcripts
+Transcripts and per-channel state live at `<repo>/transcripts/` (its own git repo, untouched by the workspace).
-This repo hosts the code for the app, and transcripts are to be inserted in the "transcripts" directory at the top level (alongside this README).
-
-The file structure looks like this:
-
-```
-.
-├── README.md
-├── transcripts
-│ └── data
-│ ├── 20180627_wCLdlbEHmLE-Introducing_The_Incredible_Salt_Mine
-│ │ ├── metadata.info.json
-│ │ └── transcript.en.vtt
-│ ├── 20180714_P1Lfy0rStkI-The_Incredible_Salt_Mine_Episode_001
-│ │ ├── metadata.info.json
-│ │ └── transcript.en.vtt
-```
-
-The directory names could be anything, but are prefaced by date for easy lazy date sorting when reading the dir.
-The `metadata.info.json` and `transcript.en.vtt` filenames are hard-coded for simplicity's sake.
-
-## Getting Started
-
-Make sure you have `pnpm` installed. Run these commands to build the static site and locally host it.
+## Getting started
```bash
-pnpm run build
-pnpm run start
+pnpm install
+pnpm dev:editor # editor at http://localhost:3001
+pnpm build # build the static site under export/out/
+pnpm start:export # serve export/out/ at http://localhost:3000
+pnpm e2e # run the cypress suite (editor)
```
-Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+## Editor UI
-## Development
+`pnpm dev:editor` (or `pnpm --filter editor run dev`) starts the editor on port 3001:
-If you want to change the code, use the development server for hot-reloading and error messages.
+- **/channels** — list, create, edit, delete channels. The form covers handling, url, audioFormat, cookiesFromBrowser, and ytdlpExtraArgs.
+- **/channels/<slug>** — pipeline panel (Store playlist / Download from playlist / Sync) and, for `transcribe` channels, a whisper panel (Transcribe missing / Retry failures / Verify).
+- **/build** — Build index (in-process) or Build static export (spawns `pnpm run build` in `export/`). Both stream logs live.
+- **/jobs** — recent jobs, running and archived. Click an id to tail its log.
+- **/settings** — site-settings form persisted to `settings.json`, plus a read-only view of `getPaths()` for env-resolution debugging.
-```bash
-pnpm run dev
-```
+## Environment variables
-## Preprocessed Transcripts in `public/`
+All paths and binaries used by the editor and CLI shims resolve through `getPaths()`. Override any of them before launching:
-`scripts/build-index.ts` (run automatically via the `prebuild` hook, or manually with `pnpm build:index`) reads everything under `transcripts/data/`, caches parsed results in an LMDB store at `transcripts/index.mdb`, and emits static JSON into `public/`:
+| Variable | Default | Purpose |
+| --- | --- | --- |
+| `TRANSCRIPTS_DIR` | `<repo>/transcripts` | Channels, archives, LMDB cache, job logs. |
+| `EXPORT_PUBLIC_DIR` | `<repo>/export/public` | Where the index writes paginated JSON. |
+| `SETTINGS_FILE` | `<repo>/settings.json` | Site-settings file. |
+| `YTDLP_BIN` | `yt-dlp` (PATH lookup) | Binary used by the pipeline. |
+| `WHISPER_BIN` | `whisper-cli` (PATH lookup) | Binary used by the whisper batch. |
+| `WHISPER_MODEL` | `~/whispercpp/whisper.cpp/models/ggml-base.en.bin` | Model file passed to whisper-cli. |
+| `PARALLEL_TRANSCRIBE_LIMIT` | `4` | Max parallel whisper jobs. |
-- `public/summaries/manifest.json` + `public/summaries/page-NNNN.json` — paginated list of video summaries.
-- `public/transcripts/<slug>.json` — per-video cues and metadata.
+## Pipeline modes
-The app fetches these files at runtime as plain static assets, which keeps `next build` fast even with 10k+ videos — Next.js never has to parse VTT files or walk the transcripts directory during `next build` itself. Incremental reruns of the index script short-circuit via mtime checks, so a rerun after adding a handful of new videos only processes the new ones.
+The editor's pipeline panel and `common/ytdlp/runYtdlp.ts` support three modes per channel:
-The trade-off: **Next.js does not watch `public/` for changes**, so adding or editing transcripts while `pnpm run dev` is running won't trigger a hot-reload. Re-run `pnpm build:index` and refresh the browser (or restart the dev server) to pick up new transcripts.
+- **Store playlist** — `yt-dlp --flat-playlist --skip-download --print url <url>`. Atomically writes `<channel>/playlist`.
+- **Download from playlist** — reads `<channel>/playlist`, parses `<channel>/archive`, writes `<channel>/playlist.tofetch` containing only URLs whose ID is *not* already archived (this app-side prefilter avoids forcing yt-dlp to re-fetch metadata for known entries on platforms like Odysee), then runs yt-dlp with `--download-archive ./archive --break-on-existing -a playlist.tofetch` plus per-handling args.
+- **Sync** — `yt-dlp --lazy-playlist --download-archive ./archive --break-on-existing <url>`. Quick incremental fetch.
-## Downloading Transcripts
+YouTube channels (`handling: "youtube"`) use `--write-auto-subs --skip-download`. Transcribe channels (`handling: "transcribe"`) download audio only (`-f bestaudio -x --audio-format <m4a|mp3|opus>`); whisper-cpp transcribes them later via the whisper panel.
-Use this command to download transcripts that end up in the expected format:
+## CLI shims
-```
-yt-dlp --ignore-config --skip-download --restrict-filenames --write-info-json -o "subtitle:%(upload_date)s_%(id)s-%(fulltitle,title)s/transcript" -o "infojson:%(upload_date)s_%(id)s-%(fulltitle,title)s/metadata" --write-auto-subs -- <VIDEO_OR_PLAYLIST>
-```
+The same controllers used by the editor are exposed as terminal shims under `common/bin/`:
-## Downloading Transcripts for a Whole Channel
-
-yt-dlp is far from bulletproof, and trying to scrape a channel with thousands of videos can be flaky. This multi-step process makes it more achievable:
+```bash
+pnpm --filter yt-dlp-transcript-common exec tsx bin/build-index.ts
+pnpm --filter yt-dlp-transcript-common exec tsx bin/transform.ts --channel <slug>
+pnpm --filter yt-dlp-transcript-common exec tsx bin/retry-failures.ts --channel <slug>
+pnpm --filter yt-dlp-transcript-common exec tsx bin/verify-transcripts.ts --channel <slug>
+```
-1. **Download a playlist (including a channel's "Videos" or "Streams" page) as a list of URLs**
- `yt-dlp --flat-playlist --skip-download --print url <CHANNEL_OR_PLAYLIST_URL> > ../playlist.txt`
+## Static-site export
-2. **Start downloading from the saved playlist, writing completed downloads to an archive**
- `yt-dlp --ignore-config --skip-download --restrict-filenames -fb --write-info-json -o "%(upload_date)s_%(id)s/video" -o "subtitle:%(upload_date)s_%(id)s/transcript" -o "infojson:%(upload_date)s_%(id)s/metadata" --write-auto-subs --force-write-archive --download-archive ../archive.txt --abort-on-error -t sleep -a ../playlist.txt`
- - `--force-write-archive --download-archive ../archive.txt` is the main point — yt-dlp will locally track completed videos in a text file and completely skip the download if it has already been done. This makes resuming the download for even a massive playlist much easier.
- - `-a ../playlist.txt` is how you load a text file line-by-line; just passing the path to the text file will not work.
- - `-t sleep` is a preset that defines multiple small wait periods to prevent rate-limiting.
- - `--abort-on-error` is nice to not keep churning while rate-limited, but could be omitted for more hands-off progress.
- - `--cookies-from-browser` can be used to download age-restricted videos, though you should only use it when needed to minimize risk to your YouTube account.
+The export app reads from the LMDB-backed paginated JSON in `export/public/{summaries,transcripts}/` and renders a search/filter UI as a static `out/` directory. It's the read-only public face of the data.
-3. **Verify subtitles are present for each video in the archive**
- Rarely, a video's subtitle can be missed in error and the download still marked as complete.
+The export build uses `transpilePackages: ["yt-dlp-transcript-common"]` so editing common is hot-reloadable without a separate build step. `lmdb`, `msgpackr`, and `msgpackr-extract` are listed in `serverExternalPackages` so Next.js doesn't try to bundle them.
diff --git a/common/jobs/listJobs.ts b/common/jobs/listJobs.ts
@@ -0,0 +1,128 @@
+import path from "node:path";
+import { readdir, stat, readFile, rm } from "node:fs/promises";
+import type { Paths } from "../lib/paths";
+import { getRegistry, type JobRecord, type JobStatus } from "./registry";
+
+export type JobListEntry = {
+ id: string;
+ kind?: string;
+ status: JobStatus | "archived";
+ startedAt: number;
+ endedAt?: number;
+ exitCode?: number;
+ inRegistry: boolean;
+ logPath: string;
+ logSize: number;
+};
+
+// Merge in-memory registry entries with any leftover .log files from
+// previous server lifetimes. Anything not in the registry is treated as
+// "archived" — we only know that it once ran, not whether it succeeded.
+export async function listAllJobs(paths: Paths): Promise<JobListEntry[]> {
+ const registry = getRegistry();
+ const live: Map<string, JobRecord> = new Map();
+ for (const r of registry.list()) live.set(r.id, r);
+
+ let files: string[] = [];
+ try {
+ files = (await readdir(paths.jobsDir)).filter((n) => n.endsWith(".log"));
+ } catch {
+ files = [];
+ }
+
+ const out: JobListEntry[] = [];
+ const seen = new Set<string>();
+
+ for (const file of files) {
+ const id = file.replace(/\.log$/, "");
+ seen.add(id);
+ const logPath = path.join(paths.jobsDir, file);
+ const live_ = live.get(id);
+ let logSize = 0;
+ let mtime = 0;
+ try {
+ const s = await stat(logPath);
+ logSize = s.size;
+ mtime = s.mtimeMs;
+ } catch {
+ continue;
+ }
+ if (live_) {
+ out.push({
+ id,
+ kind: live_.kind,
+ status: live_.status,
+ startedAt: live_.startedAt,
+ endedAt: live_.endedAt,
+ exitCode: live_.exitCode,
+ inRegistry: true,
+ logPath,
+ logSize,
+ });
+ } else {
+ out.push({
+ id,
+ status: "archived",
+ startedAt: mtime,
+ endedAt: mtime,
+ inRegistry: false,
+ logPath,
+ logSize,
+ });
+ }
+ }
+
+ // Registry entries with no log file (shouldn't happen normally, but be
+ // defensive).
+ for (const r of live.values()) {
+ if (!seen.has(r.id)) {
+ out.push({
+ id: r.id,
+ kind: r.kind,
+ status: r.status,
+ startedAt: r.startedAt,
+ endedAt: r.endedAt,
+ exitCode: r.exitCode,
+ inRegistry: true,
+ logPath: r.logPath,
+ logSize: 0,
+ });
+ }
+ }
+
+ return out.sort((a, b) => b.startedAt - a.startedAt);
+}
+
+export async function readLogChunk(
+ logPath: string,
+ fromBytes: number,
+): Promise<{ content: string; nextOffset: number }> {
+ let raw: string;
+ try {
+ raw = await readFile(logPath, "utf8");
+ } catch {
+ return { content: "", nextOffset: fromBytes };
+ }
+ if (fromBytes >= raw.length) {
+ return { content: "", nextOffset: raw.length };
+ }
+ return { content: raw.slice(fromBytes), nextOffset: raw.length };
+}
+
+export async function clearArchivedLogs(paths: Paths): Promise<number> {
+ const live = new Set(getRegistry().list().map((r) => r.id));
+ let files: string[];
+ try {
+ files = (await readdir(paths.jobsDir)).filter((n) => n.endsWith(".log"));
+ } catch {
+ return 0;
+ }
+ let deleted = 0;
+ for (const file of files) {
+ const id = file.replace(/\.log$/, "");
+ if (live.has(id)) continue;
+ await rm(path.join(paths.jobsDir, file), { force: true });
+ deleted++;
+ }
+ return deleted;
+}
diff --git a/common/lib/paths.ts b/common/lib/paths.ts
@@ -12,6 +12,7 @@ export type Paths = {
exportPublicDir: string;
exportSummariesDir: string;
exportTranscriptsDir: string;
+ settingsFile: string;
ytdlpBin: string;
whisperBin: string;
whisperModel: string;
@@ -38,6 +39,8 @@ export function getPaths(): Paths {
exportPublicDir,
exportSummariesDir: path.join(exportPublicDir, "summaries"),
exportTranscriptsDir: path.join(exportPublicDir, "transcripts"),
+ settingsFile:
+ process.env.SETTINGS_FILE ?? path.join(monorepoRoot, "settings.json"),
ytdlpBin: process.env.YTDLP_BIN ?? "yt-dlp",
whisperBin: process.env.WHISPER_BIN ?? "whisper-cli",
whisperModel:
diff --git a/common/lib/settings.ts b/common/lib/settings.ts
@@ -1,5 +1,4 @@
import fs from "node:fs";
-import path from "node:path";
import { getPaths } from "./paths";
export type SiteSettings = {
@@ -26,7 +25,7 @@ let cached: SiteSettings | null = null;
export function getSettings(): SiteSettings {
if (cached) return cached;
- const file = path.join(getPaths().monorepoRoot, "settings.json");
+ const file = getPaths().settingsFile;
let parsed: Partial<SiteSettings> = {};
try {
parsed = JSON.parse(fs.readFileSync(file, "utf8")) as Partial<SiteSettings>;
@@ -48,3 +47,15 @@ function clampPageBytes(value: unknown): number {
if (n > TRANSCRIPT_PAGE_HARD_CAP_BYTES) return TRANSCRIPT_PAGE_HARD_CAP_BYTES;
return Math.floor(n);
}
+
+export async function writeSettings(next: SiteSettings): Promise<void> {
+ const file = getPaths().settingsFile;
+ const merged: SiteSettings = {
+ ...next,
+ maxTranscriptPageBytes: clampPageBytes(next.maxTranscriptPageBytes),
+ };
+ const tmp = `${file}.tmp-${process.pid}`;
+ await fs.promises.writeFile(tmp, JSON.stringify(merged, null, 2) + "\n");
+ await fs.promises.rename(tmp, file);
+ cached = null;
+}
diff --git a/editor/app/api/jobs/[id]/log/route.ts b/editor/app/api/jobs/[id]/log/route.ts
@@ -0,0 +1,28 @@
+import path from "node:path";
+import { NextResponse } from "next/server";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { readLogChunk } from "yt-dlp-transcript-common/jobs/listJobs";
+import { getRegistry } from "yt-dlp-transcript-common/jobs/registry";
+
+export const dynamic = "force-dynamic";
+
+export async function GET(
+ request: Request,
+ { params }: { params: Promise<{ id: string }> },
+) {
+ const { id } = await params;
+ if (!/^[A-Za-z0-9_-]+$/.test(id)) {
+ return NextResponse.json({ error: "Invalid id" }, { status: 400 });
+ }
+ const url = new URL(request.url);
+ const fromRaw = url.searchParams.get("from");
+ const from = fromRaw ? Math.max(0, Number.parseInt(fromRaw, 10) || 0) : 0;
+
+ const paths = getPaths();
+ const logPath = path.join(paths.jobsDir, `${id}.log`);
+ const { content, nextOffset } = await readLogChunk(logPath, from);
+
+ const record = getRegistry().get(id);
+ const status = record ? record.status : "archived";
+ return NextResponse.json({ content, nextOffset, status });
+}
diff --git a/editor/app/api/test/invalidate-cache/route.ts b/editor/app/api/test/invalidate-cache/route.ts
@@ -14,6 +14,10 @@ export async function GET() {
}
function invalidate() {
+ // Reset the in-memory job registry too so tests don't observe stale jobs
+ // from a prior spec in the same dev-server lifetime.
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ (globalThis as any).__yttJobRegistry__ = undefined;
revalidatePath("/", "layout");
return NextResponse.json({ ok: true });
}
diff --git a/editor/app/jobs/[id]/_components/JobLogTail.tsx b/editor/app/jobs/[id]/_components/JobLogTail.tsx
@@ -0,0 +1,76 @@
+"use client";
+
+import { useEffect, useRef, useState } from "react";
+
+type Props = {
+ jobId: string;
+ initiallyRunning: boolean;
+ testId?: string;
+};
+
+type LogResponse = {
+ content: string;
+ nextOffset: number;
+ status: string;
+};
+
+export function JobLogTail({ jobId, initiallyRunning, testId }: Props) {
+ const [log, setLog] = useState("");
+ const [status, setStatus] = useState<string>(
+ initiallyRunning ? "running" : "loaded",
+ );
+ const offsetRef = useRef(0);
+ const preRef = useRef<HTMLPreElement | null>(null);
+
+ useEffect(() => {
+ if (preRef.current) preRef.current.scrollTop = preRef.current.scrollHeight;
+ }, [log]);
+
+ useEffect(() => {
+ let cancelled = false;
+ let timer: ReturnType<typeof setTimeout> | null = null;
+
+ async function tick() {
+ try {
+ const res = await fetch(
+ `/api/jobs/${encodeURIComponent(jobId)}/log?from=${offsetRef.current}`,
+ { cache: "no-store" },
+ );
+ if (!res.ok) return;
+ const data = (await res.json()) as LogResponse;
+ if (cancelled) return;
+ if (data.content) setLog((prev) => prev + data.content);
+ offsetRef.current = data.nextOffset;
+ setStatus(data.status);
+ if (data.status === "running") {
+ timer = setTimeout(tick, 600);
+ }
+ } catch {
+ if (!cancelled) timer = setTimeout(tick, 1500);
+ }
+ }
+ tick();
+ return () => {
+ cancelled = true;
+ if (timer) clearTimeout(timer);
+ };
+ }, [jobId]);
+
+ return (
+ <div className="flex flex-col gap-2" data-testid={testId}>
+ <div className="text-xs text-zinc-500">
+ Status:{" "}
+ <span data-testid={testId ? `${testId}-status` : undefined}>
+ {status}
+ </span>
+ </div>
+ <pre
+ ref={preRef}
+ data-testid={testId ? `${testId}-pre` : undefined}
+ className="text-xs font-mono bg-zinc-100 dark:bg-zinc-900 border border-zinc-200 dark:border-zinc-800 rounded p-3 max-h-[28rem] overflow-auto whitespace-pre-wrap"
+ >
+ {log || "(empty log)"}
+ </pre>
+ </div>
+ );
+}
diff --git a/editor/app/jobs/[id]/page.tsx b/editor/app/jobs/[id]/page.tsx
@@ -0,0 +1,65 @@
+import Link from "next/link";
+import path from "node:path";
+import { notFound } from "next/navigation";
+import { listAllJobs } from "yt-dlp-transcript-common/jobs/listJobs";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { JobLogTail } from "./_components/JobLogTail";
+
+export const dynamic = "force-dynamic";
+
+export default async function JobDetailPage({
+ params,
+}: {
+ params: Promise<{ id: string }>;
+}) {
+ const { id } = await params;
+ const jobs = await listAllJobs(getPaths());
+ const job = jobs.find((j) => j.id === id);
+ if (!job) notFound();
+ return (
+ <div className="flex flex-col gap-4">
+ <div className="flex items-center gap-2 text-sm text-zinc-500">
+ <Link
+ href="/jobs"
+ className="underline hover:text-zinc-900 dark:hover:text-zinc-100"
+ >
+ Jobs
+ </Link>
+ <span>/</span>
+ <span className="font-mono">{id}</span>
+ </div>
+ <h1 className="text-2xl font-semibold">{job.kind ?? "job"}</h1>
+ <dl className="text-sm grid grid-cols-2 sm:grid-cols-4 gap-3">
+ <Cell label="Status" value={job.status} />
+ <Cell label="Started" value={new Date(job.startedAt).toLocaleString()} />
+ <Cell
+ label="Ended"
+ value={job.endedAt ? new Date(job.endedAt).toLocaleString() : "—"}
+ />
+ <Cell
+ label="Exit code"
+ value={typeof job.exitCode === "number" ? String(job.exitCode) : "—"}
+ />
+ </dl>
+ <p className="text-xs text-zinc-500 font-mono">
+ log: {path.relative(getPaths().monorepoRoot, job.logPath)}
+ </p>
+ <JobLogTail
+ jobId={id}
+ initiallyRunning={job.status === "running"}
+ testId="job-log"
+ />
+ </div>
+ );
+}
+
+function Cell({ label, value }: { label: string; value: string }) {
+ return (
+ <div className="rounded border border-zinc-200 dark:border-zinc-800 px-3 py-2">
+ <div className="text-xs uppercase tracking-wide text-zinc-500">
+ {label}
+ </div>
+ <div className="mt-1">{value}</div>
+ </div>
+ );
+}
diff --git a/editor/app/jobs/_components/ClearArchivedButton.tsx b/editor/app/jobs/_components/ClearArchivedButton.tsx
@@ -0,0 +1,29 @@
+"use client";
+
+import { useState } from "react";
+import { useRouter } from "next/navigation";
+import { clearArchivedAction } from "../actions";
+
+export function ClearArchivedButton() {
+ const [busy, setBusy] = useState(false);
+ const router = useRouter();
+ return (
+ <button
+ type="button"
+ data-testid="jobs-clear-archived"
+ onClick={async () => {
+ setBusy(true);
+ try {
+ await clearArchivedAction();
+ router.refresh();
+ } finally {
+ setBusy(false);
+ }
+ }}
+ disabled={busy}
+ className="px-3 py-2 rounded-md border border-zinc-300 dark:border-zinc-700 text-sm hover:bg-zinc-100 dark:hover:bg-zinc-800 disabled:opacity-50"
+ >
+ {busy ? "Clearing…" : "Clear archived logs"}
+ </button>
+ );
+}
diff --git a/editor/app/jobs/actions.ts b/editor/app/jobs/actions.ts
@@ -0,0 +1,18 @@
+"use server";
+
+import { revalidatePath } from "next/cache";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { clearArchivedLogs } from "yt-dlp-transcript-common/jobs/listJobs";
+import { getRegistry } from "yt-dlp-transcript-common/jobs/registry";
+
+export async function cancelJobAction(id: string): Promise<{ ok: boolean }> {
+ const ok = getRegistry().cancel(id);
+ revalidatePath("/jobs");
+ return { ok };
+}
+
+export async function clearArchivedAction(): Promise<{ deleted: number }> {
+ const deleted = await clearArchivedLogs(getPaths());
+ revalidatePath("/jobs");
+ return { deleted };
+}
diff --git a/editor/app/jobs/page.tsx b/editor/app/jobs/page.tsx
@@ -1,10 +1,105 @@
-export default function JobsPage() {
+import Link from "next/link";
+import { listAllJobs } from "yt-dlp-transcript-common/jobs/listJobs";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { ClearArchivedButton } from "./_components/ClearArchivedButton";
+
+export const dynamic = "force-dynamic";
+
+function fmtDuration(ms: number): string {
+ if (ms < 1000) return `${ms}ms`;
+ const s = ms / 1000;
+ if (s < 60) return `${s.toFixed(1)}s`;
+ const m = s / 60;
+ return `${m.toFixed(1)}m`;
+}
+
+function statusColor(status: string): string {
+ switch (status) {
+ case "running":
+ return "bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-200";
+ case "done":
+ return "bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-200";
+ case "failed":
+ return "bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-200";
+ case "cancelled":
+ return "bg-amber-100 text-amber-800 dark:bg-amber-900 dark:text-amber-200";
+ default:
+ return "bg-zinc-100 text-zinc-700 dark:bg-zinc-800 dark:text-zinc-300";
+ }
+}
+
+export default async function JobsPage() {
+ const jobs = await listAllJobs(getPaths());
return (
<div className="flex flex-col gap-4">
- <h1 className="text-2xl font-semibold">Jobs</h1>
- <p className="text-sm text-zinc-500">
- Job log viewer lands in Phase 8.
- </p>
+ <div className="flex items-center justify-between">
+ <h1 className="text-2xl font-semibold">Jobs</h1>
+ <ClearArchivedButton />
+ </div>
+ {jobs.length === 0 ? (
+ <p
+ data-testid="jobs-empty"
+ className="text-sm text-zinc-500 border border-dashed border-zinc-300 dark:border-zinc-700 rounded p-4"
+ >
+ No jobs have run yet.
+ </p>
+ ) : (
+ <table
+ data-testid="jobs-table"
+ className="text-sm border border-zinc-200 dark:border-zinc-800 rounded-md overflow-hidden"
+ >
+ <thead className="bg-zinc-100 dark:bg-zinc-900">
+ <tr>
+ <th className="text-left font-medium px-3 py-2">ID</th>
+ <th className="text-left font-medium px-3 py-2">Kind</th>
+ <th className="text-left font-medium px-3 py-2">Status</th>
+ <th className="text-left font-medium px-3 py-2">Started</th>
+ <th className="text-left font-medium px-3 py-2">Duration</th>
+ <th className="text-right font-medium px-3 py-2">Log size</th>
+ </tr>
+ </thead>
+ <tbody>
+ {jobs.map((j) => {
+ const dur = j.endedAt
+ ? j.endedAt - j.startedAt
+ : Date.now() - j.startedAt;
+ return (
+ <tr
+ key={j.id}
+ data-testid={`job-row-${j.id}`}
+ className="border-t border-zinc-200 dark:border-zinc-800"
+ >
+ <td className="px-3 py-2 font-mono text-xs">
+ <Link
+ href={`/jobs/${j.id}`}
+ className="underline hover:text-zinc-900 dark:hover:text-zinc-100"
+ >
+ {j.id}
+ </Link>
+ </td>
+ <td className="px-3 py-2">{j.kind ?? "—"}</td>
+ <td className="px-3 py-2">
+ <span
+ className={`text-xs uppercase tracking-wide px-2 py-0.5 rounded ${statusColor(j.status)}`}
+ >
+ {j.status}
+ </span>
+ </td>
+ <td className="px-3 py-2 text-xs text-zinc-500">
+ {new Date(j.startedAt).toLocaleString()}
+ </td>
+ <td className="px-3 py-2 text-xs text-zinc-500">
+ {fmtDuration(dur)}
+ </td>
+ <td className="px-3 py-2 text-xs text-zinc-500 text-right">
+ {j.logSize.toLocaleString()} B
+ </td>
+ </tr>
+ );
+ })}
+ </tbody>
+ </table>
+ )}
</div>
);
}
diff --git a/editor/app/page.tsx b/editor/app/page.tsx
@@ -1,7 +1,11 @@
import { stat } from "node:fs/promises";
import path from "node:path";
+import Link from "next/link";
import { listChannels } from "yt-dlp-transcript-common/controller/channels";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { getRegistry } from "yt-dlp-transcript-common/jobs/registry";
+
+export const dynamic = "force-dynamic";
async function getLastBuildTime(paths: ReturnType<typeof getPaths>) {
try {
@@ -21,21 +25,69 @@ export default async function Dashboard() {
const channels = await listChannels(paths);
const totalVideos = channels.reduce((sum, c) => sum + c.videoCount, 0);
const lastBuild = await getLastBuildTime(paths);
+ const runningJobs = getRegistry()
+ .list()
+ .filter((j) => j.status === "running");
return (
<div className="flex flex-col gap-6">
<h1 className="text-2xl font-semibold">Dashboard</h1>
- <dl className="grid grid-cols-1 sm:grid-cols-3 gap-4">
+ <dl className="grid grid-cols-1 sm:grid-cols-4 gap-4">
<Stat label="channels" value={String(channels.length)} />
<Stat label="videos" value={totalVideos.toLocaleString()} />
<Stat
label="last build"
value={lastBuild ? new Date(lastBuild).toLocaleString() : "never"}
/>
+ <StatLink
+ label="running jobs"
+ value={String(runningJobs.length)}
+ href="/jobs"
+ />
</dl>
+
+ {channels.length > 0 && (
+ <section className="flex flex-col gap-3">
+ <h2 className="text-lg font-semibold">Channels</h2>
+ <table className="text-sm border border-zinc-200 dark:border-zinc-800 rounded-md overflow-hidden">
+ <thead className="bg-zinc-100 dark:bg-zinc-900">
+ <tr>
+ <th className="text-left font-medium px-3 py-2">Slug</th>
+ <th className="text-left font-medium px-3 py-2">Handling</th>
+ <th className="text-right font-medium px-3 py-2">Videos</th>
+ <th className="text-left font-medium px-3 py-2">Last sync</th>
+ </tr>
+ </thead>
+ <tbody>
+ {channels.map((c) => (
+ <tr
+ key={c.slug}
+ className="border-t border-zinc-200 dark:border-zinc-800"
+ >
+ <td className="px-3 py-2 font-mono">
+ <Link
+ href={`/channels/${c.slug}`}
+ className="underline hover:text-zinc-900 dark:hover:text-zinc-100"
+ >
+ {c.slug}
+ </Link>
+ </td>
+ <td className="px-3 py-2">{c.config.handling}</td>
+ <td className="px-3 py-2 text-right">{c.videoCount}</td>
+ <td className="px-3 py-2 text-xs text-zinc-500">
+ {c.config.lastSyncedAt
+ ? new Date(c.config.lastSyncedAt).toLocaleString()
+ : "never"}
+ </td>
+ </tr>
+ ))}
+ </tbody>
+ </table>
+ </section>
+ )}
+
<p className="text-sm text-zinc-500">
- Transcripts dir:{" "}
- <code className="text-xs">{paths.transcriptsDir}</code>
+ Transcripts dir: <code className="text-xs">{paths.transcriptsDir}</code>
</p>
</div>
);
@@ -54,3 +106,26 @@ function Stat({ label, value }: { label: string; value: string }) {
</div>
);
}
+
+function StatLink({
+ label,
+ value,
+ href,
+}: {
+ label: string;
+ value: string;
+ href: string;
+}) {
+ return (
+ <Link
+ href={href}
+ data-testid={`stat-${label.replace(/\s+/g, "-")}`}
+ className="rounded-md border border-zinc-200 dark:border-zinc-800 bg-white dark:bg-zinc-900 px-4 py-3 hover:border-zinc-400 dark:hover:border-zinc-600"
+ >
+ <div className="text-xs uppercase tracking-wide text-zinc-500">
+ {label}
+ </div>
+ <div className="text-2xl font-semibold mt-1">{value}</div>
+ </Link>
+ );
+}
diff --git a/editor/app/settings/_components/SettingsForm.tsx b/editor/app/settings/_components/SettingsForm.tsx
@@ -0,0 +1,110 @@
+"use client";
+
+import { useActionState } from "react";
+import {
+ saveSettingsAction,
+ type SaveResult,
+} from "../actions";
+import type { SiteSettings } from "yt-dlp-transcript-common/lib/settings";
+
+type Props = {
+ initial: SiteSettings;
+};
+
+export function SettingsForm({ initial }: Props) {
+ const [state, formAction] = useActionState<SaveResult | undefined, FormData>(
+ saveSettingsAction,
+ undefined,
+ );
+ return (
+ <form
+ action={formAction}
+ data-testid="settings-form"
+ className="flex flex-col gap-4 max-w-xl"
+ >
+ <Field
+ label="Site title"
+ name="siteTitle"
+ defaultValue={initial.siteTitle}
+ required
+ />
+ <Field
+ label="Header title"
+ name="headerTitle"
+ defaultValue={initial.headerTitle}
+ required
+ />
+ <Field
+ label="Site description"
+ name="siteDescription"
+ defaultValue={initial.siteDescription}
+ />
+ <Field
+ label="Home tagline"
+ name="homeTagline"
+ defaultValue={initial.homeTagline}
+ />
+ <Field
+ label="Max transcript page bytes"
+ name="maxTranscriptPageBytes"
+ defaultValue={String(initial.maxTranscriptPageBytes)}
+ hint="Bytes per generated transcript page (256 KB – 20 MB). Affects pagination next build."
+ type="number"
+ />
+ <div className="flex items-center gap-3">
+ <button
+ type="submit"
+ className="px-3 py-2 rounded-md bg-zinc-900 dark:bg-zinc-100 text-zinc-100 dark:text-zinc-900 text-sm font-medium hover:opacity-90"
+ >
+ Save settings
+ </button>
+ {state?.ok === true && (
+ <span
+ data-testid="settings-saved"
+ className="text-sm text-green-700 dark:text-green-300"
+ >
+ Saved.
+ </span>
+ )}
+ {state && state.ok === false && (
+ <span
+ data-testid="settings-error"
+ className="text-sm text-red-700 dark:text-red-300"
+ >
+ {state.error}
+ </span>
+ )}
+ </div>
+ </form>
+ );
+}
+
+function Field({
+ label,
+ name,
+ defaultValue,
+ hint,
+ required,
+ type = "text",
+}: {
+ label: string;
+ name: string;
+ defaultValue?: string;
+ hint?: string;
+ required?: boolean;
+ type?: string;
+}) {
+ return (
+ <label className="flex flex-col gap-1 text-sm">
+ <span className="font-medium">{label}</span>
+ <input
+ type={type}
+ name={name}
+ defaultValue={defaultValue}
+ required={required}
+ className="rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 px-2 py-1 text-sm"
+ />
+ {hint && <span className="text-xs text-zinc-500">{hint}</span>}
+ </label>
+ );
+}
diff --git a/editor/app/settings/actions.ts b/editor/app/settings/actions.ts
@@ -0,0 +1,50 @@
+"use server";
+
+import { revalidatePath } from "next/cache";
+import {
+ TRANSCRIPT_PAGE_HARD_CAP_BYTES,
+ TRANSCRIPT_PAGE_MIN_BYTES,
+ writeSettings,
+ type SiteSettings,
+} from "yt-dlp-transcript-common/lib/settings";
+
+export type SaveResult = { ok: true } | { ok: false; error: string };
+
+export async function saveSettingsAction(
+ _prev: SaveResult | undefined,
+ formData: FormData,
+): Promise<SaveResult> {
+ const siteTitle = String(formData.get("siteTitle") ?? "").trim();
+ const siteDescription = String(formData.get("siteDescription") ?? "").trim();
+ const headerTitle = String(formData.get("headerTitle") ?? "").trim();
+ const homeTagline = String(formData.get("homeTagline") ?? "").trim();
+ const maxBytesRaw = String(formData.get("maxTranscriptPageBytes") ?? "").trim();
+
+ if (!siteTitle) return { ok: false, error: "Site title is required" };
+ if (!headerTitle) return { ok: false, error: "Header title is required" };
+
+ const parsed = Number.parseInt(maxBytesRaw, 10);
+ if (!Number.isFinite(parsed)) {
+ return { ok: false, error: "maxTranscriptPageBytes must be a number" };
+ }
+ if (
+ parsed < TRANSCRIPT_PAGE_MIN_BYTES ||
+ parsed > TRANSCRIPT_PAGE_HARD_CAP_BYTES
+ ) {
+ return {
+ ok: false,
+ error: `maxTranscriptPageBytes must be between ${TRANSCRIPT_PAGE_MIN_BYTES} and ${TRANSCRIPT_PAGE_HARD_CAP_BYTES}`,
+ };
+ }
+
+ const next: SiteSettings = {
+ siteTitle,
+ siteDescription,
+ headerTitle,
+ homeTagline,
+ maxTranscriptPageBytes: parsed,
+ };
+ await writeSettings(next);
+ revalidatePath("/settings");
+ return { ok: true };
+}
diff --git a/editor/app/settings/page.tsx b/editor/app/settings/page.tsx
@@ -1,8 +1,14 @@
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { getSettings } from "yt-dlp-transcript-common/lib/settings";
+import { SettingsForm } from "./_components/SettingsForm";
+
+export const dynamic = "force-dynamic";
export default function SettingsPage() {
const paths = getPaths();
- const rows: [string, string][] = [
+ const settings = getSettings();
+
+ const pathRows: [string, string][] = [
["monorepoRoot", paths.monorepoRoot],
["transcriptsDir", paths.transcriptsDir],
["channelsDir", paths.channelsDir],
@@ -14,28 +20,48 @@ export default function SettingsPage() {
["whisperModel", paths.whisperModel],
["parallelTranscribeLimit", String(paths.parallelTranscribeLimit)],
];
+
return (
- <div className="flex flex-col gap-4">
+ <div className="flex flex-col gap-8">
<h1 className="text-2xl font-semibold">Settings</h1>
- <p className="text-sm text-zinc-500">
- Resolved paths from <code className="text-xs">getPaths()</code>.
- Editable settings UI lands in Phase 8.
- </p>
- <table className="text-sm">
- <tbody>
- {rows.map(([key, value]) => (
- <tr
- key={key}
- className="border-b border-zinc-200 dark:border-zinc-800"
- >
- <th className="text-left font-medium py-2 pr-4 align-top">
- {key}
- </th>
- <td className="py-2 font-mono text-xs break-all">{value}</td>
- </tr>
- ))}
- </tbody>
- </table>
+
+ <section className="flex flex-col gap-3">
+ <div>
+ <h2 className="text-lg font-semibold">Site settings</h2>
+ <p className="text-sm text-zinc-500">
+ Persisted in <code>settings.json</code> at the monorepo root.
+ Used by the export build to title the static site.
+ </p>
+ </div>
+ <SettingsForm initial={settings} />
+ </section>
+
+ <section className="flex flex-col gap-3 border-t border-zinc-200 dark:border-zinc-800 pt-6">
+ <div>
+ <h2 className="text-lg font-semibold">Resolved paths</h2>
+ <p className="text-sm text-zinc-500">
+ Read-only view of <code>getPaths()</code>. Set the matching env
+ vars (TRANSCRIPTS_DIR, EXPORT_PUBLIC_DIR, YTDLP_BIN, WHISPER_BIN,
+ WHISPER_MODEL, PARALLEL_TRANSCRIBE_LIMIT) before launching to
+ override.
+ </p>
+ </div>
+ <table className="text-sm">
+ <tbody>
+ {pathRows.map(([key, value]) => (
+ <tr
+ key={key}
+ className="border-b border-zinc-200 dark:border-zinc-800"
+ >
+ <th className="text-left font-medium py-2 pr-4 align-top">
+ {key}
+ </th>
+ <td className="py-2 font-mono text-xs break-all">{value}</td>
+ </tr>
+ ))}
+ </tbody>
+ </table>
+ </section>
</div>
);
}
diff --git a/editor/cypress/e2e/dashboard.cy.ts b/editor/cypress/e2e/dashboard.cy.ts
@@ -10,28 +10,30 @@ describe("Dashboard", () => {
cy.get('[data-testid="stat-videos"]').findByText("0");
});
- it("links to Channels, Build, Jobs, Settings", () => {
+ it("sidebar links to Channels, Build, Jobs, Settings", () => {
cy.visit("/");
- cy.findByRole("link", { name: /channels/i }).should(
- "have.attr",
- "href",
- "/channels",
- );
- cy.findByRole("link", { name: /build/i }).should(
- "have.attr",
- "href",
- "/build",
- );
- cy.findByRole("link", { name: /jobs/i }).should(
- "have.attr",
- "href",
- "/jobs",
- );
- cy.findByRole("link", { name: /settings/i }).should(
- "have.attr",
- "href",
- "/settings",
- );
+ cy.get("aside nav").within(() => {
+ cy.findByRole("link", { name: /channels/i }).should(
+ "have.attr",
+ "href",
+ "/channels",
+ );
+ cy.findByRole("link", { name: /build/i }).should(
+ "have.attr",
+ "href",
+ "/build",
+ );
+ cy.findByRole("link", { name: /jobs/i }).should(
+ "have.attr",
+ "href",
+ "/jobs",
+ );
+ cy.findByRole("link", { name: /settings/i }).should(
+ "have.attr",
+ "href",
+ "/settings",
+ );
+ });
});
it("settings page lists getPaths() values", () => {
diff --git a/editor/cypress/e2e/jobs.cy.ts b/editor/cypress/e2e/jobs.cy.ts
@@ -0,0 +1,38 @@
+describe("Jobs", () => {
+ it("empty state when no jobs have run", () => {
+ cy.resetData("empty");
+ // Clear any stray .jobs/*.log from other tests in this lifetime.
+ cy.visit("/jobs");
+ cy.get('[data-testid="jobs-clear-archived"]').click();
+ cy.visit("/jobs");
+ cy.findByTestId("jobs-empty");
+ });
+
+ it("kicks off a build-index job, lists it, and tails its log", () => {
+ cy.resetData("one-youtube-channel-with-data");
+ cy.visit("/build");
+ cy.findByTestId("build-index").find("button").click();
+ cy.findByTestId("build-index-log", { timeout: 30_000 }).should(
+ "contain.text",
+ "Done",
+ );
+
+ cy.visit("/jobs");
+ cy.findByTestId("jobs-table").within(() => {
+ cy.contains("build-index");
+ });
+ // Click into the most recent job. Pick the first job-row-* element we find.
+ cy.get('[data-testid^="job-row-"]')
+ .first()
+ .find("a")
+ .click();
+ cy.findByTestId("job-log-pre", { timeout: 10_000 }).should(
+ "contain.text",
+ "Done",
+ );
+ cy.findByTestId("job-log-status").should((el) => {
+ const text = el.text().trim();
+ expect(text).to.be.oneOf(["done", "running"]);
+ });
+ });
+});
diff --git a/editor/cypress/e2e/settings.cy.ts b/editor/cypress/e2e/settings.cy.ts
@@ -0,0 +1,40 @@
+describe("Settings", () => {
+ it("shows resolved paths and current settings", () => {
+ cy.resetData("empty");
+ cy.visit("/settings");
+ cy.findByText("transcriptsDir");
+ cy.findByText("ytdlpBin");
+ cy.findByLabelText(/site title/i)
+ .invoke("val")
+ .should("not.be.empty");
+ cy.findByLabelText(/header title/i)
+ .invoke("val")
+ .should("not.be.empty");
+ });
+
+ it("saves settings via the form", () => {
+ cy.resetData("empty");
+ cy.visit("/settings");
+ cy.findByLabelText(/site title/i).clear().type("Cypress Title");
+ cy.findByLabelText(/header title/i).clear().type("Cypress Header");
+ cy.findByLabelText(/site description/i).clear().type("Cypress description");
+ cy.findByRole("button", { name: /save settings/i }).click();
+ cy.findByTestId("settings-saved", { timeout: 10_000 });
+ cy.task("readJson", "test-settings.json").then((s) => {
+ expect((s as { siteTitle: string }).siteTitle).to.equal("Cypress Title");
+ expect((s as { headerTitle: string }).headerTitle).to.equal(
+ "Cypress Header",
+ );
+ });
+ });
+
+ it("rejects out-of-range maxTranscriptPageBytes", () => {
+ cy.resetData("empty");
+ cy.visit("/settings");
+ cy.findByLabelText(/max transcript page bytes/i)
+ .clear()
+ .type("42");
+ cy.findByRole("button", { name: /save settings/i }).click();
+ cy.findByTestId("settings-error").should("contain.text", "between");
+ });
+});
diff --git a/editor/package.json b/editor/package.json
@@ -5,7 +5,7 @@
"type": "module",
"scripts": {
"dev": "next dev --port 3001",
- "dev:test": "TRANSCRIPTS_DIR=$(pwd)/test-transcripts EXPORT_PUBLIC_DIR=$(pwd)/test-transcripts/.export-public YTDLP_BIN=$(pwd)/cypress/fixtures/bin/fake-ytdlp.mjs WHISPER_BIN=$(pwd)/cypress/fixtures/bin/fake-whisper.mjs WHISPER_MODEL=/dev/null next dev --port 3001",
+ "dev:test": "TRANSCRIPTS_DIR=$(pwd)/test-transcripts EXPORT_PUBLIC_DIR=$(pwd)/test-transcripts/.export-public SETTINGS_FILE=$(pwd)/test-settings.json YTDLP_BIN=$(pwd)/cypress/fixtures/bin/fake-ytdlp.mjs WHISPER_BIN=$(pwd)/cypress/fixtures/bin/fake-whisper.mjs WHISPER_MODEL=/dev/null next dev --port 3001",
"build": "next build",
"start": "next start --port 3001",
"lint": "eslint",
