commit 174e147e1cf2c36cf0b6f8513fa236cf45592d9d parent b7e0d2b8ecb326a3aaba91912a252835e2454960 Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st> Date: Sun, 26 Apr 2026 21:22:47 -0400 feat(editor): build static export from UI with streaming logs and e2e tests Adds the job registry, mutex, and streaming primitives so the editor can run long-running operations and stream logs back to the browser. - common/jobs/registry.ts: process-singleton (globalThis.__yttJobRegistry__) Map<JobId, JobRecord> with named-key mutexes and bounded job history. cancel() sends SIGTERM then SIGKILL after 5s. - common/jobs/streamCommand.ts: - runManagedCommand spawns a child via execa({all:true, buffer:false}), tees combined stdout+stderr into <jobsDir>/<id>.log AND a ReadableStream<string> returned to the action. Enforces mutex and returns the recipe-website-style busy message when held. - runManagedFunction wraps an async function with onLog + AbortSignal, same teeing behavior, no child process. - common/components/StreamActionLog.tsx: client component that takes a trigger action, renders a button + auto-scrolling pre, and reads the ReadableStream chunk by chunk into state. - editor/app/build/buildAction.ts: buildIndexAction (in-process via runManagedFunction → buildIndex) and buildExportAction (spawns pnpm run build in monorepoRoot/export, propagating TRANSCRIPTS_DIR and EXPORT_PUBLIC_DIR so the child sees the same getPaths()). - editor/app/build/page.tsx: two StreamActionLog instances. - editor/next.config.ts: serverExternalPackages = ["lmdb", "msgpackr", "msgpackr-extract"] so Next's bundler doesn't try to traverse lmdb's cbor-x branch. Cypress: - New fixture one-youtube-channel-with-data/ with synthetic metadata.info.json + transcript.en.vtt so build-index has real data to ingest. - editor/cypress/e2e/build.cy.ts: build-index against an empty fixture (asserts manifest.totalCount === 0), build-index against the data fixture (asserts totalCount === 1, channel listed, transcript page emitted), and build-export button renders enabled. Verified: pnpm --filter editor run e2e-dev:headless passes 13/13 specs; pnpm --filter export run build still produces the existing static output. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Diffstat:
10 files changed, 528 insertions(+), 4 deletions(-)
diff --git a/common/components/StreamActionLog.tsx b/common/components/StreamActionLog.tsx @@ -0,0 +1,88 @@ +"use client"; + +import { useEffect, useRef, useState } from "react"; +import type { StreamActionResult } from "../jobs/streamCommand"; + +type Props = { + trigger: () => Promise<StreamActionResult>; + buttonLabel: string; + runningLabel?: string; + testId?: string; +}; + +export function StreamActionLog({ + trigger, + buttonLabel, + runningLabel = "Running…", + testId, +}: Props) { + const [running, setRunning] = useState(false); + const [log, setLog] = useState(""); + const [error, setError] = useState<string | null>(null); + const [jobId, setJobId] = useState<string | null>(null); + const preRef = useRef<HTMLPreElement | null>(null); + + useEffect(() => { + if (preRef.current) preRef.current.scrollTop = preRef.current.scrollHeight; + }, [log]); + + async function handleClick() { + setError(null); + setLog(""); + setJobId(null); + setRunning(true); + try { + const result = await trigger(); + if (!result.ok) { + setError(result.error); + return; + } + setJobId(result.jobId); + const reader = result.stream.getReader(); + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value) setLog((prev) => prev + value); + } + } catch (e) { + setError((e as Error).message); + } finally { + setRunning(false); + } + } + + return ( + <div className="flex flex-col gap-2" data-testid={testId}> + <div className="flex items-center gap-3"> + <button + type="button" + onClick={handleClick} + disabled={running} + className="px-3 py-2 rounded-md bg-zinc-900 dark:bg-zinc-100 text-zinc-100 dark:text-zinc-900 text-sm font-medium hover:opacity-90 disabled:opacity-50" + > + {running ? runningLabel : buttonLabel} + </button> + {jobId && ( + <span className="text-xs text-zinc-500 font-mono">job {jobId}</span> + )} + </div> + {error && ( + <div + data-testid={testId ? `${testId}-error` : undefined} + className="rounded border border-red-300 bg-red-50 dark:border-red-800 dark:bg-red-950 px-3 py-2 text-sm text-red-700 dark:text-red-300" + > + {error} + </div> + )} + {(log || running) && ( + <pre + ref={preRef} + data-testid={testId ? `${testId}-log` : undefined} + className="text-xs font-mono bg-zinc-100 dark:bg-zinc-900 border border-zinc-200 dark:border-zinc-800 rounded p-3 max-h-96 overflow-auto whitespace-pre-wrap" + > + {log || "Waiting for output…"} + </pre> + )} + </div> + ); +} diff --git a/common/jobs/registry.ts b/common/jobs/registry.ts @@ -0,0 +1,88 @@ +import type { ChildProcess } from "node:child_process"; + +export type JobStatus = "running" | "done" | "failed" | "cancelled"; + +export type JobRecord = { + id: string; + kind: string; + mutexKey?: string; + status: JobStatus; + startedAt: number; + endedAt?: number; + exitCode?: number; + logPath: string; + child?: ChildProcess; + abortController?: AbortController; +}; + +class JobRegistry { + private jobs = new Map<string, JobRecord>(); + private mutexes = new Set<string>(); + + acquireMutex(key: string): boolean { + if (this.mutexes.has(key)) return false; + this.mutexes.add(key); + return true; + } + + releaseMutex(key: string): void { + this.mutexes.delete(key); + } + + hasMutex(key: string): boolean { + return this.mutexes.has(key); + } + + register(record: JobRecord): void { + this.jobs.set(record.id, record); + // Bound the registry. Drop oldest finished records past 100. + if (this.jobs.size > 100) { + const finished = Array.from(this.jobs.values()) + .filter((j) => j.status !== "running") + .sort((a, b) => (a.endedAt ?? 0) - (b.endedAt ?? 0)); + for (const drop of finished.slice(0, this.jobs.size - 100)) { + this.jobs.delete(drop.id); + } + } + } + + get(id: string): JobRecord | undefined { + return this.jobs.get(id); + } + + list(): JobRecord[] { + return Array.from(this.jobs.values()).sort( + (a, b) => b.startedAt - a.startedAt, + ); + } + + cancel(id: string): boolean { + const job = this.jobs.get(id); + if (!job || job.status !== "running") return false; + job.status = "cancelled"; + job.abortController?.abort(); + if (job.child) { + job.child.kill("SIGTERM"); + setTimeout(() => { + if (job.child && !job.child.killed) job.child.kill("SIGKILL"); + }, 5_000).unref?.(); + } + return true; + } +} + +declare global { + // eslint-disable-next-line no-var + var __yttJobRegistry__: JobRegistry | undefined; +} + +export function getRegistry(): JobRegistry { + if (!globalThis.__yttJobRegistry__) { + globalThis.__yttJobRegistry__ = new JobRegistry(); + } + return globalThis.__yttJobRegistry__; +} + +export function newJobId(): string { + return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`; +} diff --git a/common/jobs/streamCommand.ts b/common/jobs/streamCommand.ts @@ -0,0 +1,189 @@ +import path from "node:path"; +import { mkdir } from "node:fs/promises"; +import { createWriteStream, type WriteStream } from "node:fs"; +import { execa } from "execa"; +import { getRegistry, newJobId, type JobRecord } from "./registry"; +import type { Paths } from "../lib/paths"; + +export type StreamActionResult = + | { ok: true; jobId: string; stream: ReadableStream<string> } + | { ok: false; error: string }; + +type CommonOpts = { + kind: string; + mutexKey: string; + paths: Paths; + busyMessage?: string; +}; + +export type RunManagedCommandOpts = CommonOpts & { + cwd: string; + command: string; + args: string[]; + env?: Record<string, string | undefined>; +}; + +export type RunManagedFunctionOpts = CommonOpts & { + fn: ( + onLog: (line: string) => void, + signal: AbortSignal, + ) => Promise<void>; +}; + +async function ensureJobsDir(paths: Paths): Promise<void> { + await mkdir(paths.jobsDir, { recursive: true }); +} + +function makeJob( + kind: string, + mutexKey: string, + paths: Paths, +): { id: string; logPath: string; record: JobRecord } { + const id = newJobId(); + const logPath = path.join(paths.jobsDir, `${id}.log`); + const record: JobRecord = { + id, + kind, + mutexKey, + status: "running", + startedAt: Date.now(), + logPath, + }; + getRegistry().register(record); + return { id, logPath, record }; +} + +export async function runManagedCommand( + opts: RunManagedCommandOpts, +): Promise<StreamActionResult> { + const registry = getRegistry(); + if (!registry.acquireMutex(opts.mutexKey)) { + return { + ok: false, + error: opts.busyMessage ?? "A build is already currently running!", + }; + } + await ensureJobsDir(opts.paths); + const { id, logPath, record } = makeJob(opts.kind, opts.mutexKey, opts.paths); + + const child = execa(opts.command, opts.args, { + cwd: opts.cwd, + env: opts.env, + all: true, + buffer: false, + reject: false, + }); + record.child = child; + + const fileStream = createWriteStream(logPath); + const stream = new ReadableStream<string>({ + start(controller) { + teeChildToControllerAndFile( + child, + fileStream, + controller, + registry, + record, + ); + }, + cancel() { + // Client disconnected. Job continues to write to its log file so a + // future viewer can re-attach (Phase 8). + }, + }); + + return { ok: true, jobId: id, stream }; +} + +function teeChildToControllerAndFile( + child: ReturnType<typeof execa>, + fileStream: WriteStream, + controller: ReadableStreamDefaultController<string>, + registry: ReturnType<typeof getRegistry>, + record: JobRecord, +): void { + child.all?.on("data", (chunk: Buffer) => { + fileStream.write(chunk); + try { + controller.enqueue(chunk.toString("utf8")); + } catch { + // Controller closed; client gone. + } + }); + child.all?.on("error", () => {}); + + child + .then((result) => { + if (record.status === "cancelled") return; + record.exitCode = result.exitCode ?? undefined; + record.status = result.exitCode === 0 ? "done" : "failed"; + }) + .catch((err) => { + if (record.status !== "cancelled") { + record.status = "failed"; + try { + controller.enqueue(`\n[error] ${(err as Error).message}\n`); + } catch {} + } + }) + .finally(() => { + record.endedAt = Date.now(); + registry.releaseMutex(record.mutexKey!); + fileStream.end(); + try { + controller.close(); + } catch {} + }); +} + +export async function runManagedFunction( + opts: RunManagedFunctionOpts, +): Promise<StreamActionResult> { + const registry = getRegistry(); + if (!registry.acquireMutex(opts.mutexKey)) { + return { + ok: false, + error: opts.busyMessage ?? "A build is already currently running!", + }; + } + await ensureJobsDir(opts.paths); + const { id, logPath, record } = makeJob(opts.kind, opts.mutexKey, opts.paths); + const abort = new AbortController(); + record.abortController = abort; + + const fileStream = createWriteStream(logPath); + const stream = new ReadableStream<string>({ + start(controller) { + const onLog = (line: string) => { + const text = line.endsWith("\n") ? line : `${line}\n`; + fileStream.write(text); + try { + controller.enqueue(text); + } catch {} + }; + + opts + .fn(onLog, abort.signal) + .then(() => { + if (record.status !== "cancelled") record.status = "done"; + }) + .catch((err) => { + if (record.status !== "cancelled") { + record.status = "failed"; + onLog(`[error] ${(err as Error).message}`); + } + }) + .finally(() => { + record.endedAt = Date.now(); + registry.releaseMutex(record.mutexKey!); + fileStream.end(); + try { + controller.close(); + } catch {} + }); + }, + cancel() {}, + }); + + return { ok: true, jobId: id, stream }; +} diff --git a/editor/app/build/buildAction.ts b/editor/app/build/buildAction.ts @@ -0,0 +1,42 @@ +"use server"; + +import { revalidatePath } from "next/cache"; +import { buildIndex } from "yt-dlp-transcript-common/controller/buildIndex"; +import { getPaths } from "yt-dlp-transcript-common/lib/paths"; +import { + runManagedCommand, + runManagedFunction, + type StreamActionResult, +} from "yt-dlp-transcript-common/jobs/streamCommand"; + +export async function buildIndexAction(): Promise<StreamActionResult> { + const paths = getPaths(); + const result = await runManagedFunction({ + kind: "build-index", + mutexKey: "build", + paths, + fn: async (onLog) => { + await buildIndex({ paths, onLog }); + revalidatePath("/"); + }, + }); + return result; +} + +export async function buildExportAction(): Promise<StreamActionResult> { + const paths = getPaths(); + return runManagedCommand({ + kind: "build-export", + mutexKey: "build", + paths, + cwd: paths.exportDir, + command: "pnpm", + args: ["run", "build"], + env: { + ...process.env, + NODE_ENV: "production", + TRANSCRIPTS_DIR: paths.transcriptsDir, + EXPORT_PUBLIC_DIR: paths.exportPublicDir, + }, + }); +} diff --git a/editor/app/build/page.tsx b/editor/app/build/page.tsx @@ -1,10 +1,44 @@ +import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog"; +import { buildExportAction, buildIndexAction } from "./buildAction"; + export default function BuildPage() { return ( - <div className="flex flex-col gap-4"> + <div className="flex flex-col gap-8"> <h1 className="text-2xl font-semibold">Build</h1> - <p className="text-sm text-zinc-500"> - Static export builder lands in Phase 5. - </p> + + <section className="flex flex-col gap-3"> + <div> + <h2 className="text-lg font-semibold">Build index</h2> + <p className="text-sm text-zinc-500"> + Re-scans <code>transcripts/channels/</code> and rewrites paginated + JSON in <code>export/public/</code>. Cheap when nothing changed + (mtime short-circuit). + </p> + </div> + <StreamActionLog + trigger={buildIndexAction} + buttonLabel="Build index" + runningLabel="Building index…" + testId="build-index" + /> + </section> + + <section className="flex flex-col gap-3 border-t border-zinc-200 dark:border-zinc-800 pt-6"> + <div> + <h2 className="text-lg font-semibold">Build static export</h2> + <p className="text-sm text-zinc-500"> + Spawns <code>pnpm run build</code> in <code>export/</code> — runs + the index build, then <code>next build</code> to produce the + static site at <code>export/out/</code>. + </p> + </div> + <StreamActionLog + trigger={buildExportAction} + buttonLabel="Build static export" + runningLabel="Building static export…" + testId="build-export" + /> + </section> </div> ); } diff --git a/editor/cypress/e2e/build.cy.ts b/editor/cypress/e2e/build.cy.ts @@ -0,0 +1,48 @@ +describe("Build", () => { + it("build index against an empty fixture writes a 0-transcript manifest", () => { + cy.resetData("empty"); + cy.visit("/build"); + cy.findByTestId("build-index").find("button").click(); + cy.findByTestId("build-index-log", { timeout: 20_000 }).should( + "contain.text", + "Done", + ); + cy.task( + "readJson", + "test-transcripts/.export-public/summaries/manifest.json", + ).then((manifest) => { + expect((manifest as { totalCount: number }).totalCount).to.equal(0); + }); + }); + + it("build index ingests synthetic video data and emits a transcript page", () => { + cy.resetData("one-youtube-channel-with-data"); + cy.visit("/build"); + cy.findByTestId("build-index").find("button").click(); + cy.findByTestId("build-index-log", { timeout: 20_000 }).should( + "contain.text", + "Done", + ); + cy.task( + "readJson", + "test-transcripts/.export-public/summaries/manifest.json", + ).then((manifest) => { + const m = manifest as { totalCount: number; channels: { name: string }[] }; + expect(m.totalCount).to.equal(1); + expect(m.channels.map((c) => c.name)).to.include("Test YouTube Channel"); + }); + cy.task( + "pathExists", + "test-transcripts/.export-public/transcripts/test-youtube/page-0000.json", + ).then((exists) => expect(exists).to.equal(true)); + }); + + it("build static export button renders and is enabled", () => { + cy.resetData("empty"); + cy.visit("/build"); + cy.findByTestId("build-export") + .find("button") + .should("not.be.disabled") + .and("contain.text", "Build static export"); + }); +}); diff --git a/editor/cypress/fixtures/test-transcripts/one-youtube-channel-with-data/channels/test-youtube/config.json b/editor/cypress/fixtures/test-transcripts/one-youtube-channel-with-data/channels/test-youtube/config.json @@ -0,0 +1,5 @@ +{ + "handling": "youtube", + "name": "Test YouTube Channel", + "url": "https://www.youtube.com/@example/videos" +} diff --git a/editor/cypress/fixtures/test-transcripts/one-youtube-channel-with-data/channels/test-youtube/data/20240101_test1234567/metadata.info.json b/editor/cypress/fixtures/test-transcripts/one-youtube-channel-with-data/channels/test-youtube/data/20240101_test1234567/metadata.info.json @@ -0,0 +1,17 @@ +{ + "id": "test1234567", + "title": "Synthetic Test Video", + "channel": "Test YouTube Channel", + "channel_id": "UCtestchannelid", + "channel_url": "https://www.youtube.com/channel/UCtestchannelid", + "uploader": "Test YouTube Channel", + "upload_date": "20240101", + "duration": 120, + "description": "A synthetic test video for cypress fixtures.", + "is_live": false, + "was_live": false, + "live_status": "not_live", + "age_limit": 0, + "extractor_key": "Youtube", + "webpage_url": "https://www.youtube.com/watch?v=test1234567" +} diff --git a/editor/cypress/fixtures/test-transcripts/one-youtube-channel-with-data/channels/test-youtube/data/20240101_test1234567/transcript.en.vtt b/editor/cypress/fixtures/test-transcripts/one-youtube-channel-with-data/channels/test-youtube/data/20240101_test1234567/transcript.en.vtt @@ -0,0 +1,12 @@ +WEBVTT +Kind: captions +Language: en + +00:00:00.000 --> 00:00:05.000 +Hello, this is a synthetic test transcript. + +00:00:05.000 --> 00:00:10.000 +The second cue covers the next five seconds. + +00:00:10.000 --> 00:00:15.000 +And this is the third and final cue. diff --git a/editor/next.config.ts b/editor/next.config.ts @@ -2,6 +2,7 @@ import type { NextConfig } from "next"; const nextConfig: NextConfig = { transpilePackages: ["yt-dlp-transcript-common"], + serverExternalPackages: ["lmdb", "msgpackr", "msgpackr-extract"], experimental: { serverActions: { bodySizeLimit: "10mb",