commit 9f75883f43f3ec2ceeb9d67fa6a0f92e83486b66
parent 120d890843d673f3335bb4db3ca39b81b60bead8
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 3 Jul 2026 01:16:57 -0400
Merge feat/queue-diagnostics-page: Queue page to see & force-release stuck jobs
Adds a /jobs/queue diagnostics page that reconciles the scheduler slot view
against registry job status, surfaces stuck slots the Active Jobs page hides,
auto-heals orphaned terminal slots on read, and lets any active slot be
force-released (individually or reap-all-stuck).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Diffstat:
12 files changed, 716 insertions(+), 1 deletion(-)
diff --git a/common/jobs/registry.ts b/common/jobs/registry.ts
@@ -211,6 +211,35 @@ class JobRegistry {
return false;
}
+ // Escape hatch for a WEDGED slot: unconditionally free the scheduler slot for
+ // this id, even when the registry record is already terminal (or evicted). This
+ // deliberately BYPASSES finalize's status guard — cancel() marks a job
+ // "cancelled" synchronously but the slot is only freed later when the child
+ // promise settles → finalize() → scheduler.complete(). If that never settles
+ // (a child that ignored SIGTERM, a crashed finalizer), the registry says
+ // "cancelled" while the scheduler still marks the slot "running", blocking every
+ // job behind it on that queueKey. This kills the child hard, marks any still-
+ // active record terminal, and calls scheduler.complete unconditionally to free
+ // the slot and promote the next queued job.
+ forceRelease(id: string): boolean {
+ const job = this.jobs.get(id);
+ if (job) {
+ job.abortController?.abort();
+ job.drainController?.abort();
+ if (job.child && !job.child.killed) job.child.kill("SIGKILL");
+ if (job.status === "queued" || job.status === "running") {
+ job.status = "cancelled";
+ job.endedAt = Date.now();
+ }
+ job.tasks = [];
+ job.drainController = undefined;
+ }
+ // Free the slot even if the record is terminal or evicted. scheduler.complete
+ // is idempotent, a no-op for unknown ids, and promotes the next queued job.
+ getScheduler().complete(id);
+ return true;
+ }
+
// Soft-cancel ("drain"): stop the job from starting NEW sub-operations but
// let in-flight ones run to completion. The batch's fn returns normally,
// which finalizes it "done" and promotes the next queued job. A queued job
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,7 @@
# Changelog
## [Unreleased]
+- **New Queue diagnostics page (`/jobs/queue`): see & force-release stuck jobs.** The job system has two sources of truth that can drift — the registry owns each job's `status`, the scheduler owns the running SLOT per queue. A cancel that never finalizes (a child that ignored SIGTERM, a crashed finalizer) leaves a job "cancelled" in the registry while the scheduler still marks its slot running, silently blocking every job behind it on that queue — and the Active Jobs page hides it (it filters to running/queued). The new **Queue** page reconciles the two: it builds from the **scheduler** as the source of truth for slots, cross-checks each against its registry record, and flags a running head as **stuck** when the record is terminal-but-holding-slot, evicted, or (softer) a live job idle past 10 minutes. It **auto-heals** the hard cases on every view/poll (frees terminal/evicted slots), shows a health strip (active queues, running, queued, **stuck**, workers), per-queue cards with the held-for duration / PID (`kill -9` hint) / last log line, and a **Force-release** button per slot (SIGKILLs the child and frees the slot unconditionally) plus a **Reap all stuck** action. Force-release is also available on any running job in Active Jobs, and Active Jobs links to Queue with a stuck-count badge. See `common/jobs/registry.ts` (`forceRelease`), `editor/app/jobs/queue/*`, `editor/app/jobs/{actions.ts,components/ForceReleaseJobButton.tsx}`, and `editor/e2e/queue.spec.ts`.
- **Jobs page: real log retention + pagination (replaces the dead "Clear archived logs" button).** The old button only deleted logs absent from the in-memory registry — which, since the registry keeps the 100 newest finished jobs and sidecars preserve their real status, was almost never anything, so it did nothing. It's replaced by a **Clear logs** dropdown that prunes finished-job logs by age (older than 7 / 30 / 90 days) or all at once; running/queued jobs are never deleted. The `.jobs` directory also **self-trims on job finish** (throttled; keep newest 500, drop >30 days) so it can't grow unbounded. Job ids are now **ULIDs** (lexicographically time-sortable, timestamp decodable from the id), letting the list **paginate** — `listAllJobs` returns one page (default 50, grown by a **Load more** link) and only `stat`s/reads the sidecar for the shown page instead of every file on every load. `jobIdTime()` decodes both ULID and the legacy `<t36>-<rand>` ids, so existing on-disk logs still sort/read correctly. See `common/jobs/{ulid,listJobs,registry,streamCommand}.ts`, `editor/app/jobs/{page.tsx,actions.ts,components/ClearLogsMenu.tsx,[id]/page.tsx}`, and `editor/e2e/jobs.spec.ts`.
- **"Move to top" button on the auto-queue policy editor.** Each reorderable rule/group in the auto-queue policy tree gains a **⤒** button beside the existing ↑/↓ swap controls that jumps the node straight to the front of its sibling list in one click (disabled on the first row, like ↑). Reordering stays local until **Save policy**, matching the swap buttons. See `editor/app/auto-queue/components/PolicyTreeEditor.tsx` and `editor/e2e/auto-queue.spec.ts`.
- **Kick VOD playback + VOD-expiry indicators.** Kick becomes a first-class platform (`Platform` union, `detectPlatform`, `platformFromMetadata` `/^kick/i`, `extractVideoId` kick branch, `defaultWebpageUrl`). Kick VODs have no iframe embed, so playback streams the HLS manifest yt-dlp resolves at download time: `summarize()` persists `manifest_url` → `hlsUrl` on the transcript summary/detail, and a new client-only `common/components/KickPlayer.tsx` plays it in a native `<video>` via the bundled **hls.js** (not react-player's file player, which loads hls.js from a CDN and would break the offline export). It exposes the same `seekTo`/`onReady`/`onProgress` handle as the YouTube player, so Kick gets full scrubbing + cue highlighting; on a fatal manifest error it falls back to an expiry notice + source link. Separately, a shared `common/lib/vodExpiry.ts` (retention: Kick 30d, Twitch 14d, tunable) drives a new `VodExpiredBadge` on search result cards for likely-deleted Kick/Twitch VODs, with a `title=` tooltip explaining each platform's retention. Cache versions bumped so stale data re-derives (`transcriptStore` `DB_VERSION` 4, `normalizeTranscript` `CUES_FILE_VERSION` 2). See `common/lib/{platform,transcripts,transcripts-server,vodExpiry,format}.ts`, `common/components/{KickPlayer,PlayerProvider,badges,TranscriptSearch}.tsx`, `common/ytdlp/runYtdlp.ts`, and `export/e2e/kick-vod.spec.ts`.
diff --git a/editor/app/api/jobs/queue/route.ts b/editor/app/api/jobs/queue/route.ts
@@ -0,0 +1,12 @@
+import { NextResponse } from "next/server";
+import { buildQueueView } from "../../../jobs/queue/buildQueueView";
+
+export const dynamic = "force-dynamic";
+
+// Backs the ~2s client poll on the Queue diagnostics screen so stuck slots (and
+// the auto-heal that clears terminal/evicted ones) stay live without a full RSC
+// refresh.
+export async function GET() {
+ const payload = await buildQueueView();
+ return NextResponse.json(payload);
+}
diff --git a/editor/app/api/test/stuck-job/route.ts b/editor/app/api/test/stuck-job/route.ts
@@ -0,0 +1,55 @@
+import path from "node:path";
+import { mkdir, writeFile } from "node:fs/promises";
+import { NextResponse } from "next/server";
+import {
+ getRegistry,
+ newJobId,
+ type JobRecord,
+} from "yt-dlp-transcript-common/jobs/registry";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+
+export const dynamic = "force-dynamic";
+
+// E2E test harness only. Fabricates the registry/scheduler DRIFT the Queue
+// diagnostics page exists to surface: a job holding a running scheduler slot that
+// has gone quiet (no in-flight tasks) far longer than STUCK_AGE_MS — the soft
+// "possibly-stalled" case. That case is deliberately NOT auto-healed, so it
+// persists across polls and the test can prove FORCE-RELEASE (not auto-heal)
+// clears it. We reproduce it directly (backdating startedAt) since a genuinely
+// wedged child would be racy. Mounted unconditionally, like the other /api/test
+// routes — the editor is a localhost admin tool, not deployed.
+export async function GET(request: Request) {
+ const url = new URL(request.url);
+ const queueKey = url.searchParams.get("queue") || "stuck-queue";
+
+ const registry = getRegistry();
+ const paths = getPaths();
+ const id = newJobId();
+ const logPath = path.join(paths.jobsDir, `${id}.log`);
+ const record: JobRecord = {
+ id,
+ kind: "whisper-all",
+ queueKey,
+ status: "queued",
+ queuedAt: Date.now(),
+ logPath,
+ };
+ registry.register(record);
+
+ // Submit through the normal enqueue path so the scheduler grants it a running
+ // slot (start flips it to "running"/startedAt). Then backdate startedAt ~20m
+ // and clear tasks so it reads as an idle, long-held slot → "possibly-stalled".
+ registry.enqueue(record, { start: () => {}, onCancel: () => {} });
+ record.startedAt = Date.now() - 20 * 60 * 1000;
+ record.tasks = [];
+
+ // A log line so the page's lastLogLine tail has something to show.
+ try {
+ await mkdir(paths.jobsDir, { recursive: true });
+ await writeFile(logPath, "[fake] wedged job holding its slot\n", "utf8");
+ } catch {
+ /* best-effort */
+ }
+
+ return NextResponse.json({ id, queueKey });
+}
diff --git a/editor/app/jobs/actions.ts b/editor/app/jobs/actions.ts
@@ -8,6 +8,7 @@ import { readJobMeta } from "yt-dlp-transcript-common/jobs/jobMeta";
import type { JobSpec } from "yt-dlp-transcript-common/jobs/jobSpec";
import type { StreamActionResult } from "yt-dlp-transcript-common/jobs/streamCommand";
import { runJobSpec } from "./runJobSpec";
+import { buildQueueView } from "./queue/buildQueueView";
export async function cancelJobAction(id: string): Promise<{ ok: boolean }> {
const ok = getRegistry().cancel(id);
@@ -15,6 +16,33 @@ export async function cancelJobAction(id: string): Promise<{ ok: boolean }> {
return { ok };
}
+// Escape hatch for a WEDGED slot (see registry.forceRelease): unconditionally
+// free the scheduler slot for this id — even if its record is already terminal
+// or evicted — SIGKILLing any still-running child. Unblocks a queue whose head
+// is stuck. Revalidates the jobs list and the queue-diagnostics page.
+export async function forceReleaseJobAction(
+ id: string,
+): Promise<{ ok: boolean }> {
+ const ok = getRegistry().forceRelease(id);
+ revalidatePath("/jobs");
+ revalidatePath("/jobs/queue");
+ return { ok };
+}
+
+// Reap every stuck slot the queue-diagnostics view found in one pass:
+// force-release each id in its stuckIds set. Returns how many were released.
+export async function reapStuckJobsAction(): Promise<{ count: number }> {
+ const registry = getRegistry();
+ const { stuckIds } = await buildQueueView();
+ let count = 0;
+ for (const id of stuckIds) {
+ if (registry.forceRelease(id)) count++;
+ }
+ revalidatePath("/jobs");
+ revalidatePath("/jobs/queue");
+ return { count };
+}
+
// Soft-cancel: let the batch's in-flight sub-operations finish, start no new
// ones, then complete and release the queue for the next job.
export async function drainJobAction(id: string): Promise<{ ok: boolean }> {
diff --git a/editor/app/jobs/active/page.tsx b/editor/app/jobs/active/page.tsx
@@ -1,6 +1,8 @@
import type { Metadata } from "next";
+import Link from "next/link";
import { getWorkerPool } from "yt-dlp-transcript-common/jobs/workerPool";
import { buildActiveJobsPayload } from "./buildActiveJobs";
+import { buildQueueView } from "../queue/buildQueueView";
import { ActiveJobsLive } from "../components/ActiveJobsLive";
import { DrainAllButton } from "../components/DrainAllButton";
import { PauseTranscriptionsButton } from "../components/PauseTranscriptionsButton";
@@ -12,16 +14,29 @@ export const dynamic = "force-dynamic";
export const metadata: Metadata = { title: "Active jobs" };
export default async function ActiveJobsPage() {
- const [initial, { bookmarks, missingSlugs }] = await Promise.all([
+ const [initial, { bookmarks, missingSlugs }, queueView] = await Promise.all([
buildActiveJobsPayload(),
loadBookmarksView(),
+ buildQueueView(),
]);
const paused = getWorkerPool().isPaused();
+ const stuck = queueView.summary.stuck;
return (
<div className="flex flex-col gap-4">
<div className="flex items-center justify-between">
<h1 className="text-2xl font-semibold">Active jobs</h1>
<div className="flex items-center gap-2">
+ <Link
+ href="/jobs/queue"
+ className="relative px-3 py-2 rounded-md border border-border text-sm font-medium hover:bg-muted"
+ >
+ Queue
+ {stuck > 0 && (
+ <span className="ml-1.5 inline-flex items-center justify-center rounded-full bg-destructive text-destructive-foreground text-xs font-semibold px-1.5 min-w-5">
+ {stuck}
+ </span>
+ )}
+ </Link>
<PauseTranscriptionsButton paused={paused} />
<DrainAllButton />
</div>
diff --git a/editor/app/jobs/components/ForceReleaseJobButton.tsx b/editor/app/jobs/components/ForceReleaseJobButton.tsx
@@ -0,0 +1,41 @@
+"use client";
+
+import { useState } from "react";
+import { useRouter } from "next/navigation";
+import { forceReleaseJobAction } from "../actions";
+
+type Props = {
+ jobId: string;
+};
+
+// Stronger sibling of CancelJobButton: frees the scheduler slot immediately,
+// SIGKILLing any still-running child (see registry.forceRelease). Filled
+// destructive styling (vs. Cancel's outline) to read as the heavier action.
+export function ForceReleaseJobButton({ jobId }: Props) {
+ const [busy, setBusy] = useState(false);
+ const router = useRouter();
+ return (
+ <button
+ type="button"
+ onClick={async () => {
+ if (
+ !window.confirm(
+ "Force-release this slot? Frees the queue immediately; any still-running child is SIGKILLed.",
+ )
+ )
+ return;
+ setBusy(true);
+ try {
+ await forceReleaseJobAction(jobId);
+ router.refresh();
+ } finally {
+ setBusy(false);
+ }
+ }}
+ disabled={busy}
+ className="px-2 py-1 rounded bg-destructive text-destructive-foreground text-xs font-medium hover:opacity-90 disabled:opacity-50"
+ >
+ {busy ? "Releasing…" : "Force-release"}
+ </button>
+ );
+}
diff --git a/editor/app/jobs/components/RunningJobsList.tsx b/editor/app/jobs/components/RunningJobsList.tsx
@@ -7,6 +7,7 @@ import { JobLogTail } from "../[id]/components/JobLogTail";
import { jobKindLabel } from "../jobKindLabels";
import { DrainJobButton } from "./DrainJobButton";
import { CancelJobButton } from "./CancelJobButton";
+import { ForceReleaseJobButton } from "./ForceReleaseJobButton";
import { BookmarkJobButton } from "./BookmarkJobButton";
import { ReorderJobButtons } from "./ReorderJobButtons";
@@ -166,6 +167,11 @@ function JobRow({
<DrainJobButton jobId={job.id} draining={job.draining} />
)}
<CancelJobButton jobId={job.id} />
+ {job.status === "running" && (
+ // Escape hatch: force-free the scheduler slot for any running job,
+ // for when a Cancel leaves the queue wedged (see registry.forceRelease).
+ <ForceReleaseJobButton jobId={job.id} />
+ )}
<button
type="button"
onClick={() => setShowLog((s) => !s)}
diff --git a/editor/app/jobs/queue/buildQueueView.ts b/editor/app/jobs/queue/buildQueueView.ts
@@ -0,0 +1,199 @@
+import path from "node:path";
+import { readFile } from "node:fs/promises";
+import { getRegistry } from "yt-dlp-transcript-common/jobs/registry";
+import { getScheduler } from "yt-dlp-transcript-common/jobs/scheduler";
+import { getWorkerPool } from "yt-dlp-transcript-common/jobs/workerPool";
+import { readJobMeta } from "yt-dlp-transcript-common/jobs/jobMeta";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+
+// A running slot is treated as "possibly stalled" (the soft case) once a
+// running/draining record has held its slot this long with no in-flight tasks.
+// Distinct from the hard cases (terminal-but-holding-slot / record-evicted),
+// which are always safe to auto-heal; the soft case is only surfaced/force-
+// releasable, never auto-healed.
+const STUCK_AGE_MS = 10 * 60 * 1000;
+
+const TERMINAL = new Set(["done", "failed", "cancelled"]);
+
+export type QueueSlotView = {
+ id: string;
+ kind: string;
+ // Registry status ("running"/"queued"/terminal), or "evicted" when the
+ // scheduler references an id the registry no longer knows about.
+ status: string;
+ ageMs: number;
+ stuck: boolean;
+ stuckReason?: string;
+ pid?: number;
+ draining: boolean;
+ lastLogLine?: string;
+};
+
+export type QueueGroupView = {
+ name: string;
+ head?: QueueSlotView;
+ backlog: QueueSlotView[];
+};
+
+export type QueueViewPayload = {
+ queues: QueueGroupView[];
+ summary: {
+ queues: number;
+ running: number;
+ queued: number;
+ stuck: number;
+ workersPaused: boolean;
+ workerCount: number;
+ };
+ stuckIds: string[];
+};
+
+// Cheap best-effort tail: read the log file and return its last non-empty line.
+// Swallows every error (missing file, read failure) — this is diagnostic sugar,
+// never load-bearing. Mirrors the readLogChunk helper the job log route uses,
+// but keeps only the final line.
+async function readLastLogLine(
+ jobsDir: string,
+ id: string,
+): Promise<string | undefined> {
+ try {
+ const raw = await readFile(path.join(jobsDir, `${id}.log`), "utf8");
+ const lines = raw.split("\n");
+ for (let i = lines.length - 1; i >= 0; i--) {
+ const line = lines[i].trim();
+ if (line) return line;
+ }
+ } catch {
+ /* best-effort */
+ }
+ return undefined;
+}
+
+// Reconciles the scheduler (source of truth for SLOTS) against the registry
+// (source of truth for job STATUS) to surface stuck jobs the registry-status
+// filter on the Active Jobs page hides. The scheduler is the primary iteration
+// source: it still references evicted/terminal ids whose slot was never freed,
+// which is exactly the drift we're hunting.
+//
+// Auto-heal: for any running slot whose record is terminal OR missing, we call
+// scheduler.complete(id) inline — a terminal record's slot MUST be freed, so
+// this is safe and self-correcting on every poll. The soft "possibly-stalled"
+// case (a live running record sitting idle past STUCK_AGE_MS) is surfaced but
+// never auto-healed, since it may just be a genuinely long-running operation.
+export async function buildQueueView(): Promise<QueueViewPayload> {
+ const registry = getRegistry();
+ const scheduler = getScheduler();
+ const pool = getWorkerPool();
+ const paths = getPaths();
+ const now = Date.now();
+
+ // Snapshot the scheduler ONCE (each queues() call returns fresh arrays, so
+ // completing slots below won't mutate this snapshot). The scheduler is the
+ // authoritative slot view: it still references evicted/terminal ids whose slot
+ // was never freed — exactly the drift we surface.
+ const views = scheduler.queues();
+
+ // Terminal/evicted running slots are auto-healed, but only AFTER the view is
+ // built so they still render as stuck this pass (they vanish next poll). The
+ // soft "possibly-stalled" case is intentionally NOT collected here.
+ const toHeal: string[] = [];
+
+ const stuckIds: string[] = [];
+ let runningCount = 0;
+ let queuedCount = 0;
+
+ const buildSlot = async (
+ id: string,
+ role: "running" | "queued",
+ ): Promise<QueueSlotView> => {
+ const rec = registry.get(id);
+ const draining = rec?.draining === true;
+ const lastLogLine = await readLastLogLine(paths.jobsDir, id);
+ let stuck = false;
+ let stuckReason: string | undefined;
+
+ if (role === "running") {
+ if (!rec) {
+ // Scheduler references an id the registry evicted while it still holds a
+ // running slot — a stuck slot with no live record to describe it.
+ stuck = true;
+ stuckReason = "record-evicted";
+ toHeal.push(id);
+ } else if (TERMINAL.has(rec.status)) {
+ stuck = true;
+ stuckReason = "terminal-but-holding-slot";
+ toHeal.push(id);
+ } else {
+ const ageMs = now - (rec.startedAt ?? rec.queuedAt);
+ const idle = !rec.tasks || rec.tasks.length === 0;
+ if (idle && ageMs > STUCK_AGE_MS) {
+ // Soft case: a live running/draining record sitting idle far too long.
+ // Force-releasable, but deliberately NOT auto-healed above.
+ stuck = true;
+ stuckReason = "possibly-stalled";
+ }
+ }
+ }
+
+ if (stuck) stuckIds.push(id);
+
+ // For an evicted record fall back to the on-disk meta sidecar for a kind.
+ let kind = rec?.kind;
+ if (!kind) {
+ const meta = await readJobMeta(paths, id).catch(() => null);
+ kind = meta?.kind ?? "unknown";
+ }
+
+ const ageMs = now - (rec?.startedAt ?? rec?.queuedAt ?? now);
+
+ return {
+ id,
+ kind,
+ status: rec ? rec.status : "evicted",
+ ageMs,
+ stuck,
+ stuckReason,
+ pid: rec?.child?.pid,
+ draining,
+ lastLogLine,
+ };
+ };
+
+ const queues: QueueGroupView[] = [];
+ for (const v of views) {
+ // The scheduler keeps concurrency 1 per queueKey, so there's at most one
+ // running head — but iterate defensively in case that ever changes.
+ const [headId, ...extraRunning] = v.running;
+ const head = headId ? await buildSlot(headId, "running") : undefined;
+ if (head) runningCount++;
+ const backlog: QueueSlotView[] = [];
+ // Any beyond-first running ids are shown in the backlog too (rare).
+ for (const id of extraRunning) {
+ backlog.push(await buildSlot(id, "running"));
+ runningCount++;
+ }
+ for (const id of v.queued) {
+ backlog.push(await buildSlot(id, "queued"));
+ queuedCount++;
+ }
+ queues.push({ name: v.name, head, backlog });
+ }
+
+ // Auto-heal the hard cases now that they've been captured into the view: a
+ // terminal/evicted record's slot MUST be freed. Idempotent and safe; the freed
+ // slot (and any promoted successor) shows correctly on the next poll.
+ for (const id of toHeal) scheduler.complete(id);
+
+ return {
+ queues,
+ summary: {
+ queues: queues.length,
+ running: runningCount,
+ queued: queuedCount,
+ stuck: stuckIds.length,
+ workersPaused: pool.isPaused(),
+ workerCount: pool.summary().length,
+ },
+ stuckIds,
+ };
+}
diff --git a/editor/app/jobs/queue/components/QueueViewLive.tsx b/editor/app/jobs/queue/components/QueueViewLive.tsx
@@ -0,0 +1,240 @@
+"use client";
+
+import Link from "next/link";
+import { useEffect, useState } from "react";
+import { useRouter } from "next/navigation";
+import { formatDuration } from "yt-dlp-transcript-common/lib/format";
+import { jobKindLabel } from "../../jobKindLabels";
+import { reapStuckJobsAction } from "../../actions";
+import { ForceReleaseJobButton } from "../../components/ForceReleaseJobButton";
+import type {
+ QueueSlotView,
+ QueueViewPayload,
+} from "../buildQueueView";
+
+// Poll the queue-diagnostics route so stuck slots (and the server-side auto-heal
+// that clears terminal/evicted ones) stay live without a full RSC refresh.
+// Mirrors ActiveJobsLive; a touch slower since this is an admin/diagnostic view.
+const POLL_MS = 2000;
+
+export function QueueViewLive({ initial }: { initial: QueueViewPayload }) {
+ const [payload, setPayload] = useState<QueueViewPayload>(initial);
+
+ useEffect(() => {
+ let cancelled = false;
+ let timer: ReturnType<typeof setTimeout> | null = null;
+ async function tick() {
+ try {
+ const res = await fetch("/api/jobs/queue", { cache: "no-store" });
+ if (res.ok && !cancelled) {
+ setPayload((await res.json()) as QueueViewPayload);
+ }
+ } catch {
+ // transient — keep polling
+ } finally {
+ if (!cancelled) timer = setTimeout(tick, POLL_MS);
+ }
+ }
+ timer = setTimeout(tick, POLL_MS);
+ return () => {
+ cancelled = true;
+ if (timer) clearTimeout(timer);
+ };
+ }, []);
+
+ const { queues, summary } = payload;
+
+ return (
+ <div className="flex flex-col gap-4">
+ <HealthStrip payload={payload} />
+ {queues.length === 0 ? (
+ <p className="text-sm text-muted-foreground border border-dashed border-border rounded p-4">
+ No active queues.
+ </p>
+ ) : (
+ <div className="flex flex-col gap-3">
+ {queues.map((q) => (
+ <QueueCard key={q.name} name={q.name} head={q.head} backlog={q.backlog} />
+ ))}
+ </div>
+ )}
+ <p className="sr-only" aria-live="polite">
+ {summary.stuck} stuck {summary.stuck === 1 ? "slot" : "slots"}
+ </p>
+ </div>
+ );
+}
+
+function HealthStrip({ payload }: { payload: QueueViewPayload }) {
+ const router = useRouter();
+ const [reaping, setReaping] = useState(false);
+ const [copied, setCopied] = useState(false);
+ const { summary } = payload;
+
+ const stat = (label: string, value: string, emphasize?: boolean) => (
+ <div className="flex flex-col">
+ <span
+ className={`text-lg font-semibold tabular-nums ${
+ emphasize ? "text-destructive" : "text-foreground"
+ }`}
+ >
+ {value}
+ </span>
+ <span className="text-xs text-muted-foreground">{label}</span>
+ </div>
+ );
+
+ return (
+ <div className="flex flex-wrap items-center justify-between gap-4 border border-border rounded-md p-3 bg-card">
+ <div className="flex flex-wrap items-center gap-6">
+ {stat("active queues", String(summary.queues))}
+ {stat("running", String(summary.running))}
+ {stat("queued", String(summary.queued))}
+ {stat("stuck", String(summary.stuck), summary.stuck > 0)}
+ {stat(
+ summary.workersPaused ? "workers (paused)" : "workers",
+ String(summary.workerCount),
+ summary.workersPaused,
+ )}
+ </div>
+ <div className="flex items-center gap-2">
+ {summary.stuck > 0 && (
+ <button
+ type="button"
+ disabled={reaping}
+ onClick={async () => {
+ if (
+ !window.confirm(
+ `Force-release all ${summary.stuck} stuck ${
+ summary.stuck === 1 ? "slot" : "slots"
+ }? Any still-running children are SIGKILLed.`,
+ )
+ )
+ return;
+ setReaping(true);
+ try {
+ await reapStuckJobsAction();
+ router.refresh();
+ } finally {
+ setReaping(false);
+ }
+ }}
+ className="px-3 py-2 rounded-md bg-destructive text-destructive-foreground text-sm font-medium hover:opacity-90 disabled:opacity-50"
+ >
+ {reaping ? "Reaping…" : "Reap all stuck"}
+ </button>
+ )}
+ <button
+ type="button"
+ onClick={async () => {
+ try {
+ await navigator.clipboard.writeText(
+ JSON.stringify(payload, null, 2),
+ );
+ setCopied(true);
+ setTimeout(() => setCopied(false), 1500);
+ } catch {
+ // clipboard blocked — no-op
+ }
+ }}
+ className="px-3 py-2 rounded-md border border-border text-sm font-medium hover:bg-muted"
+ >
+ {copied ? "Copied" : "Copy diagnostics"}
+ </button>
+ </div>
+ </div>
+ );
+}
+
+function QueueCard({
+ name,
+ head,
+ backlog,
+}: {
+ name: string;
+ head?: QueueSlotView;
+ backlog: QueueSlotView[];
+}) {
+ const warn = head?.stuck === true;
+ return (
+ <section
+ aria-label={`Queue ${name}`}
+ className={`flex flex-col gap-2 border rounded-md p-3 ${
+ warn ? "border-warning bg-warning-soft" : "border-border bg-card"
+ }`}
+ >
+ <div className="flex items-baseline justify-between gap-2">
+ <span className="font-mono text-sm font-medium">{name || "—"}</span>
+ <span className="text-xs text-muted-foreground">
+ {head ? "1 running" : "idle"}
+ {backlog.length > 0 && ` · ${backlog.length} waiting`}
+ </span>
+ </div>
+ {head && <SlotRow slot={head} isHead />}
+ {backlog.length > 0 && (
+ <ul className="flex flex-col gap-1.5 pl-2 border-l-2 border-border">
+ {backlog.map((s) => (
+ <li key={s.id}>
+ <SlotRow slot={s} />
+ </li>
+ ))}
+ </ul>
+ )}
+ </section>
+ );
+}
+
+function SlotRow({ slot, isHead }: { slot: QueueSlotView; isHead?: boolean }) {
+ const held = formatDuration(Math.round(slot.ageMs / 1000)) || "0:00";
+ return (
+ <div className="flex flex-col gap-1 text-sm">
+ <div className="flex flex-wrap items-center gap-2">
+ <span
+ className={`text-xs uppercase tracking-wide px-2 py-0.5 rounded ${
+ isHead
+ ? "bg-info-soft text-info"
+ : "bg-muted text-muted-foreground"
+ }`}
+ >
+ {isHead ? "running" : slot.status}
+ </span>
+ <span className="text-xs font-medium" title={slot.kind}>
+ {jobKindLabel(slot.kind)}
+ </span>
+ {slot.draining && (
+ <span className="text-[10px] uppercase tracking-wide px-1.5 py-0.5 rounded bg-warning-soft text-warning">
+ draining
+ </span>
+ )}
+ <Link
+ href={`/jobs/${slot.id}`}
+ className="font-mono text-xs underline hover:text-foreground"
+ >
+ Job {slot.id}
+ </Link>
+ <span className="text-xs text-muted-foreground">held for {held}</span>
+ {typeof slot.pid === "number" && (
+ <span
+ className="font-mono text-xs text-muted-foreground"
+ title={`kill -9 ${slot.pid}`}
+ >
+ pid {slot.pid}
+ </span>
+ )}
+ {slot.stuck && (
+ <span className="text-[10px] uppercase tracking-wide px-1.5 py-0.5 rounded bg-destructive-soft text-destructive">
+ stuck{slot.stuckReason ? ` · ${slot.stuckReason}` : ""}
+ </span>
+ )}
+ <div className="ml-auto">
+ <ForceReleaseJobButton jobId={slot.id} />
+ </div>
+ </div>
+ {slot.lastLogLine && (
+ <p className="font-mono text-xs text-muted-foreground truncate">
+ {slot.lastLogLine}
+ </p>
+ )}
+ </div>
+ );
+}
diff --git a/editor/app/jobs/queue/page.tsx b/editor/app/jobs/queue/page.tsx
@@ -0,0 +1,31 @@
+import type { Metadata } from "next";
+import Link from "next/link";
+import { buildQueueView } from "./buildQueueView";
+import { QueueViewLive } from "./components/QueueViewLive";
+
+export const dynamic = "force-dynamic";
+
+export const metadata: Metadata = { title: "Queue" };
+
+export default async function QueuePage() {
+ const initial = await buildQueueView();
+ return (
+ <div className="flex flex-col gap-4">
+ <div className="flex items-center justify-between">
+ <div className="flex flex-col gap-0.5">
+ <h1 className="text-2xl font-semibold">Queue</h1>
+ <p className="text-sm text-muted-foreground">
+ Scheduler slots per queue — spot and force-release stuck jobs.
+ </p>
+ </div>
+ <Link
+ href="/jobs/active"
+ className="px-3 py-2 rounded-md border border-border text-sm font-medium hover:bg-muted"
+ >
+ Active jobs
+ </Link>
+ </div>
+ <QueueViewLive initial={initial} />
+ </div>
+ );
+}
diff --git a/editor/e2e/queue.spec.ts b/editor/e2e/queue.spec.ts
@@ -0,0 +1,58 @@
+// Queue diagnostics page: a job that holds a running scheduler slot but has gone
+// quiet ("possibly-stalled") is surfaced as STUCK — the Active Jobs page's
+// registry-status filter would otherwise hide it — and Force-release frees the
+// slot. The stuck slot is fabricated deterministically via the /api/test
+// harness (see app/api/test/stuck-job) rather than racing a real wedged child.
+
+import { test, expect } from "@playwright/test";
+import { resetData } from "./helpers";
+import { baseUrl } from "./baseUrl";
+
+async function fabricateStuckJob(queueKey: string): Promise<string> {
+ const res = await fetch(
+ `${baseUrl}/api/test/stuck-job?queue=${encodeURIComponent(queueKey)}`,
+ );
+ const body = (await res.json()) as { id: string };
+ return body.id;
+}
+
+test("a stuck slot shows on /jobs/queue and Force-release clears it", async ({
+ page,
+}) => {
+ await resetData(null);
+ const id = await fabricateStuckJob("qStuck");
+
+ await page.goto("/jobs/queue");
+
+ // Health strip renders with the stuck count emphasized.
+ await expect(page.getByRole("heading", { name: "Queue" })).toBeVisible();
+ await expect(page.getByText("stuck", { exact: true })).toBeVisible();
+
+ // The fabricated slot appears in its queue card, flagged stuck.
+ const card = page.locator("section[aria-label='Queue qStuck']");
+ await expect(card).toBeVisible();
+ await expect(card.getByText(`Job ${id}`)).toBeVisible();
+ await expect(card.getByText(/possibly-stalled/)).toBeVisible();
+
+ // Force-release it (accept the confirm) — the slot is freed and the card goes.
+ page.on("dialog", (d) => d.accept());
+ await card.getByRole("button", { name: /^Force-release$/ }).click();
+
+ await expect(page.locator("section[aria-label='Queue qStuck']")).toHaveCount(
+ 0,
+ { timeout: 10_000 },
+ );
+ await expect(page.getByText("No active queues.")).toBeVisible({
+ timeout: 10_000,
+ });
+});
+
+test("loads with an empty queue and renders the health strip", async ({
+ page,
+}) => {
+ await resetData(null);
+ await page.goto("/jobs/queue");
+ await expect(page.getByRole("heading", { name: "Queue" })).toBeVisible();
+ await expect(page.getByText("active queues", { exact: true })).toBeVisible();
+ await expect(page.getByText("No active queues.")).toBeVisible();
+});
