commit 30cd9597d5b8eaa8ac369a276f6c58d390e470d5 parent bcb66f976f72aa2a6391a2e3733191141596890d Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st> Date: Tue, 4 Aug 2026 21:27:00 -0400 Make auto-refresh ask before re-rendering, and stop generating reports in a GET AutoRefresh called router.refresh() unconditionally on a timer: a full server-side re-render of the whole page tree, every 5s, on every route, changed or not. ~4.9s of work per tick against the real corpus, and ~22% of a core held permanently by one idle tab. It now polls /api/pulse — a change token built from in-memory registry, queue and worker-pool state plus two stat() calls, with an idle fast path that skips even the badge read when the client's rev still matches — and refreshes only when the token moves. Measured: 0 tree refreshes over 5 idle seconds, where the old one did 5. e2e/auto-refresh.spec.ts gains that as a regression test and its five existing specs pass unchanged. Supporting changes: - Sidebar job/cleanable pills move to a shared client poller (components/ pulse.ts, one interval for the whole app), so the LAYOUT no longer has to re-render for a badge to change. - 8 of the 15 revalidatePath("/") calls removed — the ones that meant "a job count changed", which the dashboard already gets from its own polled endpoints. They were wiping the client router cache app-wide, which is what staleTimes now depends on. The 7 that genuinely change dashboard content (channel create/update/delete/rename, report refresh, changelog) stay. - 10 redundant router.refresh() calls removed from client components whose action already revalidates every page they render on. The ambiguous ones — where the component also renders on a page the action does NOT revalidate — are deliberately left; removing those would trade a round trip for stale UI. Also, the /channels/[slug] landmine: the page ran generateChannelSnapshot inline when no snapshot existed — a full walk of every video dir in the channel, inside a GET, multi-minute on the 11,224-video channel, triggered by clicking a link. It now renders a "No report yet" state with a button that queues the same job every other surface uses. Eight e2e specs relied on the side effect and move to an explicit generateReport() helper (which retries its click: a pre-hydration click fires nothing at all). Plus the remaining serial loops: buildQueueView built slots one at a time, each reading an ENTIRE job log for its last line (now a bounded 8KB tail read, all slots concurrent, healing pass still deterministic), and listJobs read a page of entries serially. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Diffstat:
33 files changed, 809 insertions(+), 138 deletions(-)
diff --git a/common/jobs/listJobs.ts b/common/jobs/listJobs.ts @@ -1,7 +1,11 @@ import path from "node:path"; import { readdir, stat, readFile, rm } from "node:fs/promises"; import type { Paths } from "../lib/paths"; +import { mapConcurrent } from "../lib/concurrency"; import { getRegistry, type JobRecord, type JobStatus } from "./registry"; + +// One page of job rows, each a couple of small sidecar reads. +const ENTRY_READ_CONCURRENCY = 32; import { metaPath, readJobMeta } from "./jobMeta"; import { jobIdTime } from "./ulid"; @@ -183,10 +187,11 @@ export async function listAllJobs( const hasMore = slice.length > limit; const page = hasMore ? slice.slice(0, limit) : slice; - const entries: JobListEntry[] = []; - for (const c of page) { - entries.push(await buildEntry(paths, c.id, live.get(c.id), c.hasLog)); - } + // One page's worth of per-job sidecar reads, concurrently rather than one at + // a time. mapConcurrent preserves input order, so the sort above still holds. + const entries = await mapConcurrent(page, ENTRY_READ_CONCURRENCY, (c) => + buildEntry(paths, c.id, live.get(c.id), c.hasLog), + ); return { entries, hasMore, total: candidates.size }; } diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] - **The editor is fast now.** Every page in the editor had a floor of about 4.4 seconds on it, and the reason was one line in the sidebar. The reclaimable-disk badge — the little "12.4 GB" pill next to Cleanup — asked for the channel list, and the function it asked was the one that counts the corpus from scratch: a `readdir` for each of the **78,350** video directories plus a digest sidecar read for each of the ~70,000 transcribed ones, **~474,559 files touched, measured at 3,985 ms**, to describe **98 videos**. Every count that walk produced was then thrown away. It sat in the root layout, so *every* document load paid it; the 5-second auto-refresh re-ran it on a timer, on every route, forever; and three widget endpoints called it on each poll. It now reads the 65 per-channel snapshots it could always have read — the same numbers, **68 ms**, a 59× improvement — and the sidebar badge itself is down to ~40 ms. Loading `/channels` went from 4.5 s to roughly a tenth of a second; the dashboard from ~10 s. The corpus-walking function still exists under a name that says what it costs (`listChannelStatsFromDisk`) for the batch jobs that genuinely need ground truth, and a test now fails the build if it ever reappears anywhere the editor renders. **The honest trade:** the video, transcript and download counts on `/channels` and the dashboard now come from each channel's last generated report rather than from disk directly, so a job that just finished can take a moment — the snapshot scheduler's ~1 second debounce — to show up. Verified against the live corpus: those three counts match a full walk **exactly** on all 65 channels. The one field that doesn't is digest coverage, which reads 0 for the 11 channels whose reports predate per-engine digest counts until their next report refresh. `/channels` now prints how old the oldest report on the page is, rather than leaving you to assume the numbers are live. +- **Auto-refresh no longer refreshes when nothing has changed.** The passive refresher called `router.refresh()` on a timer — a full server-side re-render of the entire page tree, every 5 seconds, on every route, whether or not anything had actually happened. Against the real corpus that was ~4.9 seconds of work per tick, and it held the editor's server process at roughly **22% of a CPU core, permanently, with a single idle tab open**. It now asks a new `/api/pulse` endpoint whether anything moved — a change token built from in-memory job, queue and worker state plus two file timestamps, no corpus reads at all — and re-renders only when the answer is yes. An idle page now performs **zero** re-renders where it used to perform one every five seconds; there's a test that fails if that ever regresses. Same setting, same 5-second default, same "0 disables" behaviour, and editing settings still repaints the sidebar immediately, because the settings file's timestamp is part of the token. The sidebar's job and reclaimable-disk pills now update on their own rather than requiring the whole page to re-render — which in turn let fifteen server actions stop invalidating the client's entire navigation cache to announce that a job count had changed. Also fixed while in here: opening a channel with no report used to **generate one inside the page load**, a full analysis of every video directory in that channel — minutes, on the big ones, with no progress and no way to stop it. It now offers a button and runs it as a normal background job. - **A page that throws no longer takes the whole editor with it, and moving between pages is instant.** There were **zero** error boundaries in the editor: anything that threw while rendering — a malformed config, a half-written snapshot — blanked the entire document, sidebar and all, with nothing to click and nothing to read. There is now a route error boundary that keeps the chrome alive, shows the error's digest so you can find it in the server log, and offers *Try again* (Next 16.2's `unstable_retry`, which actually re-fetches, rather than the older `reset`, which only clears the error state). Alongside it: `loading.tsx` skeletons for the routes with the most to render, and a 15-second client router cache (`staleTimes`), which is what makes bouncing between two sidebar links immediate instead of a fresh server round trip each way. One honest note, since it's easy to assume otherwise: in Next 16 `loading.tsx` does **not** guarantee a fallback appears during a client-side navigation — the framework's own reference says so, and testing confirmed it. Fast sidebar navigation here comes from prefetching plus that cache, not from the skeletons; the skeletons cover document loads. - **Cleaning audio now checks the video still exists upstream, and keeps it forever if it doesn't.** The transcribed-audio sweep hard-deletes a video's `audio.*` files once whisper has produced a transcript — `remove()`, no trash, no undo — and nothing had ever asked whether the video was still *there*. So a video YouTube had since removed, privated, or put behind a membership, sitting outside the keep-latest window, got its source audio deleted precisely when that local copy had become the only copy. Before deleting anything, the sweep now resolves each candidate's availability and writes a `do-not-clean.json` marker on any video found permanently gone (`deleted` / `private` / `members_only` — the same rule the keep-latest deletion pass uses, now shared as `isPermanentlyGone`), protecting it from this and every future sweep. The check is **cheap-first, not one probe per video**: a cached availability verdict costs nothing and is the only tier that catches `members_only` (a members-only video stays listed in its channel's playlist, so a listing diff can never flag it); then **one** flat-playlist call per channel narrows the field to candidates that have dropped out of the listing; only those few get a per-video probe, which is also what distinguishes a deleted video from an *unlisted* one that legitimately left the listing and is still fetchable by URL. Anything the check cannot resolve — a probe error, an age-gate, a video with no URL to probe — is **left alone with no marker written** and retried next run: the sweep never deletes on incomplete information, and a rate-limited or offline source therefore cleans nothing rather than cleaning wrongly. The summary line breaks the total down (`Skipped 4 (0 protected, 3 gone-from-source pinned, 1 unverified)`) whenever the check acted. On by default; **Check availability before cleaning audio** in Settings turns it off for an offline setup or channels with no URL, where the check can never resolve and cleanup would otherwise stop deleting anything. Two related fixes ride along: the sweep now shares `isRealAudioFile` with the rest of the app instead of its own hand-rolled filter, so it no longer deletes the `audio.live_chat.json` sidecar or the `.part.good`/`.part.testing` audio-check snapshots (which the reclaim estimate never counted, so the two had quietly drifted); and `runAvailabilityCheck` gains `ignoreShard`, because a saved shard slice on disk would otherwise replace an explicit `onlyIds` list wholesale. Scoped to the primary sweep only — the wrong-format, extra-format and auto-sub purges are unchanged, as are the explicit per-video deletes, which still ignore markers deliberately. See `common/controller/verifyBeforeClean.ts`, `common/controller/cleanAudioFromTranscribed.ts`, `common/lib/availability.ts`, and `editor/e2e/pre-clean-availability.spec.ts`. - **The monitor widget can now reclaim disk, not just report it.** The widget's cleanable-data strip showed a single global number ("4.2 GB reclaimable") with nothing to act on — reclaiming it meant leaving the widget for `/cleanup` or `/actionable`. A new opt-in **"Needs cleaning"** section (URL flag `cleanlist=1`, plus a **Channels needing cleanup** checkbox in the builder and the in-widget gear) lists the channels actually holding that audio, each with its reclaim estimate (`⌫ 2.5 MB`, the video count in the tooltip), capped at 6 channels with a `+N more` line like the needs-work list. With `controls=1` each row gains the same per-channel **Clean audio** button as the `/actionable` page — the existing `window.confirm` still guards the delete — so a pinned interactive widget clears disk pressure the way it already clears a download backlog. This is also the first surface on which a channel that is *fully downloaded and transcribed* but still holding reclaimable audio is actionable: the needs-work list is fed by a backlog route with a download/transcribe precondition, so such a channel never appeared there. It costs no extra polling — the per-channel rows come from the same `/api/widget/cleanable` snapshot read that already backed the total, and the total is now a sum over those rows so the section and the strip above it can't disagree. The dashboard's needs-work panel and its shared route are untouched. See `editor/app/cleanup/lib/loadCleanup.ts` (`cleanableChannels`), `editor/app/api/widget/cleanable/route.ts`, `editor/app/widget/{lib/config.ts,components/{MonitorWidget,WidgetConfigForm}.tsx}`, and `editor/e2e/widget.spec.ts`. diff --git a/editor/app/api/pulse/route.ts b/editor/app/api/pulse/route.ts @@ -0,0 +1,128 @@ +import { createHash } from "node:crypto"; +import { statSync } from "node:fs"; +import { NextResponse } from "next/server"; +import { getPaths } from "yt-dlp-transcript-common/lib/paths"; +import { getRegistry } from "yt-dlp-transcript-common/jobs/registry"; +import { getScheduler } from "yt-dlp-transcript-common/jobs/scheduler"; +import { getWorkerPool } from "yt-dlp-transcript-common/jobs/workerPool"; +import { cleanableTotalBytes } from "../../cleanup/lib/loadCleanup"; + +export const dynamic = "force-dynamic"; + +// The change token the global AutoRefresh polls instead of blindly re-rendering +// the whole page tree every 5 seconds. +// +// The old refresher called router.refresh() on a timer, on every route, whether +// or not anything had changed — a full server re-render that cost ~4.9 s and +// held next-server at ~22% of a core forever with a single idle tab open. This +// endpoint answers "has anything changed?" from IN-MEMORY state plus two +// stat() calls, so the idle path touches no corpus data at all. +export type PulsePayload = { + // Opaque token. Identical to your last one means nothing has changed and + // there is nothing to re-render. + rev: string; + // True when this response carries the full body below. False is the idle + // path: the client sent a rev that still matches, so nothing else was + // computed. + changed: boolean; + // Sidebar badge counts, so the sidebar never needs a tree refresh to update. + activeJobs: number; + runningJobs: number; + cleanableBytes: number; + // Anything live right now — lets the client poll faster while work is moving. + busy: boolean; +}; + +// Everything here is either in-memory or a stat(). NO readdir, no corpus +// contact, no JSON parsing of anything large — asserted by a test, because the +// entire point of this endpoint is that it is cheap enough to poll. +// +// The singletons are read LIVE inside the request rather than captured at +// module scope: e2e/helpers.ts's resetData clears them via +// /api/test/invalidate-cache, and a closed-over reference would keep polling a +// registry the tests had already thrown away. +function computeRev(): { rev: string; activeJobs: number; runningJobs: number; busy: boolean } { + const paths = getPaths(); + const jobs = getRegistry().list(); + + let activeJobs = 0; + let runningJobs = 0; + const parts: string[] = []; + for (const j of jobs) { + if (j.status === "running" || j.status === "queued") activeJobs++; + if (j.status === "running") runningJobs++; + // JobRecord has no single updatedAt, so the token is built from the fields + // that actually move: lifecycle timestamps, drain state, and progress. If a + // job's progress advances, pages showing that progress should re-render — + // that is a change, not noise. + parts.push( + [ + j.id, + j.status, + j.startedAt ?? "", + j.endedAt ?? "", + j.draining ? "d" : "", + j.progress + ? `${j.progress.initial}/${j.progress.current ?? ""}/${j.progress.target}` + : "", + j.tasks?.length ?? 0, + ].join(":"), + ); + } + + // Queue shape and worker-pool state, both in-memory. + const scheduler = getScheduler(); + for (const q of scheduler.queues()) { + parts.push(`q:${q.name}:${q.running.join(",")}:${q.queued.join(",")}`); + } + const pool = getWorkerPool(); + const workers = pool.summary(); + parts.push(`w:${pool.isPaused() ? "paused" : "live"}:${workers.length}`); + for (const w of workers) parts.push(`w:${w.id}:${w.busy ? 1 : 0}`); + + // Files the layout renders from. mtime only — neither is read here. + parts.push(`s:${mtime(paths.settingsFile)}`); + parts.push(`c:${mtime(paths.editorChangelogFile)}`); + + const rev = createHash("sha1").update(parts.join("|")).digest("base64url"); + return { rev, activeJobs, runningJobs, busy: runningJobs > 0 || activeJobs > 0 }; +} + +function mtime(file: string): number { + try { + return statSync(file).mtimeMs; + } catch { + return 0; + } +} + +export async function GET(request: Request) { + const known = new URL(request.url).searchParams.get("rev"); + const { rev, activeJobs, runningJobs, busy } = computeRev(); + + // Idle fast path. The client already has this rev, so nothing it displays can + // have changed — skip the only part of this endpoint that touches disk. + if (known && known === rev) { + return NextResponse.json({ + rev, + changed: false, + activeJobs, + runningJobs, + cleanableBytes: -1, // not recomputed; the client keeps its last value + busy, + } satisfies PulsePayload); + } + + // Something moved: it's worth the ~40 ms to refresh the badge value too. + // (Before the corpus walk was removed this number cost ~4.4 s, which is + // precisely why it must never be on the idle path.) + const cleanableBytes = await cleanableTotalBytes(getPaths()); + return NextResponse.json({ + rev, + changed: true, + activeJobs, + runningJobs, + cleanableBytes, + busy, + } satisfies PulsePayload); +} diff --git a/editor/app/auto-queue/actions.ts b/editor/app/auto-queue/actions.ts @@ -131,6 +131,5 @@ export async function prioritizeChannelDownloadAction( } await startAutoRunner("download"); revalidatePath("/auto-queue"); - revalidatePath("/"); return { ok: true }; } diff --git a/editor/app/build/buildAction.ts b/editor/app/build/buildAction.ts @@ -46,7 +46,6 @@ export async function buildIndexAction( paths, fn: async (onLog) => { await buildIndex({ paths, onLog }); - revalidatePath("/"); }, }); return result; diff --git a/editor/app/channels/[slug]/components/NoReportYet.tsx b/editor/app/channels/[slug]/components/NoReportYet.tsx @@ -0,0 +1,47 @@ +import Link from "next/link"; +import { InlineActionButton } from "../../../actionable/components/InlineActionButton"; + +// Shown when a channel has no generated report yet. +// +// This page used to GENERATE the snapshot inline when one was missing — a full +// channel analysis, walking every video directory, inside a GET. On the largest +// channel here that is a multi-minute page load triggered by nothing more than +// clicking a link, with no progress and no way to cancel it. Worse, it made the +// expensive path the invisible one: the page looked slow rather than looking +// like it was doing a job. +// +// So the page now asks. "Refresh report" queues the same refresh-report job +// every other surface uses, it shows up in /jobs like any other work, and the +// page fills in once it lands. +export function NoReportYet({ slug, name }: { slug: string; name: string }) { + return ( + <div className="flex flex-col gap-4"> + <div className="flex items-center gap-2 text-sm text-muted-foreground"> + <Link href="/channels" className="underline hover:text-foreground"> + Channels + </Link> + <span>/</span> + <span>{slug}</span> + </div> + <h1 className="text-2xl font-semibold">{name}</h1> + <div className="rounded-lg border border-dashed border-border p-6 flex flex-col gap-3 items-start"> + <h2 className="text-lg font-medium">No report yet</h2> + <p className="text-sm text-muted-foreground max-w-prose"> + This channel hasn’t been analysed yet, so there are no buckets, + counts or cleanup estimates to show. Generating one reads every video + directory in the channel — on a large channel that takes a while, so + it runs as a background job rather than holding this page open. + </p> + <InlineActionButton variant={{ kind: "refreshReport", slug }} /> + <p className="text-xs text-muted-foreground"> + Follow it on the{" "} + <Link href="/jobs" className="underline"> + Jobs + </Link>{" "} + page — it’s a <code>refresh-report</code> job. Reload here when + it finishes. + </p> + </div> + </div> + ); +} diff --git a/editor/app/channels/[slug]/page.tsx b/editor/app/channels/[slug]/page.tsx @@ -16,9 +16,9 @@ import "yt-dlp-transcript-common/social/blueskyFetcher"; import "yt-dlp-transcript-common/social/xGalleryDlFetcher"; import "yt-dlp-transcript-common/social/xPlaywrightFetcher"; import { SocialChannelPanel } from "./components/SocialChannelPanel"; +import { NoReportYet } from "./components/NoReportYet"; import { excludedDownloadIdSet, - generateChannelSnapshot, normalizeAvailability, normalizeExcludedFromDownload, normalizeMaybeMissing, @@ -223,8 +223,17 @@ export default async function ChannelDetailPage({ const platformDefaultQueueKey = config.platform ? platformQueueKey(config.platform) : queueKeyForUrl(config.url); - const existing = await readChannelSnapshot(paths, slug); - const snapshot = existing ?? (await generateChannelSnapshot(paths, slug)); + // ⚠️ This used to be `existing ?? await generateChannelSnapshot(paths, slug)` + // — a full channel analysis run INSIDE A GET. On the 11,224-video channel + // that is a multi-minute request that walks every video directory, and the + // only thing that triggers it is opening a page. A render must never + // generate; it reads what's there and offers to generate what isn't. + const snapshot = await readChannelSnapshot(paths, slug); + if (!snapshot) { + return ( + <NoReportYet slug={slug} name={config.name ?? slug} /> + ); + } // Saved-video store summary for this channel + whether backups are configured, // for the Cleanup stage's Retention & persistence section (Phase 5). const savedTotals = await savedVideoTotals({ paths, channelSlug: slug }); @@ -542,7 +551,10 @@ export default async function ChannelDetailPage({ slug={slug} channelName={config.name ?? slug} generatedAt={snapshot.generatedAt} - fresh={!existing} + // Always false now: this page no longer generates a snapshot during the + // render, so one can never have been produced a moment ago. The header + // shows `generatedAt` instead, which is the honest signal anyway. + fresh={false} stages={stages} transcodeApplies={transcodeApplies} /> diff --git a/editor/app/components/AutoRefresh.tsx b/editor/app/components/AutoRefresh.tsx @@ -1,46 +1,107 @@ "use client"; import { useEffect, useRef, useTransition } from "react"; -import { useRouter } from "next/navigation"; +import { usePathname, useRouter, useSearchParams } from "next/navigation"; +import { configurePulse, getRevCounter, subscribePulse } from "./pulse"; type Props = { // Refresh cadence in milliseconds. 0 (or negative) disables passive refresh. intervalMs: number; }; -// Mounted once globally in the root layout. Periodically calls router.refresh() -// so every server-rendered page (and the shared sidebar badges) reconciles fresh -// data without a manual reload. Idle guards: it never refreshes while the tab is -// hidden, does one immediate refresh when the tab becomes visible again, and -// skips a tick while a previous refresh is still settling. +// Routes that display nothing a server re-render could update. The monitor +// widget is embedded chrome-less in a pinned window and re-fetches every +// payload it shows on its own cadence, so a tree refresh there is pure waste. +// +// The dashboard ("/") is a deliberate NON-member, despite also polling its own +// endpoints: the sidebar's admin title is rendered by the ROOT LAYOUT from +// settings.json, and a tree refresh is the only thing that can update it. +// Excluding "/" would silently break that — it is the exact behaviour +// e2e/auto-refresh.spec.ts pins ("re-fetches server data on the interval +// without a reload", asserted on "/"). The cost of leaving it in is now +// approximately nothing anyway: a refresh only happens when the pulse token +// has actually moved, so an idle dashboard never re-renders either way. +// +// A predicate rather than an allow-list of every route, so a NEW page defaults +// to refreshing (the safe direction) instead of silently going stale. +function selfPolling(pathname: string): boolean { + return pathname.startsWith("/widget"); +} + +// Don't START a tree refresh within this long of a navigation. A refresh +// landing on top of an in-flight navigation fights it for the same router +// state, and the user sees the destination flicker or stall. +const NAV_QUIET_MS = 1000; + +// Mounted once globally in the root layout. +// +// It used to call router.refresh() unconditionally on a timer: a full +// server-side re-render of every page, every 5 seconds, on every route, whether +// or not anything had changed. Against the real corpus that was ~4.9 s of work +// per tick and held next-server at ~22% of a core forever with one idle tab +// open. +// +// Now it asks /api/pulse — an in-memory change token, no corpus contact — and +// only refreshes when the token actually moves. At idle that is one small +// request per interval and ZERO renders. +// +// The user-facing contract is deliberately unchanged: same setting, same 5 s +// default, same 0-disables semantics. Editing settings.json still repaints, +// because the settings file's mtime is part of the token. export function AutoRefresh({ intervalMs }: Props) { const router = useRouter(); + const pathname = usePathname(); + const searchParams = useSearchParams(); const [isPending, startTransition] = useTransition(); - // Read pending state from a ref inside the interval so the effect doesn't have - // to re-subscribe (and reset its timer) on every transition state change. + + // Read through refs inside the subscription so it doesn't have to re-subscribe + // (and restart the shared poller) on every transition or route change. const isPendingRef = useRef(isPending); isPendingRef.current = isPending; + const refreshEnabledRef = useRef(true); + refreshEnabledRef.current = !selfPolling(pathname); + + // Timestamp of the last navigation, so a refresh never collides with one. + const lastNavAt = useRef(0); + useEffect(() => { + lastNavAt.current = Date.now(); + }, [pathname, searchParams]); + + useEffect(() => { + configurePulse(intervalMs); + }, [intervalMs]); useEffect(() => { if (!Number.isFinite(intervalMs) || intervalMs <= 0) return; - const tick = () => { - if (typeof document !== "undefined" && document.hidden) return; - if (isPendingRef.current) return; // a refresh is already in flight + // Only fires when the pulse token actually changed. + let seen = getRevCounter(); + const unsubscribe = subscribePulse(() => { + const counter = getRevCounter(); + if (counter === seen) return; // nothing changed — nothing to re-render + + // Every guard below RETURNS WITHOUT ADVANCING `seen`, so a change that is + // deferred is retried on the next tick rather than swallowed. Advancing + // the marker before the guards would drop the update permanently — e.g. a + // settings edit landing during the post-navigation quiet window would + // simply never be applied. + if (!refreshEnabledRef.current) { + // This route doesn't use the tree refresh at all, so there is nothing + // to defer to — consume it and move on. + seen = counter; + return; + } + if (isPendingRef.current) return; // a refresh is already settling + // Keep polling during the quiet window (so the token stays current) but + // don't start a refresh on top of an in-flight navigation. + if (Date.now() - lastNavAt.current < NAV_QUIET_MS) return; + + seen = counter; startTransition(() => router.refresh()); - }; - - const onVisibility = () => { - // Catch up immediately when the user returns to the tab. - if (!document.hidden) tick(); - }; - - const id = setInterval(tick, intervalMs); - document.addEventListener("visibilitychange", onVisibility); - return () => { - clearInterval(id); - document.removeEventListener("visibilitychange", onVisibility); - }; + }); + + return unsubscribe; + // `router` is stable; startTransition is stable. }, [intervalMs, router]); return null; diff --git a/editor/app/components/CleanableBadge.tsx b/editor/app/components/CleanableBadge.tsx @@ -1,6 +1,6 @@ import { getPaths } from "yt-dlp-transcript-common/lib/paths"; -import { formatBytes } from "yt-dlp-transcript-common/lib/format"; import { cleanableTotalBytes } from "../cleanup/lib/loadCleanup"; +import { CleanableBadgeLive } from "./SidebarBadges"; // The sidebar's reclaimable-disk pill, as its own async component so the layout // can stream it behind a Suspense boundary. @@ -18,16 +18,10 @@ import { cleanableTotalBytes } from "../cleanup/lib/loadCleanup"; // rendering". This is hardening rather than a fix: after the corpus walk was // removed from cleanableTotalBytes it costs ~40 ms, not ~4.4 s. The point is // that it can never put the whole document behind it again. +// The server render produces the SEED; CleanableBadgeLive keeps it current from +// the shared pulse poller thereafter, so this number no longer depends on the +// layout being re-rendered. export async function CleanableBadge() { const bytes = await cleanableTotalBytes(getPaths()); - if (bytes <= 0) return null; - return ( - <span - data-testid="cleanable-badge" - aria-label={`${formatBytes(bytes)} reclaimable`} - className="ml-auto text-xs rounded-full border border-warning/30 bg-warning-soft text-warning px-2 py-0.5 leading-none tabular-nums" - > - {formatBytes(bytes)} - </span> - ); + return <CleanableBadgeLive seed={bytes} />; } diff --git a/editor/app/components/ShardControl.tsx b/editor/app/components/ShardControl.tsx @@ -1,6 +1,5 @@ "use client"; -import { useRouter } from "next/navigation"; import { useState, useTransition } from "react"; import type { ShardOp } from "yt-dlp-transcript-common/controller/shard"; import { @@ -36,7 +35,6 @@ export function ShardControl({ index, onIndexChange, }: Props) { - const router = useRouter(); const [clearing, startTransition] = useTransition(); const [saving, startSaveTransition] = useTransition(); const [error, setError] = useState<string | null>(null); @@ -59,7 +57,6 @@ export function ShardControl({ setError(r.error); return; } - router.refresh(); }); } @@ -78,7 +75,6 @@ export function ShardControl({ setError(r.error); return; } - router.refresh(); }); } diff --git a/editor/app/components/SidebarBadges.tsx b/editor/app/components/SidebarBadges.tsx @@ -0,0 +1,58 @@ +"use client"; + +import { formatBytes } from "yt-dlp-transcript-common/lib/format"; +import { usePulse } from "./pulse"; + +// The sidebar's live pills — active/running job counts and reclaimable disk. +// +// These used to be plain server-rendered numbers, which meant the ONLY way to +// update them was to re-render the entire page tree. That is what made the +// global auto-refresh re-render everything every five seconds: the badges were +// hostage to it. Now they update from the shared /api/pulse poller, which is +// what lets the tree refresh be skipped entirely on routes that don't need it. +// +// Each is seeded from the server render so the first paint is correct and there +// is no flash of zero before the first poll lands. + +export function JobsBadge({ + seed, + metric, +}: { + seed: number; + metric: "active" | "running"; +}) { + const pulse = usePulse(); + // Until the first poll completes there is no rev, so trust the server seed. + const count = + pulse.rev === null + ? seed + : metric === "active" + ? pulse.activeJobs + : pulse.runningJobs; + if (count <= 0) return null; + const noun = metric === "running" ? "running job" : "active job"; + return ( + <span + data-testid={`badge-${metric}`} + aria-label={`${count} ${noun}${count === 1 ? "" : "s"}`} + className="ml-auto text-xs rounded-full border border-brand/30 bg-brand-soft text-brand px-2 py-0.5 leading-none" + > + {count} + </span> + ); +} + +export function CleanableBadgeLive({ seed }: { seed: number }) { + const pulse = usePulse(); + const bytes = pulse.rev === null ? seed : pulse.cleanableBytes; + if (bytes <= 0) return null; + return ( + <span + data-testid="cleanable-badge" + aria-label={`${formatBytes(bytes)} reclaimable`} + className="ml-auto text-xs rounded-full border border-warning/30 bg-warning-soft text-warning px-2 py-0.5 leading-none tabular-nums" + > + {formatBytes(bytes)} + </span> + ); +} diff --git a/editor/app/components/pulse.ts b/editor/app/components/pulse.ts @@ -0,0 +1,137 @@ +"use client"; + +import { useEffect, useState } from "react"; +import type { PulsePayload } from "../api/pulse/route"; + +// One poller for the whole app, shared through a module-level store. +// +// AutoRefresh and every sidebar badge want the same ~2 ms answer to "has +// anything changed?", and N components each running their own interval would +// reintroduce the polling cost this endpoint exists to remove. So the poller is +// a singleton: the first subscriber starts it, the last one to leave stops it. +// +// Deliberately not React Context — the badges live inside the SERVER-rendered +// sidebar in the root layout, which cannot be wrapped in a client provider +// without making the whole layout a client component. + +export type PulseState = { + rev: string | null; + activeJobs: number; + runningJobs: number; + cleanableBytes: number; + busy: boolean; +}; + +type Listener = (state: PulseState) => void; + +let state: PulseState = { + rev: null, + activeJobs: 0, + runningJobs: 0, + cleanableBytes: 0, + busy: false, +}; +const listeners = new Set<Listener>(); +let timer: ReturnType<typeof setTimeout> | null = null; +let inFlight = false; +let intervalMs = 5000; +// Bumped whenever `rev` changes, so components can react to "something changed" +// without diffing the token themselves. +let revCounter = 0; + +export function getPulseState(): PulseState { + return state; +} + +export function getRevCounter(): number { + return revCounter; +} + +function emit() { + for (const l of listeners) l(state); +} + +async function poll(): Promise<void> { + if (inFlight) return; + if (typeof document !== "undefined" && document.hidden) return; + inFlight = true; + try { + const url = state.rev + ? `/api/pulse?rev=${encodeURIComponent(state.rev)}` + : "/api/pulse"; + const res = await fetch(url, { cache: "no-store" }); + if (!res.ok) return; + const data = (await res.json()) as PulsePayload; + const revChanged = data.rev !== state.rev; + state = { + rev: data.rev, + activeJobs: data.activeJobs, + runningJobs: data.runningJobs, + // -1 is the idle path's "not recomputed" marker; keep the last known + // value rather than flashing the badge to zero. + cleanableBytes: + data.cleanableBytes >= 0 ? data.cleanableBytes : state.cleanableBytes, + busy: data.busy, + }; + if (revChanged) revCounter++; + emit(); + } catch { + // A failed poll is not worth surfacing; the next tick tries again. + } finally { + inFlight = false; + } +} + +function schedule() { + if (timer) clearTimeout(timer); + if (intervalMs <= 0) return; + timer = setTimeout(async () => { + await poll(); + schedule(); + }, intervalMs); +} + +export function configurePulse(ms: number): void { + intervalMs = Number.isFinite(ms) ? ms : 0; + if (listeners.size > 0) schedule(); +} + +function start() { + if (timer || intervalMs <= 0) return; + void poll(); + schedule(); +} + +function stop() { + if (timer) clearTimeout(timer); + timer = null; +} + +export function subscribePulse(listener: Listener): () => void { + listeners.add(listener); + if (listeners.size === 1) { + start(); + // Catch up the moment the tab comes back, the same guard the old refresher + // had — a tab left in the background for an hour should not show hour-old + // numbers for a further five seconds. + document.addEventListener("visibilitychange", onVisibility); + } + return () => { + listeners.delete(listener); + if (listeners.size === 0) { + stop(); + document.removeEventListener("visibilitychange", onVisibility); + } + }; +} + +function onVisibility() { + if (!document.hidden) void poll(); +} + +// Subscribe a component to the shared poller. +export function usePulse(): PulseState { + const [local, setLocal] = useState<PulseState>(state); + useEffect(() => subscribePulse(setLocal), []); + return local; +} diff --git a/editor/app/jobs/actions.ts b/editor/app/jobs/actions.ts @@ -160,7 +160,6 @@ async function setDownloadsPaused( } catch (e) { return { ok: false, error: (e as Error).message }; } - revalidatePath("/"); revalidatePath("/jobs"); return { ok: true }; } @@ -198,7 +197,6 @@ async function setDigestsPaused(paused: boolean): Promise<DigestPauseResult> { } catch (e) { return { ok: false, error: (e as Error).message }; } - revalidatePath("/"); revalidatePath("/jobs"); return { ok: true }; } @@ -219,7 +217,6 @@ export type DigestSweepResult = { ok: boolean; jobId?: string; error?: string }; export async function startDigestSweepAction(): Promise<DigestSweepResult> { try { const jobId = await startDigestSweep(); - revalidatePath("/"); revalidatePath("/jobs"); return jobId ? { ok: true, jobId } @@ -232,7 +229,6 @@ export async function startDigestSweepAction(): Promise<DigestSweepResult> { export async function stopDigestSweepAction(): Promise<DigestSweepResult> { try { await stopDigestSweep(); - revalidatePath("/"); revalidatePath("/jobs"); return { ok: true }; } catch (e) { diff --git a/editor/app/jobs/components/ClearLogsMenu.tsx b/editor/app/jobs/components/ClearLogsMenu.tsx @@ -1,7 +1,6 @@ "use client"; import { useState, useTransition } from "react"; -import { useRouter } from "next/navigation"; import { ChevronDown } from "lucide-react"; import { DropdownMenu, @@ -23,14 +22,12 @@ const OPTIONS: { scope: ClearLogsScope; label: string }[] = [ // pruneJobLogs). The directory also self-trims on job finish, so this is the // manual escape hatch. export function ClearLogsMenu() { - const router = useRouter(); const [pending, startTransition] = useTransition(); const [open, setOpen] = useState(false); function clear(scope: ClearLogsScope): void { startTransition(async () => { await clearFinishedLogsAction(scope); - router.refresh(); }); } diff --git a/editor/app/jobs/components/ForceReleaseJobButton.tsx b/editor/app/jobs/components/ForceReleaseJobButton.tsx @@ -1,7 +1,6 @@ "use client"; import { useState } from "react"; -import { useRouter } from "next/navigation"; import { forceReleaseJobAction } from "../actions"; type Props = { @@ -13,7 +12,6 @@ type Props = { // destructive styling (vs. Cancel's outline) to read as the heavier action. export function ForceReleaseJobButton({ jobId }: Props) { const [busy, setBusy] = useState(false); - const router = useRouter(); return ( <button type="button" @@ -27,7 +25,6 @@ export function ForceReleaseJobButton({ jobId }: Props) { setBusy(true); try { await forceReleaseJobAction(jobId); - router.refresh(); } finally { setBusy(false); } diff --git a/editor/app/jobs/components/RetryAllFailedButton.tsx b/editor/app/jobs/components/RetryAllFailedButton.tsx @@ -1,14 +1,12 @@ "use client"; import { useState } from "react"; -import { useRouter } from "next/navigation"; import { retryAllFailedAction } from "../actions"; // Re-run every currently-listed failed job that has a replay descriptor. Shown // only when there's at least one such job (the page decides). export function RetryAllFailedButton() { const [busy, setBusy] = useState(false); - const router = useRouter(); return ( <button type="button" @@ -17,7 +15,6 @@ export function RetryAllFailedButton() { setBusy(true); try { await retryAllFailedAction(); - router.refresh(); } finally { setBusy(false); } diff --git a/editor/app/jobs/components/RetryJobButton.tsx b/editor/app/jobs/components/RetryJobButton.tsx @@ -1,7 +1,6 @@ "use client"; import { useState } from "react"; -import { useRouter } from "next/navigation"; import { retryJobAction } from "../actions"; type Props = { @@ -13,7 +12,6 @@ type Props = { export function RetryJobButton({ jobId }: Props) { const [busy, setBusy] = useState(false); const [note, setNote] = useState<string | null>(null); - const router = useRouter(); return ( <span className="inline-flex items-center gap-1"> <button @@ -25,7 +23,6 @@ export function RetryJobButton({ jobId }: Props) { try { const res = await retryJobAction(jobId); if (!res.ok) setNote(res.error); - router.refresh(); } finally { setBusy(false); } diff --git a/editor/app/jobs/queue/buildQueueView.ts b/editor/app/jobs/queue/buildQueueView.ts @@ -1,5 +1,6 @@ import path from "node:path"; -import { readFile } from "node:fs/promises"; +import { open } from "node:fs/promises"; +import type { FileHandle } from "node:fs/promises"; import { getRegistry } from "yt-dlp-transcript-common/jobs/registry"; import { getScheduler } from "yt-dlp-transcript-common/jobs/scheduler"; import { getWorkerPool } from "yt-dlp-transcript-common/jobs/workerPool"; @@ -48,23 +49,44 @@ export type QueueViewPayload = { stuckIds: string[]; }; -// Cheap best-effort tail: read the log file and return its last non-empty line. -// Swallows every error (missing file, read failure) — this is diagnostic sugar, -// never load-bearing. Mirrors the readLogChunk helper the job log route uses, -// but keeps only the final line. +// How much of the log tail to read for the one line we want. A transcription +// job's log runs to megabytes and this used to read all of it, per slot, per +// poll, to display its final line. +const TAIL_BYTES = 8 * 1024; + +// Cheap best-effort tail: read the END of the log file and return its last +// non-empty line. Swallows every error (missing file, read failure) — this is +// diagnostic sugar, never load-bearing. +// +// Reading a fixed window from the end means a log whose last 8 KB is entirely +// blank yields nothing rather than scanning back further; that is the intended +// trade for a status line, and the first (possibly truncated) line in the +// window is discarded so a partial line is never displayed as a whole one. async function readLastLogLine( jobsDir: string, id: string, ): Promise<string | undefined> { + const file = path.join(jobsDir, `${id}.log`); + let handle: FileHandle | undefined; try { - const raw = await readFile(path.join(jobsDir, `${id}.log`), "utf8"); - const lines = raw.split("\n"); - for (let i = lines.length - 1; i >= 0; i--) { + handle = await open(file, "r"); + const { size } = await handle.stat(); + const start = Math.max(0, size - TAIL_BYTES); + const length = size - start; + if (length <= 0) return undefined; + const buf = Buffer.alloc(length); + await handle.read(buf, 0, length, start); + const lines = buf.toString("utf8").split("\n"); + // Mid-file window: drop the leading fragment, which may be half a line. + const from = start > 0 ? 1 : 0; + for (let i = lines.length - 1; i >= from; i--) { const line = lines[i].trim(); if (line) return line; } } catch { /* best-effort */ + } finally { + await handle?.close().catch(() => {}); } return undefined; } @@ -102,14 +124,19 @@ export async function buildQueueView(): Promise<QueueViewPayload> { let runningCount = 0; let queuedCount = 0; + // buildSlot is PURE: it reports whether the slot needs healing rather than + // pushing to toHeal/stuckIds itself. Every slot is built concurrently below, + // so a side effect in here would land in completion order — and the healing + // pass must stay deterministic. const buildSlot = async ( id: string, role: "running" | "queued", - ): Promise<QueueSlotView> => { + ): Promise<{ view: QueueSlotView; heal: boolean }> => { const rec = registry.get(id); const draining = rec?.draining === true; const lastLogLine = await readLastLogLine(paths.jobsDir, id); let stuck = false; + let heal = false; let stuckReason: string | undefined; if (role === "running") { @@ -118,11 +145,11 @@ export async function buildQueueView(): Promise<QueueViewPayload> { // running slot — a stuck slot with no live record to describe it. stuck = true; stuckReason = "record-evicted"; - toHeal.push(id); + heal = true; } else if (TERMINAL.has(rec.status)) { stuck = true; stuckReason = "terminal-but-holding-slot"; - toHeal.push(id); + heal = true; } else { const ageMs = now - (rec.startedAt ?? rec.queuedAt); const idle = !rec.tasks || rec.tasks.length === 0; @@ -135,8 +162,6 @@ export async function buildQueueView(): Promise<QueueViewPayload> { } } - if (stuck) stuckIds.push(id); - // For an evicted record fall back to the on-disk meta sidecar for a kind. let kind = rec?.kind; if (!kind) { @@ -147,38 +172,62 @@ export async function buildQueueView(): Promise<QueueViewPayload> { const ageMs = now - (rec?.startedAt ?? rec?.queuedAt ?? now); return { - id, - kind, - status: rec ? rec.status : "evicted", - ageMs, - stuck, - stuckReason, - pid: rec?.child?.pid, - draining, - lastLogLine, + view: { + id, + kind, + status: rec ? rec.status : "evicted", + ageMs, + stuck, + stuckReason, + pid: rec?.child?.pid, + draining, + lastLogLine, + }, + heal, }; }; + // Flatten every slot across every queue into one list, build them all at + // once, then reassemble in the original order. This used to be a serial + // `await` per slot, each of which read a whole job log. + type SlotRef = { queue: number; role: "running" | "queued"; id: string }; + const refs: SlotRef[] = []; + for (const [queue, v] of views.entries()) { + for (const id of v.running) refs.push({ queue, role: "running", id }); + for (const id of v.queued) refs.push({ queue, role: "queued", id }); + } + const built = await Promise.all( + refs.map((r) => buildSlot(r.id, r.role)), + ); + const queues: QueueGroupView[] = []; + let cursor = 0; for (const v of views) { // The scheduler keeps concurrency 1 per queueKey, so there's at most one // running head — but iterate defensively in case that ever changes. - const [headId, ...extraRunning] = v.running; - const head = headId ? await buildSlot(headId, "running") : undefined; - if (head) runningCount++; const backlog: QueueSlotView[] = []; - // Any beyond-first running ids are shown in the backlog too (rare). - for (const id of extraRunning) { - backlog.push(await buildSlot(id, "running")); + let head: QueueSlotView | undefined; + for (let i = 0; i < v.running.length; i++) { + const { view } = built[cursor++]; + if (i === 0) head = view; + else backlog.push(view); runningCount++; } - for (const id of v.queued) { - backlog.push(await buildSlot(id, "queued")); + for (let i = 0; i < v.queued.length; i++) { + backlog.push(built[cursor++].view); queuedCount++; } queues.push({ name: v.name, head, backlog }); } + // Fold the per-slot findings in `refs` order, so the healing pass and the + // stuck list are identical run to run regardless of which read finished first. + for (const [i, r] of refs.entries()) { + const { view, heal } = built[i]; + if (view.stuck) stuckIds.push(r.id); + if (heal) toHeal.push(r.id); + } + // Auto-heal the hard cases now that they've been captured into the view: a // terminal/evicted record's slot MUST be freed. Idempotent and safe; the freed // slot (and any promoted successor) shows correctly on the next poll. diff --git a/editor/app/jobs/queue/components/QueueViewLive.tsx b/editor/app/jobs/queue/components/QueueViewLive.tsx @@ -2,7 +2,6 @@ import Link from "next/link"; import { useEffect, useState } from "react"; -import { useRouter } from "next/navigation"; import { formatDuration } from "yt-dlp-transcript-common/lib/format"; import { jobKindLabel } from "../../jobKindLabels"; import { reapStuckJobsAction } from "../../actions"; @@ -66,7 +65,6 @@ export function QueueViewLive({ initial }: { initial: QueueViewPayload }) { } function HealthStrip({ payload }: { payload: QueueViewPayload }) { - const router = useRouter(); const [reaping, setReaping] = useState(false); const [copied, setCopied] = useState(false); const { summary } = payload; @@ -114,7 +112,6 @@ function HealthStrip({ payload }: { payload: QueueViewPayload }) { setReaping(true); try { await reapStuckJobsAction(); - router.refresh(); } finally { setReaping(false); } diff --git a/editor/app/layout.tsx b/editor/app/layout.tsx @@ -20,6 +20,7 @@ import { SiteScopeSelect } from "./components/SiteScopeSelect"; import { Toaster } from "yt-dlp-transcript-common/components/ui/sonner"; import { NAV_GROUPS, type NavLink } from "./lib/nav"; import { CleanableBadge } from "./components/CleanableBadge"; +import { JobsBadge } from "./components/SidebarBadges"; import "./globals.css"; export async function generateMetadata(): Promise<Metadata> { @@ -87,38 +88,37 @@ export default async function RootLayout({ <Link key={link.href} href={link.href} className={navItemClass}> <Icon className="size-4 shrink-0 text-muted-foreground" aria-hidden="true" /> <span>{link.label}</span> - {/* Streamed: the nav renders immediately and the pill fills in. See - CleanableBadge for why this boundary does nothing for sibling - navigation and everything for document loads. */} + {/* Streamed so the nav paints before the number is known, then kept + live by the shared /api/pulse poller rather than by re-rendering + this whole layout. See CleanableBadge for why the Suspense + boundary matters on document loads and does nothing for sibling + navigation. */} <Suspense fallback={null}> <CleanableBadge /> </Suspense> </Link> ); } - const badgeCount = - link.badgeKey === "jobs" - ? activeJobs - : link.badgeKey === "running" - ? runningJobs - : 0; - const badgeNoun = - link.badgeKey === "running" ? "running job" : "active job"; - const showBadge = - (link.badgeKey === "jobs" || link.badgeKey === "running") && - badgeCount > 0; + // Job pills update client-side from the same poller. Seeded from this + // render so the first paint is correct; after that they move on their own, + // which is what makes it safe to skip the tree refresh on routes that + // don't need one. + if (link.badgeKey === "jobs" || link.badgeKey === "running") { + return ( + <Link key={link.href} href={link.href} className={navItemClass}> + <Icon className="size-4 shrink-0 text-muted-foreground" aria-hidden="true" /> + <span>{link.label}</span> + <JobsBadge + seed={link.badgeKey === "jobs" ? activeJobs : runningJobs} + metric={link.badgeKey === "jobs" ? "active" : "running"} + /> + </Link> + ); + } return ( <Link key={link.href} href={link.href} className={navItemClass}> <Icon className="size-4 shrink-0 text-muted-foreground" aria-hidden="true" /> <span>{link.label}</span> - {showBadge && ( - <span - aria-label={`${badgeCount} ${badgeNoun}${badgeCount === 1 ? "" : "s"}`} - className="ml-auto text-xs rounded-full border border-brand/30 bg-brand-soft text-brand px-2 py-0.5 leading-none" - > - {badgeCount} - </span> - )} </Link> ); }; diff --git a/editor/app/sites/components/SiteListActions.tsx b/editor/app/sites/components/SiteListActions.tsx @@ -1,11 +1,9 @@ "use client"; import { useState, useTransition } from "react"; -import { useRouter } from "next/navigation"; import { deleteSiteAction, migrateToSitesAction } from "../actions"; export function DeleteSiteButton({ siteId }: { siteId: string }) { - const router = useRouter(); const [pending, start] = useTransition(); const [error, setError] = useState<string | null>(null); return ( @@ -18,7 +16,6 @@ export function DeleteSiteButton({ siteId }: { siteId: string }) { start(async () => { const res = await deleteSiteAction(siteId); if (!res.ok) setError(res.error); - else router.refresh(); }); }} className="px-2 py-1 rounded border border-destructive/30 text-xs text-destructive disabled:opacity-50" @@ -31,7 +28,6 @@ export function DeleteSiteButton({ siteId }: { siteId: string }) { } export function MigrateButton() { - const router = useRouter(); const [pending, start] = useTransition(); const [error, setError] = useState<string | null>(null); return ( @@ -43,7 +39,6 @@ export function MigrateButton() { start(async () => { const res = await migrateToSitesAction(); if (!res.ok) setError(res.error); - else router.refresh(); }) } className="px-3 py-2 rounded-md bg-primary text-primary-foreground text-sm font-medium hover:opacity-90 disabled:opacity-50" diff --git a/editor/app/workers/actions.ts b/editor/app/workers/actions.ts @@ -57,13 +57,20 @@ export async function drainWorkerAction(id: string): Promise<WorkerActionResult> return ok ? { ok: true } : { ok: false, error: `Unknown worker "${id}"` }; } +// NOTE — why there is no revalidatePath("/") in the pause/resume actions any +// more. The dashboard's paused state comes from its own polled /api/workers +// payload, not from a server render, so revalidating "/" changed nothing it +// displays. What it DID do is wipe the client router cache app-wide, which is +// what makes navigating back to a page you were just on instant (see +// experimental.staleTimes in next.config.ts). Same reasoning applies to the +// pause/sweep actions in ../jobs/actions.ts. + // Temporary pause-all: disable every worker, remembering each one's current // state. Running batches pause (they wait for a worker) rather than failing. export async function pauseAllWorkersAction(): Promise<WorkerActionResult> { getWorkerPool().pauseAll(); await persistTranscriptionsPaused(true); refresh(); - revalidatePath("/"); return { ok: true }; } @@ -72,7 +79,6 @@ export async function resumeAllWorkersAction(): Promise<WorkerActionResult> { getWorkerPool().resumeAll(); await persistTranscriptionsPaused(false); refresh(); - revalidatePath("/"); return { ok: true }; } diff --git a/editor/e2e/actionable.spec.ts b/editor/e2e/actionable.spec.ts @@ -1,7 +1,7 @@ import { mkdir, writeFile } from "node:fs/promises"; import { dirname } from "node:path"; import { test, expect } from "@playwright/test"; -import { resetData, resolvePath, readJson, jobRowByKind } from "./helpers"; +import { resetData, resolvePath, readJson, jobRowByKind, generateReport} from "./helpers"; const YT_SNAPSHOT_REL = "test-transcripts/channels/test-youtube/snapshot.json"; @@ -41,6 +41,7 @@ test("populates the undownloaded section after a snapshot exists", async ({ }) => { await resetData("youtube-with-playlist"); // Visiting the channel detail page auto-generates the snapshot. + await generateReport(page, "test-youtube"); await page.goto("/channels/test-youtube"); await page.goto("/actionable"); @@ -61,6 +62,7 @@ test("populates the undownloaded section after a snapshot exists", async ({ test("populates the awaiting-transcription section", async ({ page }) => { await resetData("one-transcribe-channel-with-audio"); + await generateReport(page, "test-transcribe"); await page.goto("/channels/test-transcribe"); await page.goto("/actionable"); @@ -154,6 +156,7 @@ test("syncing a channel auto-regenerates its report", async ({ page }) => { // Then trigger an actual pipeline action that completes locally — // Refresh report — and verify the snapshot's generatedAt advances. await resetData("one-transcribe-channel-with-audio"); + await generateReport(page, "test-transcribe"); await page.goto("/channels/test-transcribe"); const first = await readJson<{ generatedAt: string }>(TR_SNAPSHOT_REL); @@ -179,6 +182,7 @@ test("inline 'Download missing' queues a download-missing job", async ({ page, }) => { await resetData("youtube-with-playlist"); + await generateReport(page, "test-youtube"); await page.goto("/channels/test-youtube"); // auto-generate snapshot await page.goto("/actionable"); await page @@ -198,6 +202,7 @@ test("inline 'Transcribe pending' queues a whisper-all job", async ({ page, }) => { await resetData("one-transcribe-channel-with-audio"); + await generateReport(page, "test-transcribe"); await page.goto("/channels/test-transcribe"); // auto-generate snapshot await page.goto("/actionable"); await page @@ -224,6 +229,7 @@ test("dashboard surfaces the 'Needs work' card when work is pending", async ({ page, }) => { await resetData("youtube-with-playlist"); + await generateReport(page, "test-youtube"); await page.goto("/channels/test-youtube"); // auto-generate snapshot await page.goto("/"); // getByRole("region"), not getByLabel: the label match is a case-insensitive diff --git a/editor/e2e/auto-refresh.spec.ts b/editor/e2e/auto-refresh.spec.ts @@ -83,6 +83,62 @@ test.describe("auto-refresh behavior", () => { }); }); + // THE regression test for the auto-refresh rewrite. + // + // The refresher used to call router.refresh() unconditionally on every tick — + // a full server re-render of the page tree whether or not anything had + // changed, on every route, forever. It now polls a cheap change token first + // and only refreshes when that token moves. With nothing writing to the data + // directory, the correct number of tree refreshes over several seconds is + // exactly zero, while the pulse keeps ticking. + test("does not refresh the tree when nothing has changed", async ({ + page, + }) => { + await resetData("empty"); + await writeSettings({ + autoRefreshIntervalSeconds: 1, + adminTitle: "Quiet Title", + }); + + let rscRefreshes = 0; + let pulses = 0; + page.on("request", (req) => { + const url = new URL(req.url()); + if (url.pathname === "/api/pulse") { + pulses++; + return; + } + // A tree refresh is an RSC request carrying the router state tree and no + // prefetch marker. Prefetches are excluded deliberately: they are cheap, + // they are what makes navigation instant, and they are not what this test + // is about. + if (!url.searchParams.has("_rsc")) return; + const headers = req.headers(); + if (headers["next-router-prefetch"]) return; + if (headers["next-router-segment-prefetch"]) return; + rscRefreshes++; + }); + + await page.goto("/"); + await expect(page.locator("aside").getByText("Quiet Title")).toBeVisible(); + + // Settle past the initial load before measuring. The very first pulse + // necessarily changes the token (the client starts with none), and that one + // refresh is additionally deferred by the post-navigation quiet window — so + // it lands a beat after the page appears. This test is about STEADY state, + // so that startup refresh belongs in the baseline, not in the measurement. + await page.waitForTimeout(3_000); + const baseline = rscRefreshes; + pulses = 0; + await page.waitForTimeout(5_000); + + expect( + rscRefreshes - baseline, + "a quiet page must not re-render the server tree", + ).toBe(0); + expect(pulses, "the pulse must still be polling").toBeGreaterThanOrEqual(4); + }); + test("does not auto-refresh when disabled (0)", async ({ page }) => { await resetData("empty"); await writeSettings({ diff --git a/editor/e2e/auto-report-refresh.spec.ts b/editor/e2e/auto-report-refresh.spec.ts @@ -1,6 +1,6 @@ import { writeFile } from "node:fs/promises"; import { test, expect } from "@playwright/test"; -import { readJson, resetData, resolvePath } from "./helpers"; +import { readJson, resetData, resolvePath, generateReport } from "./helpers"; import { baseUrl } from "./baseUrl"; // Verifies the global debounced snapshot scheduler: a direct (non-managed) @@ -36,6 +36,7 @@ test("a direct action auto-refreshes the channel report via the debounce", async // Visiting the channel page generates the initial snapshot from disk: both // transcribed-with-audio videos are cleanable. + await generateReport(page, SLUG); await page.goto(`/channels/${SLUG}`); const before = await readJson<Snap>(SNAPSHOT_REL); expect(before.buckets.transcribedWithAudio).toContain("vidA"); diff --git a/editor/e2e/cleanup-page.spec.ts b/editor/e2e/cleanup-page.spec.ts @@ -1,6 +1,6 @@ import { writeFile } from "node:fs/promises"; import { test, expect } from "@playwright/test"; -import { readJson, resetData, resolvePath } from "./helpers"; +import { readJson, resetData, resolvePath, generateReport } from "./helpers"; import { baseUrl } from "./baseUrl"; import type { ChannelConfig } from "yt-dlp-transcript-common/lib/channelConfig"; @@ -32,6 +32,7 @@ test("cleanup page totals reclaimable audio and the per-channel toggle excludes await fetch(`${baseUrl}/api/test/invalidate-cache`).catch(() => {}); // Visiting the channel page regenerates the snapshot (with cleanupBytes). + await generateReport(page, SLUG); await page.goto(`/channels/${SLUG}`); // The cleanup page hero counts the included channel. diff --git a/editor/e2e/dashboard.spec.ts b/editor/e2e/dashboard.spec.ts @@ -5,7 +5,7 @@ // persisted downloads pause blocking a manual sync. import { test, expect } from "@playwright/test"; -import { resetData, writeSettings, readJson } from "./helpers"; +import { resetData, writeSettings, readJson, generateReport} from "./helpers"; const ONE_WORKER = { workers: [ @@ -46,6 +46,7 @@ test("renders the pipeline band, needs-work panel and enriched channels table", await writeSettings(ONE_WORKER); // Visiting the channel detail page auto-generates the snapshot that feeds the // undownloaded counts. + await generateReport(page, "test-youtube"); await page.goto("/channels/test-youtube"); await page.goto("/"); @@ -78,6 +79,7 @@ test("renders the pipeline band, needs-work panel and enriched channels table", test("pausing downloads persists and blocks a manual sync", async ({ page }) => { await resetData("youtube-with-playlist"); await writeSettings(ONE_WORKER); + await generateReport(page, "test-youtube"); await page.goto("/channels/test-youtube"); await page.goto("/"); diff --git a/editor/e2e/do-not-clean.spec.ts b/editor/e2e/do-not-clean.spec.ts @@ -1,6 +1,6 @@ import { stat, writeFile } from "node:fs/promises"; import { test, expect } from "@playwright/test"; -import { pathExists, readJson, resetData, resolvePath } from "./helpers"; +import { pathExists, readJson, resetData, resolvePath, generateReport } from "./helpers"; import { baseUrl } from "./baseUrl"; // Mirror common/lib/format.ts formatBytes so the e2e assertion matches the UI @@ -47,6 +47,7 @@ test("'do not clean' protects a video's audio from cleanup; toggling off restore // Regenerating the snapshot (visiting the channel page) must omit the // protected id from transcribedWithAudio while keeping the unprotected one. + await generateReport(page, SLUG); await page.goto(`/channels/${SLUG}`); const snapshot = await readJson<{ buckets: { transcribedWithAudio?: string[] }; @@ -73,6 +74,7 @@ test("'do not clean' protects a video's audio from cleanup; toggling off restore .click(); await expect(page.getByLabel("media archived")).toHaveCount(0); + await generateReport(page, SLUG); await page.goto(`/channels/${SLUG}`); await page.getByRole("button", { name: "Cleanup stage summary" }).click(); await page.getByRole("button", { name: "Clean audio", exact: true }).click(); @@ -99,6 +101,7 @@ test("snapshot records reclaimable cleanup bytes and the Cleanup stage shows the const expectedBytes = sizeA + sizeB; // Visiting the channel page regenerates the snapshot from disk. + await generateReport(page, SLUG); await page.goto(`/channels/${SLUG}`); const snapshot = await readJson<{ cleanupBytes?: { transcribedWithAudio?: number }; diff --git a/editor/e2e/helpers.ts b/editor/e2e/helpers.ts @@ -9,6 +9,7 @@ import { utimes, writeFile, } from "node:fs/promises"; +import type { Page } from "@playwright/test"; import { baseUrl } from "./baseUrl"; const here = dirname(fileURLToPath(import.meta.url)); @@ -117,6 +118,41 @@ export function resolvePath(relPath: string): string { return resolve(editorRoot, relPath); } +// Generate a channel's report, the way a user does. +// +// This used to be a SIDE EFFECT of viewing the channel page: with no +// snapshot.json present the page ran a full channel analysis inline, inside a +// GET. On a large channel that is a multi-minute page load caused by nothing +// more than clicking a link, so the page now shows a "No report yet" state with +// a button instead. Specs that need a report ask for one. +// +// Idempotent: if a report already exists the page renders normally and this +// returns immediately. +export async function generateReport( + page: Page, + slug: string, +): Promise<void> { + const snapshotRel = `test-transcripts/channels/${slug}/snapshot.json`; + if (await pathExists(snapshotRel)) return; + await page.goto(`/channels/${slug}`); + const button = page.getByRole("button", { name: /refresh report/i }); + await button.waitFor({ state: "visible", timeout: 15_000 }); + + // The click is RETRIED, deliberately. A click that lands before React has + // hydrated fires nothing at all — no request, no job, no error — which is the + // long-standing flake pattern in this suite (see the openChart helper). The + // action is idempotent (it regenerates the same report), so clicking twice is + // harmless and clicking zero times is not. + for (let attempt = 0; attempt < 5; attempt++) { + await button.click({ timeout: 5_000 }).catch(() => {}); + for (let i = 0; i < 30; i++) { + if (await pathExists(snapshotRel)) return; + await new Promise((r) => setTimeout(r, 100)); + } + } + throw new Error(`generateReport: no snapshot appeared for "${slug}"`); +} + export async function copyFixture(name: string) { const dst = join(fixturesRoot, "test-transcripts", name); await rm(dst, { recursive: true, force: true }); diff --git a/editor/e2e/incomplete-transcript.spec.ts b/editor/e2e/incomplete-transcript.spec.ts @@ -9,7 +9,7 @@ import { mkdir, writeFile } from "node:fs/promises"; import { test, expect } from "@playwright/test"; -import { pathExists, readJson, resetData, resolvePath } from "./helpers"; +import { pathExists, readJson, resetData, resolvePath, generateReport } from "./helpers"; const CHANNEL = "test-transcribe"; const DATA = `test-transcripts/channels/${CHANNEL}/data`; @@ -76,6 +76,7 @@ test("incomplete-transcript filter, glyph, panel banner, and actionable", async await seed(); // --- Channel list: the chip filters to exactly the flagged video --- + await generateReport(page, CHANNEL); await page.goto(`/channels/${CHANNEL}`); const list = page.getByLabel("videos", { exact: true }); await expect(list.getByLabel("open vidTrunc")).toBeVisible(); @@ -131,6 +132,7 @@ test("channel bulk bar: clear incomplete resets the video and enables auto-runne page, }) => { await seed(); + await generateReport(page, CHANNEL); await page.goto(`/channels/${CHANNEL}`); // Select the flagged video via the new quick-select, then clear it. @@ -187,6 +189,7 @@ test("channel bulk bar: re-download & re-transcribe queues a batch fix", async ( page, }) => { await seed(); + await generateReport(page, CHANNEL); await page.goto(`/channels/${CHANNEL}`); await page @@ -207,6 +210,7 @@ test("actionable: section exposes per-channel + global fix buttons; per-channel }) => { await seed(); // Visiting the channel materializes its snapshot so /actionable lists it. + await generateReport(page, CHANNEL); await page.goto(`/channels/${CHANNEL}`); await page.goto(`/actionable`); const section = page.getByRole("region", { @@ -241,6 +245,7 @@ test("actionable: global clear-all clears every flagged video and empties the se }) => { await seed(); // Visiting the channel materializes its snapshot so /actionable lists it. + await generateReport(page, CHANNEL); await page.goto(`/channels/${CHANNEL}`); await page.goto(`/actionable`); const section = page.getByRole("region", { diff --git a/editor/e2e/pulse.spec.ts b/editor/e2e/pulse.spec.ts @@ -0,0 +1,78 @@ +import { test, expect } from "@playwright/test"; +import { resetData, writeSettings } from "./helpers"; + +// /api/pulse is the change token the global auto-refresh polls instead of +// re-rendering the page tree on a timer. Its entire justification is that it is +// cheap enough to ask constantly, so "cheap" is a contract, not an aspiration: +// in-memory job/queue/worker state plus two stat() calls, and NO corpus +// contact on the idle path. +// +// If someone later adds a channel walk, a snapshot read or a directory listing +// to it, these fail — which is the point. + +test.describe("/api/pulse", () => { + test.beforeEach(async () => { + await resetData("channel-with-counts"); + }); + + test("is fast, and stays fast when asked repeatedly", async ({ request }) => { + // Warm-up: the first hit in a dev server pays compilation, which is not + // what's being measured. + const first = await request.get("/api/pulse"); + expect(first.ok()).toBe(true); + const body = await first.json(); + expect(typeof body.rev).toBe("string"); + expect(body.rev.length).toBeGreaterThan(0); + // A first call with no known rev must return the full payload. + expect(body.changed).toBe(true); + expect(typeof body.cleanableBytes).toBe("number"); + + // The idle path: hand back the rev we were just given. Nothing has changed, + // so the endpoint must say so without recomputing anything expensive. + const started = Date.now(); + const rounds = 5; + for (let i = 0; i < rounds; i++) { + const res = await request.get( + `/api/pulse?rev=${encodeURIComponent(body.rev)}`, + ); + expect(res.ok()).toBe(true); + const idle = await res.json(); + expect(idle.rev).toBe(body.rev); + expect(idle.changed).toBe(false); + } + const perCall = (Date.now() - started) / rounds; + // Generous versus the ~2 ms the handler actually costs — this is a + // tripwire for "someone put a corpus walk in the poller", not a benchmark, + // and it runs on a shared machine. + expect(perCall, `idle pulse averaged ${perCall}ms per call`).toBeLessThan( + 250, + ); + }); + + test("the token moves when something the editor displays changes", async ({ + request, + }) => { + const before = await (await request.get("/api/pulse")).json(); + + // settings.json's mtime is part of the token — this is what makes editing + // settings repaint the sidebar without a reload. + await writeSettings({ + autoRefreshIntervalSeconds: 5, + adminTitle: "Pulse Moved", + }); + + await expect + .poll( + async () => (await (await request.get("/api/pulse")).json()).rev, + { timeout: 5_000 }, + ) + .not.toBe(before.rev); + }); + + test("the token is stable when nothing changes", async ({ request }) => { + const a = await (await request.get("/api/pulse")).json(); + await new Promise((r) => setTimeout(r, 1_200)); + const b = await (await request.get("/api/pulse")).json(); + expect(b.rev).toBe(a.rev); + }); +}); diff --git a/editor/e2e/reconcile.spec.ts b/editor/e2e/reconcile.spec.ts @@ -5,8 +5,12 @@ import { pathExists, resetData, resolvePath } from "./helpers"; // yt-dlp names Twitch VOD dirs `v<id>` (its extractor id) while the app keys // videos by the canonical URL id `<id>`, so the audio/metadata land in // data/v<id>/ and the app-written download-outcome.json lands in data/<id>/. -// generateChannelSnapshot (run on first channel-page load when no snapshot.json -// exists) now reconciles those dirs back to the canonical layout. +// generateChannelSnapshot reconciles those dirs back to the canonical layout. +// +// This used to be triggered by merely OPENING the channel page, which ran a +// full channel analysis inside a GET (multi-minute on a large channel). The +// page now shows a "no report yet" state and asks, so the test asks too — the +// subject here was always reconcileVideoDirs, not what happened to invoke it. const CHANNEL = "twitch-reconcile"; const ROOT = `test-transcripts/channels/${CHANNEL}`; @@ -59,13 +63,22 @@ test("channel snapshot reconciles Twitch v<id> dirs into canonical <id> dirs", a await resetData(null); await seedChannel(); - // Loading the channel page generates the snapshot (no snapshot.json yet), - // which runs reconcileVideoDirs first. await page.goto(`/channels/${CHANNEL}`); await expect( page.getByRole("heading", { name: "Twitch Reconcile" }), ).toBeVisible({ timeout: 15_000 }); + // No snapshot.json yet, so the page offers to build one instead of building + // it behind your back. Generating runs reconcileVideoDirs first. + await expect(page.getByRole("heading", { name: "No report yet" })).toBeVisible(); + await page.getByRole("button", { name: /refresh report/i }).click(); + + // The reconcile is the observable effect; poll for it rather than racing the + // button's own status text. + await expect + .poll(() => pathExists(`${ROOT}/data/123/audio.mp3`), { timeout: 15_000 }) + .toBe(true); + // Case 1: v123 merged into 123 — all three files now in 123, v123 gone. expect(await pathExists(`${ROOT}/data/123/audio.mp3`)).toBe(true); expect(await pathExists(`${ROOT}/data/123/metadata.info.json`)).toBe(true); diff --git a/editor/e2e/widget.spec.ts b/editor/e2e/widget.spec.ts @@ -10,7 +10,7 @@ import { mkdir, writeFile } from "node:fs/promises"; import { dirname } from "node:path"; import { test, expect } from "@playwright/test"; -import { resetData, resolvePath, writeSettings } from "./helpers"; +import { resetData, resolvePath, writeSettings, generateReport} from "./helpers"; import { baseUrl } from "./baseUrl"; const CLEAN_SLUG = "test-transcribe"; @@ -190,6 +190,7 @@ test("section params gate what renders", async ({ page }) => { test("act=1 shows a compact needs-work list of channels", async ({ page }) => { await resetData("youtube-with-playlist"); // Visiting the channel detail page auto-generates the snapshot the list reads. + await generateReport(page, "test-youtube"); await page.goto("/channels/test-youtube"); await page.goto("/widget?act=1"); @@ -205,6 +206,7 @@ test("act=1&controls=1 gives the list per-channel Download actions", async ({ page, }) => { await resetData("youtube-with-playlist"); + await generateReport(page, "test-youtube"); await page.goto("/channels/test-youtube"); // auto-generate snapshot await page.goto("/widget?act=1&controls=1");