commit 6d501f7683c3f272540505f907689e684b7e37cd
parent d5c1d51f0e5e5c228568f836d8e30eba13465e0b
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 14 Sep 2026 17:20:58 -0400
views: the pulse's rule against constructing becomes its signature
`/api/pulse` reads `globalThis.__ytt*__` rather than the getters because the
getters CREATE: `/api/test/invalidate-cache` clears exactly those globals
between e2e specs, and a poll landing a moment later used to rebuild them,
re-seeding a worker pool from settings mid-reset — ~16 unrelated specs failing
in non-deterministic combinations while each passed alone. That was a comment
and a habit.
`computePulse(o: ObserveInputs)` in `common/views/pulse.ts` makes it a type.
Every singleton is nullable, because "not created yet" is the honest answer a
pulse must be able to give; the two mtimes and the snapshot generation arrive as
plain numbers so nothing here stats a file, and the hash is injected so nothing
here imports node:crypto.
THE REV BYTES ARE UNCHANGED — same parts, same order, same separators, same
base64url sha1 over the same joined string. The only edits are the sources of
the values. Every open tab holds a rev from the running build, and a different
token would make all of them re-render once for nothing; `pulse.test.ts` asserts
the hashed STRING (with an identity digest) rather than a hash literal so a
reordered part says what moved.
The route keeps its `GET`, the idle fast path and `cleanableTotalBytes`, which
is the one thing on it that touches disk and stays off the idle path.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
4 files changed, 276 insertions(+), 118 deletions(-)
diff --git a/common/views/pulse.test.ts b/common/views/pulse.test.ts
@@ -0,0 +1,153 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import type { JobRecord } from "../jobs/registry";
+import type { ObserveInputs } from "./inputs";
+import { computePulse } from "./pulse";
+
+// THE TOKEN'S BYTES ARE THE CONTRACT, so most of what follows asserts the
+// HASHED STRING rather than the hash: `digest` is injected, so a test can read
+// exactly what would be hashed. A reordered part or a changed separator makes
+// every open tab re-render once for nothing, and a hash literal would only say
+// "different" without saying how.
+
+const echo = (s: string) => s;
+const sha1 = (s: string) => createHash("sha1").update(s).digest("base64url");
+
+const job = (over: Partial<JobRecord> & { id: string }): JobRecord =>
+ ({
+ kind: "sync",
+ queueKey: "q",
+ status: "running",
+ queuedAt: 0,
+ logPath: "/dev/null",
+ ...over,
+ }) as JobRecord;
+
+function observe(over: Partial<ObserveInputs> = {}): ObserveInputs {
+ return {
+ registry: null,
+ scheduler: null,
+ pool: null,
+ snapshotGeneration: 0,
+ settingsMtime: 0,
+ changelogMtime: 0,
+ digest: echo,
+ ...over,
+ };
+}
+
+const registryOf = (jobs: JobRecord[]) => ({
+ list: () => jobs,
+ get: (id: string) => jobs.find((j) => j.id === id),
+});
+
+test("nothing constructed yet is a valid, idle answer", () => {
+ // Every singleton is nullable because the observer reads the globals directly
+ // and "not created yet" means "nothing to report". A pulse must be able to say
+ // the system is idle without making a system to ask.
+ const out = computePulse(observe());
+ assert.equal(out.activeJobs, 0);
+ assert.equal(out.runningJobs, 0);
+ assert.equal(out.busy, false);
+ // Three parts, in this order, and nothing else.
+ assert.equal(out.rev, "snap:0|s:0|c:0");
+ // Deterministic: the same nothing hashes to the same token.
+ assert.equal(computePulse(observe()).rev, out.rev);
+});
+
+test("queued work is active but not running", () => {
+ const out = computePulse(
+ observe({
+ registry: registryOf([
+ job({ id: "a", status: "running" }),
+ job({ id: "b", status: "queued" }),
+ job({ id: "c", status: "done" }),
+ ]),
+ }),
+ );
+ assert.equal(out.activeJobs, 2);
+ assert.equal(out.runningJobs, 1);
+ assert.equal(out.busy, true);
+ // A finished job is still in the token — its status moved, and that is a
+ // change worth re-rendering for.
+ assert.equal(
+ out.rev,
+ "a:running:::::0|b:queued:::::0|c:done:::::0|snap:0|s:0|c:0",
+ );
+});
+
+test("the token moves with a job's progress", () => {
+ const at = (current: number) =>
+ computePulse(
+ observe({
+ digest: sha1,
+ registry: registryOf([
+ job({ id: "a", progress: { metric: "downloads", initial: 0, current, target: 100 } }),
+ ]),
+ }),
+ ).rev;
+ assert.notEqual(at(1), at(2));
+ assert.equal(at(7), at(7));
+ // And the shape of that part is `initial/current/target`.
+ assert.match(
+ computePulse(
+ observe({
+ registry: registryOf([
+ job({ id: "a", progress: { metric: "downloads", initial: 0, current: 7, target: 100 } }),
+ ]),
+ }),
+ ).rev,
+ /^a:running::::0\/7\/100:0\|/,
+ );
+});
+
+test("the token moves with the snapshot generation", () => {
+ // A report regenerated on the debounce lands after the job that triggered it
+ // has already finished, so without this counter the pages whose counts come
+ // from snapshots would stay stale until something unrelated moved.
+ const rev = (snapshotGeneration: number) =>
+ computePulse(observe({ digest: sha1, snapshotGeneration })).rev;
+ assert.notEqual(rev(4), rev(5));
+ assert.equal(rev(5), rev(5));
+});
+
+test("the token moves with either watched file's mtime", () => {
+ const base = computePulse(observe({ digest: sha1 })).rev;
+ assert.notEqual(base, computePulse(observe({ digest: sha1, settingsMtime: 1 })).rev);
+ assert.notEqual(base, computePulse(observe({ digest: sha1, changelogMtime: 1 })).rev);
+});
+
+test("the queue shape and the pool are in the token, in that order", () => {
+ const out = computePulse(
+ observe({
+ registry: registryOf([job({ id: "a", status: "running" })]),
+ scheduler: {
+ queues: () => [{ name: "sync", running: ["a"], queued: ["b", "c"] }],
+ } as unknown as ObserveInputs["scheduler"],
+ pool: {
+ summary: () => [{ id: "w1", busy: true }, { id: "w2", busy: false }],
+ isPaused: () => true,
+ canStopPartial: () => false,
+ } as unknown as ObserveInputs["pool"],
+ }),
+ );
+ assert.equal(
+ out.rev,
+ "a:running:::::0|q:sync:a:b,c|w:paused:2|w:w1:1|w:w2:0|snap:0|s:0|c:0",
+ );
+});
+
+test("identical inputs hash to an identical token", () => {
+ const make = () =>
+ observe({
+ digest: sha1,
+ registry: registryOf([
+ job({ id: "a", startedAt: 5, tasks: [{}, {}] as JobRecord["tasks"] }),
+ ]),
+ snapshotGeneration: 3,
+ settingsMtime: 111,
+ changelogMtime: 222,
+ });
+ assert.equal(computePulse(make()).rev, computePulse(make()).rev);
+});
diff --git a/common/views/pulse.ts b/common/views/pulse.ts
@@ -0,0 +1,108 @@
+import type { ObserveInputs } from "./inputs";
+
+// THE CHANGE TOKEN the global AutoRefresh polls instead of blindly re-rendering
+// the whole page tree every 5 seconds.
+//
+// The old refresher called router.refresh() on a timer, on every route, whether
+// or not anything had changed — a full server re-render that cost ~4.9 s and
+// held next-server at ~22% of a core forever with a single idle tab open. The
+// endpoint this backs answers "has anything changed?" from IN-MEMORY state plus
+// two stat() calls, so the idle path touches no corpus data at all.
+export type PulsePayload = {
+ // Opaque token. Identical to your last one means nothing has changed and
+ // there is nothing to re-render.
+ rev: string;
+ // True when this response carries the full body below. False is the idle
+ // path: the client sent a rev that still matches, so nothing else was
+ // computed.
+ changed: boolean;
+ // Sidebar badge counts, so the sidebar never needs a tree refresh to update.
+ activeJobs: number;
+ runningJobs: number;
+ cleanableBytes: number;
+ // Anything live right now — lets the client poll faster while work is moving.
+ busy: boolean;
+};
+
+// ⚠️ THIS OBSERVES; IT MUST NEVER CONSTRUCT.
+//
+// The three singletons are LAZY: asking for one CREATES it. That is fine for a
+// page acting on jobs and catastrophic for a status poll running every few
+// seconds in the background, because `/api/test/invalidate-cache` clears exactly
+// these globals between e2e specs — a poll landing a moment later silently
+// rebuilt them, re-seeding a worker pool from settings mid-reset. The symptom
+// was ~16 unrelated specs (branding, build, jobs, downloads) failing in
+// non-deterministic combinations while each passed in isolation.
+//
+// That rule is structural here rather than remembered: every singleton arrives
+// as an argument and every one of them is NULLABLE, so "not created yet" is a
+// legitimate answer meaning "nothing to report". A pulse must be able to say the
+// system is idle without making a system to ask. `views/inputs.ts` is the shape;
+// `editor/app/lib/liveInputs.ts` reads it.
+//
+// THE TOKEN'S BYTES ARE A CONTRACT. Every open tab holds a rev from a previous
+// deploy's build of this function; changing the order of the parts, their
+// separators or the encoding would not corrupt anything, but it would make every
+// client's next poll report a change and re-render the tree once for nothing.
+// Add fields at the end.
+export function computePulse(o: ObserveInputs): {
+ rev: string;
+ activeJobs: number;
+ runningJobs: number;
+ busy: boolean;
+} {
+ const jobs = o.registry ? o.registry.list() : [];
+
+ let activeJobs = 0;
+ let runningJobs = 0;
+ const parts: string[] = [];
+ for (const j of jobs) {
+ if (j.status === "running" || j.status === "queued") activeJobs++;
+ if (j.status === "running") runningJobs++;
+ // JobRecord has no single updatedAt, so the token is built from the fields
+ // that actually move: lifecycle timestamps, drain state, and progress. If a
+ // job's progress advances, pages showing that progress should re-render —
+ // that is a change, not noise.
+ parts.push(
+ [
+ j.id,
+ j.status,
+ j.startedAt ?? "",
+ j.endedAt ?? "",
+ j.draining ? "d" : "",
+ j.progress
+ ? `${j.progress.initial}/${j.progress.current ?? ""}/${j.progress.target}`
+ : "",
+ j.tasks?.length ?? 0,
+ ].join(":"),
+ );
+ }
+
+ // Queue shape and worker-pool state, both in-memory — and both skipped
+ // entirely when the singleton doesn't exist yet (see above).
+ if (o.scheduler) {
+ for (const q of o.scheduler.queues()) {
+ parts.push(`q:${q.name}:${q.running.join(",")}:${q.queued.join(",")}`);
+ }
+ }
+ if (o.pool) {
+ const workers = o.pool.summary();
+ parts.push(`w:${o.pool.isPaused() ? "paused" : "live"}:${workers.length}`);
+ for (const w of workers) parts.push(`w:${w.id}:${w.busy ? 1 : 0}`);
+ }
+
+ // A report regenerated on the snapshot scheduler's debounce lands AFTER the
+ // job that triggered it has already reached its final status — so without
+ // this counter, the pages whose counts come from snapshots (/channels, the
+ // dashboard) would go stale until some unrelated thing changed. One integer,
+ // in memory, and it means "a report was rewritten" without the endpoint
+ // having to stat 65 files to find out.
+ parts.push(`snap:${o.snapshotGeneration}`);
+
+ // Files the layout renders from. mtime only — neither is read.
+ parts.push(`s:${o.settingsMtime}`);
+ parts.push(`c:${o.changelogMtime}`);
+
+ const rev = o.digest(parts.join("|"));
+ return { rev, activeJobs, runningJobs, busy: runningJobs > 0 || activeJobs > 0 };
+}
diff --git a/editor/app/api/pulse/route.ts b/editor/app/api/pulse/route.ts
@@ -1,131 +1,28 @@
-import { createHash } from "node:crypto";
-import { statSync } from "node:fs";
import { NextResponse } from "next/server";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import {
+ computePulse,
+ type PulsePayload,
+} from "yt-dlp-transcript-common/views/pulse";
+import { observeInputs } from "../../lib/liveInputs";
import { cleanableTotalBytes } from "../../cleanup/lib/loadCleanup";
export const dynamic = "force-dynamic";
// The change token the global AutoRefresh polls instead of blindly re-rendering
-// the whole page tree every 5 seconds.
+// the whole page tree every 5 seconds. The token itself is
+// `common/views/pulse.ts`; `observeInputs()` is the observer's reads.
//
-// The old refresher called router.refresh() on a timer, on every route, whether
-// or not anything had changed — a full server re-render that cost ~4.9 s and
-// held next-server at ~22% of a core forever with a single idle tab open. This
-// endpoint answers "has anything changed?" from IN-MEMORY state plus two
-// stat() calls, so the idle path touches no corpus data at all.
-export type PulsePayload = {
- // Opaque token. Identical to your last one means nothing has changed and
- // there is nothing to re-render.
- rev: string;
- // True when this response carries the full body below. False is the idle
- // path: the client sent a rev that still matches, so nothing else was
- // computed.
- changed: boolean;
- // Sidebar badge counts, so the sidebar never needs a tree refresh to update.
- activeJobs: number;
- runningJobs: number;
- cleanableBytes: number;
- // Anything live right now — lets the client poll faster while work is moving.
- busy: boolean;
-};
-
-// ⚠️ THIS ENDPOINT OBSERVES; IT MUST NEVER CONSTRUCT.
-//
-// getRegistry() / getWorkerPool() / getScheduler() are lazy singletons: calling
-// them CREATES the thing if it doesn't exist. That is fine for a page acting on
-// jobs and catastrophic for a status poll running every few seconds in the
-// background, because /api/test/invalidate-cache clears exactly these globals
-// between e2e specs — a poll landing a moment later silently rebuilt them,
-// re-seeding a worker pool from settings mid-reset. The symptom was ~16
-// unrelated specs (branding, build, jobs, downloads) failing in
-// non-deterministic combinations while each passed in isolation.
-//
-// So read the globals directly and treat "not created yet" as "nothing to
-// report". A pulse must be able to say "the system is idle" without making a
-// system to ask.
+// ⚠️ THIS ENDPOINT OBSERVES; IT MUST NEVER CONSTRUCT. That is now a property of
+// the two functions it calls rather than a rule this file remembers — see the
+// header of either one for the ~16 flaky specs that bought it.
//
-// Everything here is either in-memory or a stat(). NO readdir, no corpus
-// contact, no large JSON parse — asserted by e2e/pulse.spec.ts, because the
-// entire point of this endpoint is that it is cheap enough to poll forever.
-// (The three globals read below are declared by the modules that own them —
-// common/jobs/{registry,scheduler,workerPool}.ts — so this file no longer
-// restates them. `editor/app/lib/liveInputs.ts` is where the observer's reads
-// are being gathered; phase 3 slice 1 C4 moves the body of computeRev() into
-// common/views/pulse.ts and leaves this route as GET plus the idle fast path.)
-
-function computeRev(): { rev: string; activeJobs: number; runningJobs: number; busy: boolean } {
- const paths = getPaths();
- const registry = globalThis.__yttJobRegistry__;
- const jobs = registry ? registry.list() : [];
-
- let activeJobs = 0;
- let runningJobs = 0;
- const parts: string[] = [];
- for (const j of jobs) {
- if (j.status === "running" || j.status === "queued") activeJobs++;
- if (j.status === "running") runningJobs++;
- // JobRecord has no single updatedAt, so the token is built from the fields
- // that actually move: lifecycle timestamps, drain state, and progress. If a
- // job's progress advances, pages showing that progress should re-render —
- // that is a change, not noise.
- parts.push(
- [
- j.id,
- j.status,
- j.startedAt ?? "",
- j.endedAt ?? "",
- j.draining ? "d" : "",
- j.progress
- ? `${j.progress.initial}/${j.progress.current ?? ""}/${j.progress.target}`
- : "",
- j.tasks?.length ?? 0,
- ].join(":"),
- );
- }
-
- // Queue shape and worker-pool state, both in-memory — and both skipped
- // entirely when the singleton doesn't exist yet (see above).
- const scheduler = globalThis.__yttScheduler__;
- if (scheduler) {
- for (const q of scheduler.queues()) {
- parts.push(`q:${q.name}:${q.running.join(",")}:${q.queued.join(",")}`);
- }
- }
- const pool = globalThis.__yttWorkerPool__;
- if (pool) {
- const workers = pool.summary();
- parts.push(`w:${pool.isPaused() ? "paused" : "live"}:${workers.length}`);
- for (const w of workers) parts.push(`w:${w.id}:${w.busy ? 1 : 0}`);
- }
-
- // A report regenerated on the snapshot scheduler's debounce lands AFTER the
- // job that triggered it has already reached its final status — so without
- // this counter, the pages whose counts come from snapshots (/channels, the
- // dashboard) would go stale until some unrelated thing changed. One integer,
- // in memory, and it means "a report was rewritten" without this endpoint
- // having to stat 65 files to find out.
- parts.push(`snap:${globalThis.__yttSnapshotScheduler__?.generation ?? 0}`);
-
- // Files the layout renders from. mtime only — neither is read here.
- parts.push(`s:${mtime(paths.settingsFile)}`);
- parts.push(`c:${mtime(paths.editorChangelogFile)}`);
-
- const rev = createHash("sha1").update(parts.join("|")).digest("base64url");
- return { rev, activeJobs, runningJobs, busy: runningJobs > 0 || activeJobs > 0 };
-}
-
-function mtime(file: string): number {
- try {
- return statSync(file).mtimeMs;
- } catch {
- return 0;
- }
-}
-
+// Everything on the idle path is either in-memory or a stat(). NO readdir, no
+// corpus contact, no large JSON parse — asserted by e2e/pulse.spec.ts, because
+// the entire point of this endpoint is that it is cheap enough to poll forever.
export async function GET(request: Request) {
const known = new URL(request.url).searchParams.get("rev");
- const { rev, activeJobs, runningJobs, busy } = computeRev();
+ const { rev, activeJobs, runningJobs, busy } = computePulse(observeInputs());
// Idle fast path. The client already has this rev, so nothing it displays can
// have changed — skip the only part of this endpoint that touches disk.
diff --git a/editor/app/components/pulse.ts b/editor/app/components/pulse.ts
@@ -1,7 +1,7 @@
"use client";
import { useEffect, useState } from "react";
-import type { PulsePayload } from "../api/pulse/route";
+import type { PulsePayload } from "yt-dlp-transcript-common/views/pulse";
// One poller for the whole app, shared through a module-level store.
//
