commit d5c1d51f0e5e5c228568f836d8e30eba13465e0b
parent f424e881bb48c44393cd72270e8b72d0dbe8c0eb
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 14 Sep 2026 17:18:21 -0400
views: a page no longer imports its data out of a route file
`app/page.tsx:20` imported `buildWidgetSyncPayload` from
`api/widget/sync/route.ts`, because that is where the builder was — 300 lines of
corpus-wide sums living inside an HTTP handler, four reads deep, next to a
`NextResponse`. Two consumers, one of them a server component, and the only path
between them ran through a route.
The fold is `common/views/widgetSync.ts` now and takes { settings, now, state,
briefs } — sync, and pure. The briefs matter most: every number on this payload
is summed off the snapshot each brief already carries (it used to be a 4.4 s
corpus walk on a POLLED endpoint), so handing them in is what lets the dashboard
render and the widget poll from the same per-request memo.
There is no shell. `editor/app/widget/lib/syncInputs.ts` is the four reads,
`app/page.tsx` composes them itself, and the route is what a route should be:
`GET`, `dynamic`, and one line between them.
The route keeps ONE `export type { WidgetSyncPayload }`, because five client
components still name the type through that path and sub-slice D owns them.
That line goes with them.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
5 files changed, 615 insertions(+), 325 deletions(-)
diff --git a/common/views/widgetSync.test.ts b/common/views/widgetSync.test.ts
@@ -0,0 +1,240 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { defaultSiteSettings } from "../lib/settings";
+import { defaultChannelPriority } from "../lib/channelPriority";
+import { emptySchedulerState } from "../jobs/syncSchedulerState";
+import { buildScheduleView } from "../jobs/syncScheduler";
+import type { ChannelBrief } from "../controller/channels";
+import type { ChannelConfig } from "../lib/channelConfig";
+import type { ChannelSnapshot } from "../controller/channelSnapshot";
+import {
+ emptyOperationCounts,
+ type OperationSnapshotEntry,
+} from "../lib/operations";
+import { buildWidgetSyncPayload, type WidgetSyncInputs } from "./widgetSync";
+
+// THE WIDGET'S CORPUS-WIDE SUMS. Every number here is folded off the snapshot
+// each brief already carries, which is why the briefs are an argument: this used
+// to be a corpus walk on a polled endpoint.
+
+const NOW = 1_700_000_000_000;
+
+const entry = (
+ over: Partial<OperationSnapshotEntry> = {},
+): OperationSnapshotEntry => ({ ...emptyOperationCounts(), ids: [], ...over });
+
+const brief = (
+ slug: string,
+ snapshot: Partial<ChannelSnapshot> | null,
+ config: Partial<ChannelConfig> = {},
+): ChannelBrief =>
+ ({
+ slug,
+ config: { name: slug, url: `https://example.com/${slug}`, ...config },
+ snapshot: snapshot
+ ? ({
+ generatedAt: "2026-01-01T00:00:00Z",
+ totals: { videos: 0, transcribed: 0, downloaded: 0 },
+ buckets: {},
+ ...snapshot,
+ } as ChannelSnapshot)
+ : null,
+ }) as ChannelBrief;
+
+function inputs(over: Partial<WidgetSyncInputs> = {}): WidgetSyncInputs {
+ return {
+ settings: defaultSiteSettings(),
+ now: NOW,
+ state: emptySchedulerState(),
+ briefs: [],
+ ...over,
+ };
+}
+
+test("two briefs are summed, per corpus and per kind", () => {
+ const briefs = [
+ brief("alpha", {
+ totals: { videos: 100, transcribed: 90, downloaded: 95 },
+ digestEngines: { ollama: 40, remote: 2 },
+ backfill: {
+ digest: entry({ missing: 5, blocked: 3, deferred: 1, eligible: 60 }),
+ diarization: entry({ missing: 4, stale: 1, missingInput: 20, eligible: 100 }),
+ "attribution-text": entry({ missing: 9, missingInput: 1, eligible: 100 }),
+ },
+ }),
+ brief("beta", {
+ totals: { videos: 10, transcribed: 10, downloaded: 10 },
+ digestEngines: { ollama: 3 },
+ backfill: {
+ digest: entry({ missing: 1, blocked: 2, deferred: 4, eligible: 8 }),
+ diarization: entry({ missing: 1, missingInput: 5, eligible: 10 }),
+ "attribution-text": entry({ missing: 2, eligible: 10 }),
+ },
+ }),
+ ];
+ const payload = buildWidgetSyncPayload(inputs({ briefs }));
+ assert.equal(payload.digest.videos, 110);
+ assert.equal(payload.digest.digested, 45);
+ assert.equal(payload.digest.channelsWithAny, 2);
+ assert.equal(payload.digest.channels, 2);
+ assert.equal(payload.digest.blocked, 5);
+ assert.equal(payload.digest.deferred, 5);
+ // Eligibility MINUS what is blocked on a transcript, per channel: (60-3) +
+ // (8-2). A blocked video is eligible in principle and cannot be digested
+ // today, and counting it would make coverage sag every time new videos land.
+ assert.equal(payload.digest.eligible, 63);
+
+ // The backfill LANE only: `digest` runs on its own queue key and its coverage
+ // is the figure beside this one, not part of it.
+ assert.equal(payload.backfill.reachable, 4 + 1 + 9 + 1 + 2);
+ assert.equal(payload.backfill.needsMedia, 20 + 1 + 5);
+ assert.deepEqual(
+ payload.backfill.kinds.map((k) => [k.id, k.reachable, k.needsMedia]),
+ [
+ ["attribution-text", 11, 1],
+ ["diarization", 6, 25],
+ ],
+ );
+ const diarization = payload.backfill.kinds.find((k) => k.id === "diarization")!;
+ assert.equal(diarization.label, "Speaker diarization");
+ assert.equal(diarization.eligible, 110);
+ // present = eligible - every work count, summed per channel.
+ assert.equal(diarization.present, 110 - 6 - 25);
+});
+
+test("one snapshot that cannot report voids that kind's coverage and nothing else", () => {
+ const briefs = [
+ brief("alpha", {
+ totals: { videos: 10, transcribed: 10, downloaded: 10 },
+ backfill: {
+ digest: entry({ missing: 1, eligible: 10 }),
+ diarization: entry({ missing: 1, eligible: 10 }),
+ "attribution-text": entry({ missing: 1, eligible: 10 }),
+ },
+ }),
+ // Written before `eligible` existed.
+ brief("beta", {
+ totals: { videos: 10, transcribed: 10, downloaded: 10 },
+ backfill: {
+ digest: entry({ missing: 1, eligible: 10 }),
+ diarization: entry({ missing: 1 }),
+ "attribution-text": entry({ missing: 1, eligible: 10 }),
+ },
+ }),
+ ];
+ const payload = buildWidgetSyncPayload(inputs({ briefs }));
+ const kind = (id: string) => payload.backfill.kinds.find((k) => k.id === id)!;
+ // A partial sum would be a denominator smaller than its own numerator, so the
+ // kind says "unknown" — and only that kind.
+ assert.equal(kind("diarization").eligible, null);
+ assert.equal(kind("diarization").present, null);
+ assert.equal(kind("attribution-text").eligible, 20);
+ assert.equal(kind("attribution-text").present, 18);
+ // Digest's own denominator is a separate sum and is unaffected.
+ assert.equal(payload.digest.eligible, 20);
+
+ // Digest goes null the same way, corpus-wide, when a channel cannot report.
+ const noDigestEligible = buildWidgetSyncPayload(
+ inputs({
+ briefs: [briefs[0], brief("gamma", { backfill: { digest: entry({ missing: 1 }) } })],
+ }),
+ );
+ assert.equal(noDigestEligible.digest.eligible, null);
+});
+
+test("kinds are sorted by reachable descending, tie-broken by id", () => {
+ const payload = buildWidgetSyncPayload(
+ inputs({
+ briefs: [
+ brief("alpha", {
+ backfill: {
+ diarization: entry({ missing: 7 }),
+ "attribution-text": entry({ missing: 7 }),
+ },
+ }),
+ ],
+ }),
+ );
+ assert.deepEqual(
+ payload.backfill.kinds.map((k) => k.id),
+ ["attribution-text", "diarization"],
+ );
+ const bigger = buildWidgetSyncPayload(
+ inputs({
+ briefs: [
+ brief("alpha", {
+ backfill: {
+ diarization: entry({ missing: 8 }),
+ "attribution-text": entry({ missing: 7 }),
+ },
+ }),
+ ],
+ }),
+ );
+ assert.deepEqual(
+ bigger.backfill.kinds.map((k) => k.id),
+ ["diarization", "attribution-text"],
+ );
+});
+
+test("nextRunAt and overdue come off buildScheduleView, over eligible channels only", () => {
+ const settings = defaultSiteSettings();
+ settings.syncScheduler = {
+ ...settings.syncScheduler,
+ enabled: true,
+ defaultIntervalMinutes: 60,
+ };
+ // Synced 30 minutes ago: next due in another 30.
+ const soon = brief("soon", null, {
+ lastSyncedAt: new Date(NOW - 30 * 60_000).toISOString(),
+ });
+ // Synced two hours ago: overdue now.
+ const late = brief("late", null, {
+ lastSyncedAt: new Date(NOW - 120 * 60_000).toISOString(),
+ });
+ const briefs = [soon, late];
+ const payload = buildWidgetSyncPayload(inputs({ settings, briefs }));
+ const view = buildScheduleView({
+ channels: briefs.map((c) => ({ slug: c.slug, config: c.config })),
+ scheduler: settings.syncScheduler,
+ state: emptySchedulerState(),
+ now: NOW,
+ priority: defaultChannelPriority(),
+ });
+ assert.deepEqual(
+ payload.scheduler.nextRunAt,
+ Math.min(...view.map((v) => v.nextDueAt!)),
+ );
+ assert.equal(payload.scheduler.overdue, true);
+ assert.equal(payload.scheduler.enabled, true);
+
+ // Scheduler off: no channel is eligible, so there is nothing due and nothing
+ // overdue — the schedule view, not a second predicate, decides that.
+ const off = buildWidgetSyncPayload(inputs({ briefs }));
+ assert.equal(off.scheduler.nextRunAt, null);
+ assert.equal(off.scheduler.overdue, false);
+ assert.equal(off.scheduler.enabled, false);
+});
+
+test("the two freshness markers", () => {
+ const state = emptySchedulerState();
+ state.lastSyncAllAt = 12_345;
+ state.runs = [{ at: 999, queued: [], skipped: [] }];
+ const payload = buildWidgetSyncPayload(
+ inputs({
+ state,
+ briefs: [
+ brief("alpha", null, { lastSyncedAt: "2026-01-02T00:00:00Z" }),
+ brief("beta", null, { lastSyncedAt: "2026-03-04T00:00:00Z" }),
+ brief("gamma", null, { lastSyncedAt: "not a date" }),
+ brief("delta", null),
+ ],
+ }),
+ );
+ assert.equal(payload.lastSyncAllAt, 12_345);
+ assert.equal(payload.scheduler.lastRunAt, 999);
+ assert.equal(
+ payload.lastIndividualSyncAt,
+ Date.parse("2026-03-04T00:00:00Z"),
+ );
+});
diff --git a/common/views/widgetSync.ts b/common/views/widgetSync.ts
@@ -0,0 +1,333 @@
+import type { SiteSettings } from "../lib/settings";
+import { digestCountOf, type ChannelBrief } from "../controller/channels";
+import { digestWorkOf } from "../controller/channelSnapshot";
+import { isGateHeld } from "../lib/pauseGates";
+import {
+ backfillLaneOperations,
+ backfillLaneOperationEntriesOf,
+ operationCostBasis,
+ operationLabel,
+ operationsGroupLabel,
+ presentOperationWork,
+ reachableOperationWork,
+} from "../lib/operations";
+import { buildScheduleView } from "../jobs/syncScheduler";
+import type { SchedulerState } from "../jobs/syncSchedulerState";
+
+// Backs the monitor widget's last-sync and scheduler-status strips. Keeps the
+// payload tiny (a handful of scalars) rather than shipping the full
+// scheduler-status payload on every poll — the widget only needs freshness
+// markers and a one-line scheduler summary. Reuses the same helpers the
+// /scheduler status page does so the two never drift.
+export type WidgetSyncPayload = {
+ // Epoch ms of the last manual full "Sync all" sweep. null = never.
+ lastSyncAllAt: number | null;
+ // Newest per-channel lastSyncedAt across all channels. null = none synced.
+ lastIndividualSyncAt: number | null;
+ scheduler: {
+ enabled: boolean;
+ lastRunAt: number | null; // last scheduled sweep (state.runs[0].at)
+ nextRunAt: number | null; // soonest eligible channel's nextDueAt
+ overdue: boolean; // some eligible channel is due now
+ };
+ // Corpus-wide digest coverage. SCALARS ONLY, honouring this payload's stated
+ // constraint — three numbers and two booleans, not a per-channel breakdown.
+ //
+ // It belongs on the poll rather than a page load because the number it
+ // reports moves over WEEKS: an operator watching an 80-day backfill needs to
+ // see it move at all, and "0.13% of 77,207" is not a figure any single
+ // channel page can show.
+ digest: {
+ digested: number; // videos carrying a non-empty ai-digest.json
+ videos: number; // every video dir in the corpus
+ // Videos that CAN be digested: transcribed and not untranscribable, summed
+ // from each channel's digest work list. The honest denominator — `videos`
+ // counts video dirs, including the ~1,700 that have no transcript and never
+ // will, so a coverage percentage against it can never reach 100%.
+ //
+ // NULL UNTIL EVERY CHANNEL CAN REPORT IT, deliberately. This is summed
+ // across channels, and a channel with no `backfill.digest` entry contributes
+ // videos to `digested` but nothing to this — so a partial sum would be a
+ // denominator smaller than its own numerator, which is a worse lie than the
+ // one it replaces. A null says "not yet knowable" and the band falls back to
+ // `videos`. Every channel on the measured corpus reports it (68/68 on
+ // 2026-08-26); the remaining case is a channel added since, whose first
+ // snapshot has not been written yet.
+ eligible: number | null;
+ channelsWithAny: number; // channels the layer has reached at all
+ // The DENOMINATOR for channelsWithAny. Without it "66 channels reached" is a
+ // count with nothing to be a fraction of, and the card cannot tell a corpus
+ // where the lane has touched everything from one where it has barely begun.
+ channels: number;
+ // Videos with no transcript yet, and videos held back for want of a current
+ // normalized one. NEVER summed with `digested` nor with each other — they are
+ // the two reasons a digest cannot happen, and each has a different fix (wait
+ // for transcription; run Normalize transcripts).
+ blocked: number;
+ deferred: number;
+ // held — computed by isGateHeld; nothing downstream inverts anything.
+ held: boolean;
+ // The lane is ARMED — `autoQueue.digest.enabled`, the switch its runner
+ // resumes from at boot. It was `sweeping` (a corpus-wide sweep is armed)
+ // until slice 1.3; the sweeps are gone and a lane's tree is its scope, so
+ // this is the same question asked of the thing that now answers it.
+ armed: boolean;
+ };
+ // Corpus-wide backfill state, and free for the same reason as `digest`: every
+ // brief already carries its channel's snapshot, so this is a sum rather than a
+ // corpus walk (which cost 4.4 s on an endpoint the widget polls).
+ //
+ // `reachable` and `needsMedia` are separate FIELDS, not a total, because on the
+ // measured corpus they are 835 and ~76,270. A single number here would report
+ // a backfill as barely begun forever, no matter how much of the reachable work
+ // was finished.
+ //
+ // THE "SCALARS ONLY" CONSTRAINT IS REVISED, NOT DROPPED. `kinds` is an array,
+ // and it is bounded by the REGISTRY — three entries of five numbers today, off
+ // exactly the sums beside it, not a row per channel. What the constraint
+ // protects is that this endpoint is POLLED: a per-channel breakdown would grow
+ // with the corpus (67 channels and climbing) and put a corpus walk back on a
+ // 15-second timer. A per-KIND breakdown cannot, because adding a kind means
+ // adding an entry to operations.ts. That distinction still holds, and it is
+ // the line to keep: bounded by the code, never by the data.
+ backfill: {
+ reachable: number; // missing + stale: what the lane can do now
+ needsMedia: number; // missing-input: needs an opt-in re-download first
+ videos: number; // videos in the corpus (the denominator)
+ // held — computed by isGateHeld; nothing downstream inverts anything. The
+ // field on disk is `backfill.enabled`, whose polarity is the opposite one;
+ // that lives in pauseGates.ts and reaches the wire already resolved. It was
+ // `enabled` here until slice 7, and a pinned widget tab that predates the
+ // rename reads `undefined ?? false` — not held — until it is reloaded.
+ held: boolean;
+ // The lane is ARMED — see `digest.armed` above.
+ armed: boolean;
+ // Whether any backfill FEATURE is on. Distinct from the gate: with no
+ // feature on there is nothing to report at all, and that must not look like
+ // "all caught up".
+ anyKind: boolean;
+ // The lane's name in the operator's terms, derived on the server from the
+ // GROUP its enabled kinds declare — "Speakers" today. The client must not
+ // resolve this itself: operations.ts reaches the filesystem.
+ //
+ // "Backfill" is a queue key. Nobody arms, pauses or runs "a backfill"; the
+ // word survives only on the controls that genuinely act on the shared queue,
+ // and even there the card now lists what is in it.
+ groupLabel: string;
+ // The same numbers, kept apart by kind — which is the only form of them that
+ // means anything. Summed, this lane reads "77,952 reachable · 77,134 need
+ // media": both correct, and together a figure in no unit, since 99.5% of the
+ // first is attribution-text at ~1 model call per transcript CHUNK and all of
+ // the second is diarization at 329 runs. The split also surfaces the largest
+ // single fact about this corpus, which the sums hide completely — 77,463
+ // videos BLOCKED on diarization's output, which is why attribution can only
+ // run text-only.
+ //
+ // Sorted by `reachable` descending, tie-broken by id, so the order is stable
+ // across polls rather than following whatever order the snapshots were
+ // written in.
+ //
+ // `eligible` and `present` are the COVERAGE half, and they are `number |
+ // null` for the same reason `digest.eligible` above is: they are summed
+ // across channels, a third of the snapshots on disk predate `eligible`, and
+ // a partial sum is a denominator smaller than its own numerator. One
+ // channel that cannot report voids the kind's whole figure — see
+ // presentOperationWork, which returns null rather than 0 "because the honest
+ // answer there is unknown".
+ kinds: {
+ id: string;
+ label: string; // operationLabel(id) — resolved here, not in the client
+ // What ONE UNIT of this operation costs, in words. The fact that hid
+ // behind the lane name: attribution-text is armed on ~194,000 model calls
+ // corpus-wide because its unit is the transcript CHUNK, has completed one
+ // video, and read as a quiet row on every screen. Stated flat, with no
+ // threshold — what is affordable is the operator's call.
+ costBasis: string;
+ reachable: number;
+ needsMedia: number;
+ blocked: number;
+ deferred: number;
+ // How many videos this operation has an opinion about. null = unknown.
+ eligible: number | null;
+ // How many it is DONE with. null = unknown. Never derived by subtracting
+ // the work counts from `eligible` at a call site — that is exactly what
+ // presentOperationWork does, once, with the right guard.
+ present: number | null;
+ }[];
+ };
+};
+
+export type WidgetSyncInputs = {
+ settings: SiteSettings;
+ now: number;
+ state: SchedulerState;
+ // Every number below is summed off the snapshot each brief already carries.
+ // This used to be a full corpus walk — a 4.4 s cost on an endpoint the widget
+ // polls — which is why the briefs arrive as an argument rather than being
+ // read here: the caller already has them, memoized per request.
+ briefs: readonly ChannelBrief[];
+};
+
+// Build the widget sync payload. Both the HTTP route and the dashboard cockpit's
+// SSR seed fold through this same function, so the first paint and the first
+// poll can never disagree.
+export function buildWidgetSyncPayload(
+ inputs: WidgetSyncInputs,
+): WidgetSyncPayload {
+ const { settings, now, state } = inputs;
+ const channels = inputs.briefs;
+
+ let lastIndividualSyncAt: number | null = null;
+ for (const c of channels) {
+ const last = c.config.lastSyncedAt;
+ if (!last) continue;
+ const parsed = Date.parse(last);
+ if (Number.isNaN(parsed)) continue;
+ if (lastIndividualSyncAt === null || parsed > lastIndividualSyncAt) {
+ lastIndividualSyncAt = parsed;
+ }
+ }
+
+ const view = buildScheduleView({
+ channels: channels.map((c) => ({ slug: c.slug, config: c.config })),
+ scheduler: settings.syncScheduler,
+ state,
+ now,
+ priority: settings.channelPriority,
+ });
+ const eligible = view.filter((v) => v.autoSyncEligible);
+ let nextRunAt: number | null = null;
+ let overdue = false;
+ for (const v of eligible) {
+ if (v.overdue) overdue = true;
+ if (v.nextDueAt !== null && (nextRunAt === null || v.nextDueAt < nextRunAt)) {
+ nextRunAt = v.nextDueAt;
+ }
+ }
+
+ // Free: the snapshot each brief already carries has both halves. This used to
+ // come off a full corpus walk — a 4.4 s cost on an endpoint the widget polls.
+ // `digestEngines` postdates 11 of the live snapshots, so those read as 0 until
+ // their next report refresh (see digestCountOf).
+ let digested = 0;
+ let videos = 0;
+ let channelsWithAny = 0;
+ // The eligible denominator, and whether it is complete. One channel that
+ // cannot report voids the whole sum — see the payload type for why a partial
+ // one would be worse than no answer at all.
+ let digestEligible = 0;
+ let digestEligibleKnown = true;
+ // Summed off the same snapshots, in the same pass. Kept apart all the way
+ // through — see the payload type.
+ let backfillReachable = 0;
+ let backfillNeedsMedia = 0;
+ // The per-kind breakdown, accumulated in the SAME pass rather than a second
+ // one. A Map keyed by kind id, so a snapshot naming a kind another snapshot
+ // does not is simply added rather than dropped.
+ const backfillByKind = new Map<
+ string,
+ {
+ reachable: number;
+ needsMedia: number;
+ blocked: number;
+ deferred: number;
+ eligible: number | null;
+ present: number | null;
+ }
+ >();
+ // Digest's two non-work counters, off the digestWorkOf() call already made
+ // below — no extra read.
+ let digestBlocked = 0;
+ let digestDeferred = 0;
+ for (const c of channels) {
+ videos += c.snapshot?.totals.videos ?? 0;
+ const n = digestCountOf(c.snapshot);
+ digested += n;
+ if (n > 0) channelsWithAny++;
+ // Eligibility, minus what is waiting on a transcript: a blocked video is
+ // eligible in principle and cannot be digested today, and including it would
+ // make the coverage bar sag every time the downloader finds new videos —
+ // which reads as digest progress going backwards.
+ const work = digestWorkOf(c.snapshot);
+ if (work.eligible == null) digestEligibleKnown = false;
+ else digestEligible += Math.max(0, work.eligible - work.blocked);
+ digestBlocked += work.blocked;
+ digestDeferred += work.deferred;
+ // The backfill LANE only. The snapshot's per-kind map now carries every
+ // catalog operation, digest included, and the widget's backfill strip has
+ // only ever meant diarization plus attribution — the digest coverage figure
+ // it shows beside this one is computed separately, from digestCountOf above.
+ for (const [id, entry] of backfillLaneOperationEntriesOf(c.snapshot?.backfill)) {
+ backfillReachable += reachableOperationWork(entry);
+ backfillNeedsMedia += entry.missingInput;
+ const acc = backfillByKind.get(id) ?? {
+ reachable: 0,
+ needsMedia: 0,
+ blocked: 0,
+ deferred: 0,
+ eligible: 0 as number | null,
+ present: 0 as number | null,
+ };
+ acc.reachable += reachableOperationWork(entry);
+ acc.needsMedia += entry.missingInput;
+ // `?? 0` at every read site: every snapshot written before these fields
+ // existed lacks them, and undefined poisons the sum to NaN.
+ acc.blocked += entry.blocked ?? 0;
+ acc.deferred += entry.deferred ?? 0;
+ // The SAME null-voids-the-sum discipline digestEligibleKnown uses above,
+ // per kind rather than corpus-wide — a kind whose channels can all report
+ // must not be dragged to "unknown" by a kind whose channels cannot.
+ acc.eligible =
+ acc.eligible == null || entry.eligible == null
+ ? null
+ : acc.eligible + entry.eligible;
+ const done = presentOperationWork(entry);
+ acc.present =
+ acc.present == null || done == null ? null : acc.present + done;
+ backfillByKind.set(id, acc);
+ }
+ }
+
+ return {
+ lastSyncAllAt: state.lastSyncAllAt,
+ lastIndividualSyncAt,
+ scheduler: {
+ enabled: settings.syncScheduler.enabled,
+ lastRunAt: state.runs[0]?.at ?? null,
+ nextRunAt,
+ overdue,
+ },
+ digest: {
+ digested,
+ eligible: digestEligibleKnown ? digestEligible : null,
+ videos,
+ channelsWithAny,
+ channels: channels.length,
+ blocked: digestBlocked,
+ deferred: digestDeferred,
+ held: isGateHeld(settings, "digest"),
+ armed: settings.autoQueue.digest.enabled,
+ },
+ backfill: {
+ reachable: backfillReachable,
+ needsMedia: backfillNeedsMedia,
+ videos,
+ held: isGateHeld(settings, "backfill"),
+ armed: settings.autoQueue.backfill.enabled,
+ anyKind: backfillLaneOperations(settings).length > 0,
+ groupLabel: operationsGroupLabel(
+ backfillLaneOperations(settings).map((k) => k.id),
+ ),
+ kinds: [...backfillByKind]
+ .map(([id, counts]) => ({
+ id,
+ label: operationLabel(id),
+ costBasis: operationCostBasis(id),
+ ...counts,
+ }))
+ // Biggest reachable first, so the kind an operator can act on most leads.
+ // Tie-broken by id so the order does not shuffle between polls.
+ .sort((a, b) => b.reachable - a.reachable || a.id.localeCompare(b.id)),
+ },
+ } satisfies WidgetSyncPayload;
+}
diff --git a/editor/app/api/widget/sync/route.ts b/editor/app/api/widget/sync/route.ts
@@ -1,331 +1,22 @@
import { NextResponse } from "next/server";
-import { getPaths } from "yt-dlp-transcript-common/lib/paths";
-import { digestCountOf } from "yt-dlp-transcript-common/controller/channels";
-import { digestWorkOf } from "yt-dlp-transcript-common/controller/channelSnapshot";
-import { getChannelBriefs } from "../../../lib/requestCache";
-import { getSettings } from "yt-dlp-transcript-common/lib/settings";
-import { isGateHeld } from "yt-dlp-transcript-common/lib/pauseGates";
import {
- backfillLaneOperations,
- backfillLaneOperationEntriesOf,
- operationCostBasis,
- operationLabel,
- operationsGroupLabel,
- presentOperationWork,
- reachableOperationWork,
-} from "yt-dlp-transcript-common/lib/operations";
-import { buildScheduleView } from "yt-dlp-transcript-common/jobs/syncScheduler";
-import { readSchedulerState } from "yt-dlp-transcript-common/jobs/syncSchedulerState";
+ buildWidgetSyncPayload,
+ type WidgetSyncPayload,
+} from "yt-dlp-transcript-common/views/widgetSync";
+import { widgetSyncInputs } from "../../../widget/lib/syncInputs";
export const dynamic = "force-dynamic";
-// Backs the monitor widget's last-sync and scheduler-status strips. Keeps the
-// payload tiny (a handful of scalars) rather than shipping the full
-// scheduler-status payload on every poll — the widget only needs freshness
-// markers and a one-line scheduler summary. Reuses the same helpers the
-// /scheduler status page does so the two never drift.
-export type WidgetSyncPayload = {
- // Epoch ms of the last manual full "Sync all" sweep. null = never.
- lastSyncAllAt: number | null;
- // Newest per-channel lastSyncedAt across all channels. null = none synced.
- lastIndividualSyncAt: number | null;
- scheduler: {
- enabled: boolean;
- lastRunAt: number | null; // last scheduled sweep (state.runs[0].at)
- nextRunAt: number | null; // soonest eligible channel's nextDueAt
- overdue: boolean; // some eligible channel is due now
- };
- // Corpus-wide digest coverage. SCALARS ONLY, honouring this payload's stated
- // constraint — three numbers and two booleans, not a per-channel breakdown.
- //
- // It belongs on the poll rather than a page load because the number it
- // reports moves over WEEKS: an operator watching an 80-day backfill needs to
- // see it move at all, and "0.13% of 77,207" is not a figure any single
- // channel page can show.
- digest: {
- digested: number; // videos carrying a non-empty ai-digest.json
- videos: number; // every video dir in the corpus
- // Videos that CAN be digested: transcribed and not untranscribable, summed
- // from each channel's digest work list. The honest denominator — `videos`
- // counts video dirs, including the ~1,700 that have no transcript and never
- // will, so a coverage percentage against it can never reach 100%.
- //
- // NULL UNTIL EVERY CHANNEL CAN REPORT IT, deliberately. This is summed
- // across channels, and a channel with no `backfill.digest` entry contributes
- // videos to `digested` but nothing to this — so a partial sum would be a
- // denominator smaller than its own numerator, which is a worse lie than the
- // one it replaces. A null says "not yet knowable" and the band falls back to
- // `videos`. Every channel on the measured corpus reports it (68/68 on
- // 2026-08-26); the remaining case is a channel added since, whose first
- // snapshot has not been written yet.
- eligible: number | null;
- channelsWithAny: number; // channels the layer has reached at all
- // The DENOMINATOR for channelsWithAny. Without it "66 channels reached" is a
- // count with nothing to be a fraction of, and the card cannot tell a corpus
- // where the lane has touched everything from one where it has barely begun.
- channels: number;
- // Videos with no transcript yet, and videos held back for want of a current
- // normalized one. NEVER summed with `digested` nor with each other — they are
- // the two reasons a digest cannot happen, and each has a different fix (wait
- // for transcription; run Normalize transcripts).
- blocked: number;
- deferred: number;
- // held — computed by isGateHeld; nothing downstream inverts anything.
- held: boolean;
- // The lane is ARMED — `autoQueue.digest.enabled`, the switch its runner
- // resumes from at boot. It was `sweeping` (a corpus-wide sweep is armed)
- // until slice 1.3; the sweeps are gone and a lane's tree is its scope, so
- // this is the same question asked of the thing that now answers it.
- armed: boolean;
- };
- // Corpus-wide backfill state, and free for the same reason as `digest`: every
- // brief already carries its channel's snapshot, so this is a sum rather than a
- // corpus walk (which cost 4.4 s on an endpoint the widget polls).
- //
- // `reachable` and `needsMedia` are separate FIELDS, not a total, because on the
- // measured corpus they are 835 and ~76,270. A single number here would report
- // a backfill as barely begun forever, no matter how much of the reachable work
- // was finished.
- //
- // THE "SCALARS ONLY" CONSTRAINT IS REVISED, NOT DROPPED. `kinds` is an array,
- // and it is bounded by the REGISTRY — three entries of five numbers today, off
- // exactly the sums beside it, not a row per channel. What the constraint
- // protects is that this endpoint is POLLED: a per-channel breakdown would grow
- // with the corpus (67 channels and climbing) and put a corpus walk back on a
- // 15-second timer. A per-KIND breakdown cannot, because adding a kind means
- // adding an entry to operations.ts. That distinction still holds, and it is
- // the line to keep: bounded by the code, never by the data.
- backfill: {
- reachable: number; // missing + stale: what the lane can do now
- needsMedia: number; // missing-input: needs an opt-in re-download first
- videos: number; // videos in the corpus (the denominator)
- // held — computed by isGateHeld; nothing downstream inverts anything. The
- // field on disk is `backfill.enabled`, whose polarity is the opposite one;
- // that lives in pauseGates.ts and reaches the wire already resolved. It was
- // `enabled` here until slice 7, and a pinned widget tab that predates the
- // rename reads `undefined ?? false` — not held — until it is reloaded.
- held: boolean;
- // The lane is ARMED — see `digest.armed` above.
- armed: boolean;
- // Whether any backfill FEATURE is on. Distinct from the gate: with no
- // feature on there is nothing to report at all, and that must not look like
- // "all caught up".
- anyKind: boolean;
- // The lane's name in the operator's terms, derived on the server from the
- // GROUP its enabled kinds declare — "Speakers" today. The client must not
- // resolve this itself: operations.ts reaches the filesystem.
- //
- // "Backfill" is a queue key. Nobody arms, pauses or runs "a backfill"; the
- // word survives only on the controls that genuinely act on the shared queue,
- // and even there the card now lists what is in it.
- groupLabel: string;
- // The same numbers, kept apart by kind — which is the only form of them that
- // means anything. Summed, this lane reads "77,952 reachable · 77,134 need
- // media": both correct, and together a figure in no unit, since 99.5% of the
- // first is attribution-text at ~1 model call per transcript CHUNK and all of
- // the second is diarization at 329 runs. The split also surfaces the largest
- // single fact about this corpus, which the sums hide completely — 77,463
- // videos BLOCKED on diarization's output, which is why attribution can only
- // run text-only.
- //
- // Sorted by `reachable` descending, tie-broken by id, so the order is stable
- // across polls rather than following whatever order the snapshots were
- // written in.
- //
- // `eligible` and `present` are the COVERAGE half, and they are `number |
- // null` for the same reason `digest.eligible` above is: they are summed
- // across channels, a third of the snapshots on disk predate `eligible`, and
- // a partial sum is a denominator smaller than its own numerator. One
- // channel that cannot report voids the kind's whole figure — see
- // presentOperationWork, which returns null rather than 0 "because the honest
- // answer there is unknown".
- kinds: {
- id: string;
- label: string; // operationLabel(id) — resolved here, not in the client
- // What ONE UNIT of this operation costs, in words. The fact that hid
- // behind the lane name: attribution-text is armed on ~194,000 model calls
- // corpus-wide because its unit is the transcript CHUNK, has completed one
- // video, and read as a quiet row on every screen. Stated flat, with no
- // threshold — what is affordable is the operator's call.
- costBasis: string;
- reachable: number;
- needsMedia: number;
- blocked: number;
- deferred: number;
- // How many videos this operation has an opinion about. null = unknown.
- eligible: number | null;
- // How many it is DONE with. null = unknown. Never derived by subtracting
- // the work counts from `eligible` at a call site — that is exactly what
- // presentOperationWork does, once, with the right guard.
- present: number | null;
- }[];
- };
-};
-
-// Build the widget sync payload. Exported so the dashboard cockpit can seed its
-// initial sync readout server-side (SSR) without going through the HTTP route.
-export async function buildWidgetSyncPayload(): Promise<WidgetSyncPayload> {
- const paths = getPaths();
- const settings = getSettings();
- const state = await readSchedulerState(paths);
- const now = Date.now();
- const channels = await getChannelBriefs(paths);
-
- let lastIndividualSyncAt: number | null = null;
- for (const c of channels) {
- const last = c.config.lastSyncedAt;
- if (!last) continue;
- const parsed = Date.parse(last);
- if (Number.isNaN(parsed)) continue;
- if (lastIndividualSyncAt === null || parsed > lastIndividualSyncAt) {
- lastIndividualSyncAt = parsed;
- }
- }
-
- const view = buildScheduleView({
- channels: channels.map((c) => ({ slug: c.slug, config: c.config })),
- scheduler: settings.syncScheduler,
- state,
- now,
- priority: settings.channelPriority,
- });
- const eligible = view.filter((v) => v.autoSyncEligible);
- let nextRunAt: number | null = null;
- let overdue = false;
- for (const v of eligible) {
- if (v.overdue) overdue = true;
- if (v.nextDueAt !== null && (nextRunAt === null || v.nextDueAt < nextRunAt)) {
- nextRunAt = v.nextDueAt;
- }
- }
-
- // Free: the snapshot each brief already carries has both halves. This used to
- // come off a full corpus walk — a 4.4 s cost on an endpoint the widget polls.
- // `digestEngines` postdates 11 of the live snapshots, so those read as 0 until
- // their next report refresh (see digestCountOf).
- let digested = 0;
- let videos = 0;
- let channelsWithAny = 0;
- // The eligible denominator, and whether it is complete. One channel that
- // cannot report voids the whole sum — see the payload type for why a partial
- // one would be worse than no answer at all.
- let digestEligible = 0;
- let digestEligibleKnown = true;
- // Summed off the same snapshots, in the same pass. Kept apart all the way
- // through — see the payload type.
- let backfillReachable = 0;
- let backfillNeedsMedia = 0;
- // The per-kind breakdown, accumulated in the SAME pass rather than a second
- // one. A Map keyed by kind id, so a snapshot naming a kind another snapshot
- // does not is simply added rather than dropped.
- const backfillByKind = new Map<
- string,
- {
- reachable: number;
- needsMedia: number;
- blocked: number;
- deferred: number;
- eligible: number | null;
- present: number | null;
- }
- >();
- // Digest's two non-work counters, off the digestWorkOf() call already made
- // below — no extra read.
- let digestBlocked = 0;
- let digestDeferred = 0;
- for (const c of channels) {
- videos += c.snapshot?.totals.videos ?? 0;
- const n = digestCountOf(c.snapshot);
- digested += n;
- if (n > 0) channelsWithAny++;
- // Eligibility, minus what is waiting on a transcript: a blocked video is
- // eligible in principle and cannot be digested today, and including it would
- // make the coverage bar sag every time the downloader finds new videos —
- // which reads as digest progress going backwards.
- const work = digestWorkOf(c.snapshot);
- if (work.eligible == null) digestEligibleKnown = false;
- else digestEligible += Math.max(0, work.eligible - work.blocked);
- digestBlocked += work.blocked;
- digestDeferred += work.deferred;
- // The backfill LANE only. The snapshot's per-kind map now carries every
- // catalog operation, digest included, and the widget's backfill strip has
- // only ever meant diarization plus attribution — the digest coverage figure
- // it shows beside this one is computed separately, from digestCountOf above.
- for (const [id, entry] of backfillLaneOperationEntriesOf(c.snapshot?.backfill)) {
- backfillReachable += reachableOperationWork(entry);
- backfillNeedsMedia += entry.missingInput;
- const acc = backfillByKind.get(id) ?? {
- reachable: 0,
- needsMedia: 0,
- blocked: 0,
- deferred: 0,
- eligible: 0 as number | null,
- present: 0 as number | null,
- };
- acc.reachable += reachableOperationWork(entry);
- acc.needsMedia += entry.missingInput;
- // `?? 0` at every read site: every snapshot written before these fields
- // existed lacks them, and undefined poisons the sum to NaN.
- acc.blocked += entry.blocked ?? 0;
- acc.deferred += entry.deferred ?? 0;
- // The SAME null-voids-the-sum discipline digestEligibleKnown uses above,
- // per kind rather than corpus-wide — a kind whose channels can all report
- // must not be dragged to "unknown" by a kind whose channels cannot.
- acc.eligible =
- acc.eligible == null || entry.eligible == null
- ? null
- : acc.eligible + entry.eligible;
- const done = presentOperationWork(entry);
- acc.present =
- acc.present == null || done == null ? null : acc.present + done;
- backfillByKind.set(id, acc);
- }
- }
-
- return {
- lastSyncAllAt: state.lastSyncAllAt,
- lastIndividualSyncAt,
- scheduler: {
- enabled: settings.syncScheduler.enabled,
- lastRunAt: state.runs[0]?.at ?? null,
- nextRunAt,
- overdue,
- },
- digest: {
- digested,
- eligible: digestEligibleKnown ? digestEligible : null,
- videos,
- channelsWithAny,
- channels: channels.length,
- blocked: digestBlocked,
- deferred: digestDeferred,
- held: isGateHeld(settings, "digest"),
- armed: settings.autoQueue.digest.enabled,
- },
- backfill: {
- reachable: backfillReachable,
- needsMedia: backfillNeedsMedia,
- videos,
- held: isGateHeld(settings, "backfill"),
- armed: settings.autoQueue.backfill.enabled,
- anyKind: backfillLaneOperations(settings).length > 0,
- groupLabel: operationsGroupLabel(
- backfillLaneOperations(settings).map((k) => k.id),
- ),
- kinds: [...backfillByKind]
- .map(([id, counts]) => ({
- id,
- label: operationLabel(id),
- costBasis: operationCostBasis(id),
- ...counts,
- }))
- // Biggest reachable first, so the kind an operator can act on most leads.
- // Tie-broken by id so the order does not shuffle between polls.
- .sort((a, b) => b.reachable - a.reachable || a.id.localeCompare(b.id)),
- },
- } satisfies WidgetSyncPayload;
-}
+// The payload is `common/views/widgetSync.ts` and its reads are
+// `widget/lib/syncInputs.ts`. This file is the HTTP edge and nothing else — it
+// used to hold the builder AND the type, which is how `app/page.tsx` came to
+// import a page's data out of a route.
+//
+// The type re-export is the last of that: five client components still name
+// `WidgetSyncPayload` through this path. Sub-slice D repoints them and deletes
+// the line.
+export type { WidgetSyncPayload };
export async function GET() {
- return NextResponse.json(await buildWidgetSyncPayload());
+ return NextResponse.json(buildWidgetSyncPayload(await widgetSyncInputs()));
}
diff --git a/editor/app/page.tsx b/editor/app/page.tsx
@@ -17,7 +17,8 @@ import {
import { resolveActiveSite } from "./lib/activeSite";
import { buildActiveJobsPayload } from "./jobs/active/buildActiveJobs";
import { buildWorkersPayload } from "./workers/buildWorkers";
-import { buildWidgetSyncPayload } from "./api/widget/sync/route";
+import { buildWidgetSyncPayload } from "yt-dlp-transcript-common/views/widgetSync";
+import { widgetSyncInputs } from "./widget/lib/syncInputs";
import { DashboardCockpit } from "./components/dashboard/DashboardCockpit";
import type { DashboardChannel } from "./components/dashboard/types";
import type { WidgetActionablePayload } from "./api/widget/actionable/route";
@@ -121,7 +122,7 @@ export default async function Dashboard({
const [jobs, sync] = await Promise.all([
buildActiveJobsPayload(),
- buildWidgetSyncPayload(),
+ widgetSyncInputs().then(buildWidgetSyncPayload),
]);
const workers = buildWorkersPayload();
const changelogSource = loadChangelog();
diff --git a/editor/app/widget/lib/syncInputs.ts b/editor/app/widget/lib/syncInputs.ts
@@ -0,0 +1,25 @@
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { getSettings } from "yt-dlp-transcript-common/lib/settings";
+import { readSchedulerState } from "yt-dlp-transcript-common/jobs/syncSchedulerState";
+import type { WidgetSyncInputs } from "yt-dlp-transcript-common/views/widgetSync";
+import { getChannelBriefs } from "../../lib/requestCache";
+
+// THE READS BEHIND THE WIDGET'S SYNC STRIP.
+//
+// The payload itself is `common/views/widgetSync.ts` — a pure fold over these
+// four values. They are gathered here rather than in the route because BOTH the
+// route and the dashboard's SSR seed need them, and a builder that lived in a
+// route file was the smell that made `app/page.tsx` import out of
+// `api/widget/sync/route.ts` to render a page.
+//
+// `getChannelBriefs` is the per-request memo (lib/requestCache.ts): the
+// dashboard derives the channel list three ways in one render, and this is one
+// of the three.
+export async function widgetSyncInputs(): Promise<WidgetSyncInputs> {
+ const paths = getPaths();
+ const [state, briefs] = await Promise.all([
+ readSchedulerState(paths),
+ getChannelBriefs(paths),
+ ]);
+ return { settings: getSettings(), now: Date.now(), state, briefs };
+}
