commit daea33ebdc259510db27f0a91804a4cb71f2d08e
parent 26689d10b832efe9345baa20d62a27facfeb089e
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 11 Sep 2026 11:34:41 -0400
common: the four lanes dispatch from a compiled tree, and paused leaves the list
S1 of plans/channel-priority.md. Two changes, both in the runner, neither in
the engine.
`listChannelMeta` takes the lane and the model and drops every channel whose
EFFECTIVE tier for that lane is `paused`. It is the single source of the
channel list for the runner loop and for `computeLeafPending`, so a paused
channel is absent from the lane's draw and from the status panel's pending
counts in one predicate — and absent from the catch-all too, which a tree
cannot express. The 30 s channel-list TTL is the clock. Per-operation
overrides decide: `{tier:"normal", overrides:{sync:"paused"}}` is still drawn
by the download lane.
`laneDispatchRoot` compiles `settings.channelPriority` into the lane's root and
REPLACES `autoQueue[lane].root` with it — the projection, the completed filter,
the pick and runOperationPick's leaf lookup all read the compiled tree. COMPILE,
not consult: strict descent is already the focus/hold semantics, so no second
mechanism, no new idle reason, and no dispatch function learns anything. An
absent document (no focus, no entries) skips the compiler entirely and the
stored trees stand byte for byte. A PolicyTreeEditor save can no longer fight
the model: the compiler wins at dispatch.
The focus resolution — the only I/O, `listSites` for a `{kind:"site"}` focus —
is cached on the settings key plus a 60 s TTL, the same two triggers the
operation lanes' run context uses, so a channel added to the focused site joins
the focus without a settings write.
A once-per-transition log line says which of three states the lane is in
(no focus / focus holding / focus exhausted). It is a LOG, not an
`AutoRunnerIdleReason`: a lane whose focus holds the rest is dispatching, not
idle. The state key is the three-valued thing and never the counts, or every
completed focus unit would re-fire the line.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
1 file changed, 249 insertions(+), 16 deletions(-)
diff --git a/common/controller/autoRunner.ts b/common/controller/autoRunner.ts
@@ -78,6 +78,16 @@ import { type DownloadOutcomeStatus } from "../lib/downloadOutcome";
import { downloadQueueKey } from "../lib/queueKeys";
import { isGateHeld } from "../lib/pauseGates";
import {
+ type ChannelPriority,
+ type FocusSummary,
+ type SiteChannelIndex,
+ compileLaneRoot,
+ focusSummary,
+ isChannelPaused,
+ resolveFocusSlugs,
+} from "../lib/channelPriority";
+import { listSites } from "../lib/site";
+import {
listChannelConfigs,
readChannelConfig,
readChannelSnapshotShared,
@@ -285,12 +295,199 @@ const SNAPSHOT_READ_CONCURRENCY = 64;
// Re-derived from the shared listChannelConfigs rather than repeating the
// readdir-then-serial-read here.
-async function listChannelMeta(paths: Paths): Promise<ChannelMeta[]> {
+//
+// PAUSED IS A FILTER ON THE CHANNEL LIST, not a shape in the tree, and this is
+// the one predicate that makes it so (plans/channel-priority.md, decision 3). A
+// tree cannot express exclusion — an `{type:"all"}` catch-all matches
+// everything, and first-match-wins would let a catch-all placed above the Low
+// group swallow Low's work — so a paused channel is removed from the LIST every
+// leaf draws from instead. This function is the single source of that list for
+// BOTH the runner loop and `computeLeafPending`, so a paused channel is absent
+// from the lane's draw and from the status panel's pending counts in one edit.
+//
+// PER LANE, through `isChannelPaused(model, slug, lane)` — the EFFECTIVE tier
+// for the lane being listed, so the per-operation override map decides: a
+// channel with `{tier:"normal", overrides:{sync:"paused"}}` (what all 15 live
+// `excludeFromSync` channels migrate to) is still drawn by the download lane,
+// and one with `{tier:"paused"}` or `overrides:{download:"paused"}` is not.
+// Re-evaluated on CHANNEL_LIST_TTL_MS, which is the clock for this decision.
+async function listChannelMeta(
+ paths: Paths,
+ kind: AutoQueueKind,
+ priority: ChannelPriority,
+): Promise<ChannelMeta[]> {
const configs = await listChannelConfigs(paths);
- return configs.map(({ slug, config }) => ({
- slug,
- platform: detectPlatform(config.url),
- }));
+ return configs
+ .filter(({ slug }) => !isChannelPaused(priority, slug, kind))
+ .map(({ slug, config }) => ({
+ slug,
+ platform: detectPlatform(config.url),
+ }));
+}
+
+// --- The compiled priority trees -------------------------------------------
+//
+// THE MODEL COMPILES, IT IS NOT CONSULTED. `settings.channelPriority` holds one
+// tier per channel plus one focus selector; `compileLaneRoot` turns that into
+// the lane's `AutoQueueGroup` (focus > normal > low > catch-all, all strict),
+// and dispatch runs the ordinary engine over it. Nothing in
+// `buildPendingByLeaf`, `selectNextWork`, `operationBatch`, `laneLimit` or
+// `pauseGates` learns a second priority mechanism, and "a zero limit is a hold,
+// never a stop" (controller/operationBatch.ts:22-25) is preserved trivially
+// because nothing here ever returns a limit.
+//
+// A FOCUS HOLDS THE REST BECAUSE STRICT DESCENT ALREADY DOES, and no second
+// mechanism is added: `pick()` (jobs/autoQueuePolicy.ts) filters a strict
+// group's children to those WITH WORK and descends into the first of them, and
+// it is re-asked on every grant. So focus work present => nothing below it is
+// picked; focus work exhausted => the next group runs; new focus work arriving
+// => the very next pick retakes the lane.
+//
+// WHERE THE HAND-EDITED TREES GO. The compiled root REPLACES
+// `settings.autoQueue[lane].root` at `laneDispatchRoot` below — the stored tree
+// is not consulted at all while a model exists. That is what makes the compiler
+// the ONE WRITER of channel priority: a `PolicyTreeEditor` save can still put a
+// channel leaf in the stored tree, but it cannot change what this lane
+// dispatches, so the two cannot fight — the compiler simply wins. (S3's
+// `saveChannelPriorityAction` then also PERSISTS the compiled roots, so the
+// stored tree and this one agree on disk; S4 makes the editor read-only for
+// compiled groups, which is the UI catching up with this fact.)
+//
+// AN ABSENT MODEL CHANGES NOTHING, BYTE FOR BYTE. `isDefaultChannelPriority`
+// below is the gate: no focus and no channel entries means the compiler never
+// runs and the stored trees stand exactly as they are today.
+
+const PRIORITY_CONTEXT_TTL_MS = 60_000;
+
+// The resolved half of the model — the half that costs I/O. Rebuilt when the
+// stored document changes or the TTL lapses, the same two triggers the
+// operation lanes' run context uses (settings key + 60 s), and for the same
+// reason: `resolveFocusSlugs` reads `transcripts/sites/*/site.json` through
+// `listSites`, and a `{kind:"site"}` focus tracks that file's membership rather
+// than freezing a list. The TTL is what makes a channel added to the focused
+// site join the focus without a settings write.
+type PriorityContext = {
+ key: string;
+ at: number;
+ model: ChannelPriority;
+ focusSlugs: string[];
+};
+
+let priorityContext: PriorityContext | null = null;
+
+// No focus and no per-channel entry: the document says nothing, so the compiler
+// must not run. Local to the runner rather than in lib/channelPriority.ts
+// because it is a dispatch-side question ("is there anything to compile"), not
+// part of the model's contract.
+function isDefaultChannelPriority(model: ChannelPriority): boolean {
+ return (
+ model.focus.kind === "none" && Object.keys(model.channels).length === 0
+ );
+}
+
+function priorityContextFor(paths: Paths): PriorityContext {
+ const model = getSettings().channelPriority;
+ // The paths go in the key so two worktrees' runners in one process cannot
+ // share a focus resolved against the other's sites directory.
+ const key = JSON.stringify([model, paths.sitesDir]);
+ const now = Date.now();
+ if (
+ priorityContext &&
+ priorityContext.key === key &&
+ now - priorityContext.at < PRIORITY_CONTEXT_TTL_MS
+ ) {
+ return priorityContext;
+ }
+ // ONLY A SITE FOCUS READS THE SITES DIRECTORY. `{kind:"channels"}` and
+ // `{kind:"none"}` resolve from the document alone, so the common case pays
+ // nothing. `listSites` is the existing reader (lib/site.ts) — there is no
+ // second one here.
+ const siteChannels: Record<string, string[]> = {};
+ if (model.focus.kind === "site") {
+ for (const site of listSites(paths)) {
+ siteChannels[site.siteId] = site.channels.map((c) => c.slug);
+ }
+ }
+ const index: SiteChannelIndex = siteChannels;
+ priorityContext = {
+ key,
+ at: now,
+ model,
+ focusSlugs: resolveFocusSlugs(model, index),
+ };
+ return priorityContext;
+}
+
+// THE ROOT THIS LANE ACTUALLY DISPATCHES FROM. One function, called by the
+// runner loop AND by computeLeafPending, so the status panel can never name a
+// leaf the runner does not have.
+//
+// `slugs` is the lane's own (already paused-filtered) channel list;
+// `compileLaneRoot` re-applies the same per-lane predicate, so handing it the
+// filtered list and handing it every slug produce the identical tree for THIS
+// lane. Compiling is pure and O(channels) — ~69 string pushes against the
+// ~6.5 MB of snapshot JSON the same tick folds — so it happens per tick and
+// only the focus resolution above is cached.
+function laneDispatchRoot(
+ kind: AutoQueueKind,
+ policy: AutoQueuePolicy,
+ ctx: PriorityContext,
+ slugs: readonly string[],
+): AutoQueueGroup {
+ if (isDefaultChannelPriority(ctx.model)) return policy.root;
+ return compileLaneRoot(kind, ctx.model, slugs, ctx.focusSlugs);
+}
+
+// The once-per-state-change line a lane writes while a focus is holding it.
+//
+// NOT AN IDLE REASON, and deliberately not: a lane whose focus group holds the
+// rest is not idle, it is dispatching focus work — `AutoRunnerIdleReason` stays
+// exactly as it is, and `no-pending` remains the true answer when the whole
+// tree is empty. This is the runner's LOG saying which of three states it is
+// in, so "why is only jeralyzer moving?" is answerable from the job log alone.
+//
+// The state is the three-valued thing, NOT the counts: the counts are in the
+// message but never in the key, or every completed focus unit would re-fire the
+// line. Same shape as the snooze line and the disk-gate line above.
+export function focusHoldState(summary: FocusSummary | null): string {
+ if (!summary || !summary.active) return "none";
+ return summary.holding ? "hold" : "free";
+}
+
+export function focusHoldLine(
+ kind: AutoQueueKind,
+ summary: FocusSummary | null,
+): string | null {
+ if (!summary || !summary.active) return null;
+ const channels = `${summary.channelCount} channel${summary.channelCount === 1 ? "" : "s"}`;
+ if (summary.holding) {
+ return (
+ `Auto-${kind}: focus (${channels}) is holding this lane — ` +
+ `${summary.focusPending} focus unit(s) pending, ` +
+ `${summary.heldChannels} channel(s) held.`
+ );
+ }
+ return (
+ `Auto-${kind}: focus (${channels}) has no work left in this lane — ` +
+ `${summary.otherPending} unit(s) released to the rest of the corpus.`
+ );
+}
+
+// The gate itself, as a closure so the runner keeps one line per transition and
+// the test can drive the transitions without a corpus. Logs on entering "hold"
+// and on entering "free"; says nothing while there is no active focus.
+export function makeFocusHoldReporter(
+ kind: AutoQueueKind,
+ onLog: (line: string) => void,
+): (summary: FocusSummary | null) => void {
+ let state = "none";
+ return (summary) => {
+ const next = focusHoldState(summary);
+ if (next === state) return;
+ state = next;
+ const line = focusHoldLine(kind, summary);
+ if (line) onLog(line);
+ };
}
// Read each channel's snapshot and project the buckets this runner kind cares
@@ -576,7 +773,15 @@ export async function computeLeafPending(
paths: Paths = getPaths(),
): Promise<LeafPending> {
const policy = getSettings().autoQueue[kind];
- const meta = await listChannelMeta(paths);
+ // THE SAME TWO PRIORITY DECISIONS THE RUNNER MAKES, in the same order: the
+ // paused filter on the channel list, then the compiled root. This function's
+ // whole contract is that its numbers are the runner's numbers, so both sides
+ // of channel priority have to be here too — otherwise the panel would count
+ // pending work for a paused channel, or attribute it to a stored leaf the
+ // runner is not dispatching from.
+ const ctx = priorityContextFor(paths);
+ const meta = await listChannelMeta(paths, kind, ctx.model);
+ const root = laneDispatchRoot(kind, policy, ctx, meta.map((m) => m.slug));
const laneOperations = laneOperationIds(kind);
const { channels, owner } = await buildChannelWork(
paths,
@@ -596,7 +801,7 @@ export async function computeLeafPending(
// leaf naming anything else finds no list and comes back empty — the zero
// retainLeaves used to apply afterwards, reached by construction.
const pending = buildPendingByLeaf(
- policy.root,
+ root,
channels,
defaultDrawsForPolicy(kind, policy, bucketLaneOperationId(kind)),
{ ...(compare ? { compare } : {}), defaultOperations: laneOperations },
@@ -633,14 +838,14 @@ export async function computeLeafPending(
currentWeights: { ...state[kind].runtime.currentWeights },
};
const pick = selectNextWork(
- policy.root,
+ root,
pending,
runtime,
live ? { ...live.active } : {},
);
let nextUp: NextUpView | null = null;
if (pick) {
- const order = flattenLeaves(policy.root);
+ const order = flattenLeaves(root);
const at = order.findIndex((l) => l.id === pick.leafId);
nextUp = {
videoId: pick.videoId,
@@ -806,12 +1011,16 @@ async function runLoop(
let metaCache: ChannelMeta[] = [];
let metaAt = 0;
+ // The root the last pick was made from. next() sets it every tick;
+ // runOperationPick reads it to resolve the leaf its own pick named.
+ let dispatchRoot: AutoQueueGroup = getSettings().autoQueue[kind].root;
// Whether the last next() saw the disk gate closed. next() runs on every
// scheduling tick, so without this the log fills with one identical line per
// tick for as long as the disk is full — which is precisely the situation in
// which the log needs to stay readable. Logged on each transition instead.
let diskIdle = false;
+ const reportFocusHold = makeFocusHoldReporter(kind, onLog);
// A graceful stop (the job record is gone after an e2e reset, or the policy was
// disabled) is modeled as a soft drain: stop picking, let in-flight finish.
@@ -1075,11 +1284,25 @@ async function runLoop(
}
// Refresh the (rarely-changing) channel list/platforms on a TTL.
+ //
+ // THE PAUSED FILTER RIDES THIS CLOCK. `listChannelMeta` drops every channel
+ // whose effective tier for THIS lane is `paused`, so the 30 s TTL is also
+ // how long a pause takes to reach dispatch. `metaAt === 0` rather than
+ // `metaCache.length === 0` is the cache-miss test now: a lane on which
+ // every channel is paused has a legitimately empty list, and the old
+ // sentinel would re-read 68 configs on every three-second tick for it.
const now = Date.now();
- if (now - metaAt > CHANNEL_LIST_TTL_MS || metaCache.length === 0) {
- metaCache = await listChannelMeta(paths);
+ const ctx = priorityContextFor(paths);
+ if (metaAt === 0 || now - metaAt > CHANNEL_LIST_TTL_MS) {
+ metaCache = await listChannelMeta(paths, kind, ctx.model);
metaAt = now;
}
+ // THE COMPILED ROOT ENTERS HERE, and this is the only place it does for the
+ // dispatch path: everything below — the projection, the completed filter,
+ // the pick and the leaf lookup in runOperationPick — reads `dispatchRoot`,
+ // never `policy.root`. See laneDispatchRoot.
+ const root = laneDispatchRoot(kind, policy, ctx, metaCache.map((m) => m.slug));
+ dispatchRoot = root;
const slugToPlatform = new Map(
metaCache.map((m) => [m.slug, m.platform ?? "unknown"]),
);
@@ -1105,7 +1328,7 @@ async function runLoop(
// the dispatch path, so it is the one where getting it wrong runs the wrong
// engine on the wrong video.
const pending = buildPendingByLeaf(
- policy.root,
+ root,
channels,
defaultDrawsForPolicy(kind, policy, bucketLaneOperationId(kind)),
{ ...(compare ? { compare } : {}), defaultOperations: laneOperations },
@@ -1115,8 +1338,16 @@ async function runLoop(
// also how the dependency order is kept — diarization finishes before
// attribution-diarized is offered the same video.
removeIds(pending, new Set(live.inFlight.keys()));
- dropCompleted(pending, policy.root, laneOperations);
+ dropCompleted(pending, root, laneOperations);
const pendingBeforeGates = countPending(pending);
+ // Say — ONCE per transition — whether a focus is holding this lane. Read off
+ // the `prio-*` leaf ids in the map just built, so it costs one pass over
+ // keys and no new read, and it is skipped entirely while no focus resolves.
+ reportFocusHold(
+ ctx.focusSlugs.length > 0
+ ? focusSummary(ctx.model, ctx.focusSlugs, pending)
+ : null,
+ );
// THE RUNNER JOB'S OWN BAR, on the metrics the per-channel jobs already
// use — so a lane's runner row reads like the manual verb's row rather than
// like an opaque long-lived loop. `target` moves as the corpus does (this
@@ -1167,7 +1398,7 @@ async function runLoop(
}
}
- const pick = selectNextWork(policy.root, pending, runtime, live.active);
+ const pick = selectNextWork(root, pending, runtime, live.active);
if (!pick) {
// Attribute the idleness. Nothing pending at all is a different situation
// from work that exists but is unreachable, and both differ from work the
@@ -1237,8 +1468,10 @@ async function runLoop(
const laneOperations = laneOperationIds(kind);
const opened = laneRun.run;
if (!opened) return { outcome: "skipped" };
- const policy = getSettings().autoQueue[kind];
- const leaf = flattenLeaves(policy.root).find(
+ // THE ROOT THE PICK CAME FROM, not the stored one: a compiled leaf id
+ // (`prio-<tier>-<slug>`) does not exist in the stored tree, and looking it
+ // up there would silently fall back to the lane's whole operation union.
+ const leaf = flattenLeaves(dispatchRoot).find(
(l) => l.id === picked.pick.leafId,
);
const wanted = new Set(
