// The auto-queue's PERSISTED SHAPE: the rule tree as it is written to // settings.json, and nothing that acts on it. // // These types live in lib/ (the model layer) rather than beside the engine in // jobs/autoQueuePolicy.ts because lib/ may not import jobs/ — settings.ts, // pauseGates.ts, operations.ts and laneMigration.ts all need to NAME an // auto-queue policy, and only the runner needs to run one. The engine re-exports every // name below, so no import site had to change. // // Enforced by ../architecture.test.ts. import type { FieldDocs } from "./fieldDocs"; // --- Tree types ------------------------------------------------------------- export type AutoQueueMode = "strict" | "round-robin" | "weighted-fair"; export type AutoQueueMatchType = "channel" | "platform" | "all"; // How videos are ordered WITHIN a rule, across every channel and bucket that // rule claims. "listed" is the historical behaviour: whatever order // buildPendingByLeaf accumulated, which is bucket order over channel order over // each bucket array's own order (channelSnapshot sorts most buckets // lexicographically by video id, and leaves undownloadedIds in playlist order). // "newest"/"oldest" sort each leaf's claimed list by an upload-date key supplied // by the caller as a comparator — the engine stays pure and never reads disk. // // This deliberately does NOT reorder RULES: the tree is what expresses // priority. A newest-first archive is one catch-all rule with order "newest". // // "cheapest" is the digest lane's historical shortest-first: the comparator is // duration-keyed rather than date-keyed, and like the recency ones it is // SUPPLIED BY THE RUNNER (the engine never reads a duration off disk). A lane // whose runner supplies no comparator for it falls back to today's order — // which is what makes it safe to name here before anything computes it. export type AutoQueueOrder = "listed" | "newest" | "oldest" | "cheapest"; // Each field is documented in AUTO_QUEUE_MATCH_FIELD_DOCS below (rendered into SETTINGS.md). export type AutoQueueMatch = { type: AutoQueueMatchType; value?: string; bucket?: string; operation?: string; }; export const AUTO_QUEUE_MATCH_FIELD_DOCS: FieldDocs = { type: "What the leaf matches: \"channel\" (one channel slug in `value`), \"platform\" (a platform name in `value`), or \"all\".", value: "Channel slug (type=channel) or platform name (type=platform). Ignored " + "for type=all. A type=channel leaf with no value matches nothing.", bucket: "Optional snapshot bucket this leaf draws from, narrowing the default " + "for the runner kind (transcription → downloadedNoTranscript, download " + "→ undownloadedIds). E.g. bucket=\"failedListed\" prioritizes retries.", operation: "Optional OPERATION this leaf draws from — a registered backfill kind " + "id, or \"digest\". Same meaning as `bucket` one level up: it narrows " + "what the leaf claims, and it draws from ChannelWork.operations rather " + "than ChannelWork.buckets.\n\n" + "It lives on the MATCH, beside `bucket`, and not on the node. A field " + "on the node would need group inheritance — \"this group is the digest " + "subtree\" — and inheritance is resolution logic buildPendingByLeaf does" + " not have. Here it needs exactly one sanitizer and exactly one " + "claiming path.\n\n" + "It is a SEPARATE id space from `bucket`, and the sanitizer enforces " + "that a leaf names at most one of the two (operation wins): " + "`defaultBuckets` is a priority-ordered union, so a name that meant a " + "bucket to one leaf and an operation to another would silently mix two " + "id spaces, and selectableBucketsForKind feeds the editor's bucket " + "dropdown, where an operation must not appear as a bucket.", }; // A node of a lane's rule tree is a leaf or a group; both are documented in // AUTO_QUEUE_NODE_FIELD_DOCS below (rendered into SETTINGS.md). export type AutoQueueLeaf = { id: string; match: AutoQueueMatch; weight?: number; maxWorkers?: number | null; }; export type AutoQueueGroup = { id: string; mode: AutoQueueMode; children: AutoQueueNode[]; weight?: number; maxWorkers?: number | null; }; export type AutoQueueNode = AutoQueueLeaf | AutoQueueGroup; export const AUTO_QUEUE_NODE_FIELD_DOCS: FieldDocs = { id: "Stable node id, unique within the lane's tree. Preserved on save when " + "valid and not taken, so persisted fairness state survives an unrelated " + "edit; a missing or duplicate id is replaced with a generated one.", match: "LEAF ONLY: which videos this leaf owns (see the match table).", weight: "Relative share under a weighted-fair parent. Default 1. Ignored " + "otherwise.", maxWorkers: "Optional ceiling on concurrent in-flight workers drawn from this node " + "(and, for a group, its whole subtree). A capped node reads as \"no " + "work\" and the parent falls through to the next sibling, like an HTB " + "class ceiling. null = no cap.", mode: "GROUP ONLY: how the children compete — \"strict\" (first child with " + "work wins), \"round-robin\", or \"weighted-fair\" (by each child's " + "`weight`). Unknown values read as \"strict\".", children: "GROUP ONLY: the child nodes, in priority order for a strict group. A " + "node with a `children` array is a group; any other node is a leaf.", }; export function isGroup(node: AutoQueueNode): node is AutoQueueGroup { return Array.isArray((node as AutoQueueGroup).children); } // --- Settings (persisted in settings.json under `autoQueue`) ---------------- // Each field is documented in AUTO_QUEUE_POLICY_FIELD_DOCS below (rendered into SETTINGS.md). export type AutoQueuePolicy = { enabled: boolean; maxWorkers: number | null; replaceAutoSubs?: boolean; order?: AutoQueueOrder; snoozeUntil?: number | null; held?: boolean; root: AutoQueueGroup; }; export const AUTO_QUEUE_POLICY_FIELD_DOCS: FieldDocs = { enabled: "Master switch for this runner (transcription / download " + "independently).", maxWorkers: "Overall ceiling on concurrent in-flight workers for this runner. null " + "= no runner-level cap (the worker pool / platform queues are the real " + "throttle).", replaceAutoSubs: "Opt in to the lowest-priority \"replace YouTube auto-captions\" lane: " + "append this kind's opt-in buckets (autoSubsOnly / " + "downloadedAutoSubsOnly) to the tail of the default union, so videos " + "whose only transcript is YouTube ASR get re-done with our own engine " + "whenever nothing more important is pending. Default false — the " + "corpus-wide cost is large (an audio download plus a transcription per " + "video). A leaf can also target the bucket by name for per-channel opt-" + "in without flipping this switch. Optional: settings written before " + "this field existed lack it; the sanitizer defaults it to false.", order: "Ordering within each rule (see AutoQueueOrder). Optional exactly like " + "replaceAutoSubs: settings files written before this field existed lack" + " it, and the sanitizer defaults them to \"listed\" (today's behaviour).", snoozeUntil: "Epoch ms until which this runner idles WITHOUT stopping: next() " + "returns null so the loop stays up, re-reads settings each iteration, " + "and resumes by itself when the moment passes. null/absent/past = not " + "snoozed. Survives a restart because it lives in settings.json, not in " + "runner memory.", held: "THE LANE'S PAUSE GATE. Shut means the lane holds: every dispatch path " + "asks lib/pauseGates.ts, whose limit()/guard returns 0 so runPool idle-" + "waits. A hold, never a stop — see that file's header.\n\n" + "OPTIONAL IN THE TYPE, FILLED BY THE SANITIZER. Until slice 1.4 four " + "separate settings fields carried this — `transcriptionsPaused`, " + "`downloadsPaused`, `digest.digestsPaused` and (inverted) " + "`backfill.enabled` — so `undefined` meant \"ask the legacy field\" and " + "`sanitizePolicy` deliberately refused to default it: a default would " + "have read a paused corpus as running. S0-pause deleted those four, on " + "the precondition that the live settings.json already carried every " + "`held` key, and the default came in with them (`defaultHeldFor` — free" + " everywhere except backfill, whose field was inverted and shipped " + "held).\n\n" + "It stays optional because a reader may be handed a PARTIAL settings " + "object (laneGuards.test.ts casts one), and `isGateHeld` answers " + "`false` for a lane that carries no key at all rather than throwing.", root: "The lane's rule tree: a group whose children are groups and leaves (see the node table). A missing root is the lane's default — empty for the runner lanes, one catch-all leaf for digest and backfill. While a channel-priority document exists, the four roots are compiled from it and not hand-edited.", }; export type AutoQueueSettings = Record; // --- Lanes ------------------------------------------------------------------ // THE FOUR LANES. A lane is the DISPATCH noun: a queue key, a policy tree, a // runner, a pause gate and a console. The OPERATION is the work noun — a // registry entry with a state per video — and a leaf in a lane's tree names // channels and, where a lane carries more than one operation, an operation. // // It widened from the two runner kinds to the four names lib/pauseGates.ts // already listed. The two id spaces were always the same space; they are now // one type, and PauseLane is an alias. // // Here rather than in jobs/autoQueueState.ts (which persists their fairness // memory) for the same reason as the tree above: pauseGates.ts and // operations.ts name a lane without running one. // // ORDER IS LOAD-BEARING ONLY AS A DEFAULT: the two runner lanes come first so // anything that iterates LANES for display keeps the order the console had // before there were four of them. export const LANES = [ "transcription", "download", "digest", "backfill", ] as const; export type AutoQueueKind = (typeof LANES)[number]; // THE PIPELINE LANES (release 18): lanes that dispatch STAGES of a pipeline, // not work picked per channel — today the one publish lane. A pipeline lane // has a runner, a pause gate (lib/pauseGates.ts) and a console, but NO rule // tree: it is deliberately NOT in LANES, because nine loops compile a channel // tree per entry of LANES (channelPriority, channelWriters, the storage watch, // operationBatch, the channel snapshot, the auto-queue schema …). Its // settings are `settings.publish`, not an `autoQueue` policy. export const PIPELINE_LANES = ["publish"] as const; export type PipelineLane = (typeof PIPELINE_LANES)[number];