import type { ChannelConfig } from "./channelConfig"; import { TRANSCRIPTION_QUEUE, platformQueueKey, queueKeyForUrl, } from "./platform"; export { TRANSCRIPTION_QUEUE }; // The two digest lanes get SEPARATE queue keys, and that separation is the whole // point. registry.ts submits every non-empty queueKey with concurrency 1, so: // - one shared digest key would serialize the lanes, wasting the network lane // while the GPU works (and vice versa) across a multi-week sweep; // - putting the local lane on TRANSCRIPTION_QUEUE would let ollama and the // transcription engine thrash the same 8 GB of VRAM. // (A queueKey of "" means "run immediately, untracked" — never right for either // of these, both of which must be serialized against themselves.) export const DIGEST_LOCAL_QUEUE = "digest:local"; export const DIGEST_REMOTE_QUEUE = "digest:remote"; // The backfill lane (controller/operationBatch.ts). Its own key for exactly the // reason the two digest keys have theirs: registry.ts submits every non-empty // queueKey at concurrency 1, so a distinct key is the ONLY way to get a lane // that runs CONCURRENTLY with transcription and the digest lanes rather than // behind them — and that concurrency is the entire point of a catch-up lane. // // Serialized against ITSELF, which is also deliberate: catch-up work is // per-video CPU (diarization is ~500-680 s/audio-hour) and two channels' worth // at once would just thrash. How much of the machine it may take while running // is a separate question, answered by backfillLimit() rather than by the queue. export const BACKFILL_QUEUE = "backfill"; // Every channel report regeneration (`refresh-report`), one at a time // corpus-wide — jobs/snapshotScheduler.ts says why (release 17 slice D0). export const REFRESH_REPORT_QUEUE = "refresh-report"; // Per-channel queue for channel-local bookkeeping jobs (clean/clear/verify). export function channelQueueKey(slug: string): string { return `channel:${slug}`; } // ONE QUEUE FOR EVERY MEDIA RELOCATION IN THE PROCESS, and it takes no slug on // purpose. // // The obvious key here is channelQueueKey(slug) — a move is a channel-local // operation and it does have to serialize against that channel's own downloads // and transcriptions. But registry.ts submits every non-empty key at // concurrency 1 and has no cap ACROSS keys, so a per-channel key serializes a // channel only against itself: ticking ten rows on /channels and pressing Move // starts ten rsyncs at once, all writing to the SAME destination volume. That // is wrong twice over. It is slower — ten interleaved sequential writes to one // spinning platter is the worst access pattern the drive has — and it breaks // the space check, which each job runs when it STARTS: ten jobs that all start // together each measure a root none of the others has written to yet, all nine // of the later ones are credited room that is already spoken for, and they // jointly overrun it. The failure is recoverable (ENOSPC aborts the copy and the // source is untouched until a verify passes) but it costs hours of copying to // learn something one queue slot would have known. // // So a relocation runs behind every other relocation, and the run-time space // check then measures a root that no other move is writing to. Serializing // against the channel's OWN jobs is not lost: both callers refuse a channel that // has running or queued jobs before they enqueue, which is a stronger rule than // the queue's (it refuses rather than waits, for a move the operator can see is // already in flight). export const RELOCATION_QUEUE = "relocate"; export function relocationQueueKey(): string { return RELOCATION_QUEUE; } // Queue a network-bound download/pipeline job lands on: the channel's platform // queue, falling back to a per-domain queue for unrecognized hosts. export function downloadQueueKey(config: ChannelConfig): string { return config.platform ? platformQueueKey(config.platform) : queueKeyForUrl(config.url); } // CLIP WINDOWS HAVE THEIR OWN QUEUE PER PLATFORM (release 19, A5): // `clips:youtube` beside `platform:youtube`. // // A window is a few seconds of a video, asked for by a tool whose operator is // waiting on it. On the platform's download queue it waited behind whatever // held that queue — a multi-hour `persist-videos`, a channel sync — because a // queue runs one job at a time and a tier only orders the jobs still waiting, // never the one running. So the windows of a platform are serialized against // EACH OTHER (one clip stream per platform, the batches' 30–45 s gap between // two windows kept by `clip-window:` in jobs/platformGap.ts), and no // longer against the platform's long downloads. // // THE PLATFORM'S STATE IS STILL SHARED: a hold or a rate-limit cooldown on the // platform refuses a window as it refuses a download, and a 429 on either backs // the platform off for both. What changes is that one window may be fetched // while one download runs on the same platform. // // Takes the platform queue a window would otherwise have run on // (`downloadQueueKey`, `queueKeyForUrl`), so an unrecognized host keeps its // per-domain key: `platform:vimeo.com` → `clips:vimeo.com`. export function clipWindowQueueKey(platformQueue: string): string { return `clips:${platformQueue.replace(/^platform:/, "")}`; } // Single definition of how a UI-supplied queue override resolves against a // feature's default queue. `undefined` (no control wired up) → the default; // `""` → immediate (the registry runs `queueKey === ""` right away); any other // string → that named queue (trimmed). export function resolveQueueKey( defaultKey: string, override: string | undefined, ): string { return override === undefined ? defaultKey : override.trim(); } // WINDOWS CUT FROM SAVED CONTAINERS (release 21 D2) run on a queue of their own: // no platform is asked, so none of a platform's queues — or its hold, cooldown // and gap — applies, and a batch of local cuts never waits behind a network // window. The pseudo-platform a batch's group of them carries is // SAVED_VIDEO_CLIP_PLATFORM. export const SAVED_VIDEO_CLIP_QUEUE = "clips:saved-video"; export const SAVED_VIDEO_CLIP_PLATFORM = "saved-video";