import { test } from "node:test"; import assert from "node:assert/strict"; import { isDrainableKind, jobKindLabel, getJobKind, kindNeedsMedia, kindNeedsText, isIngestKind, isPublishStageJobKind, jobKindIds, } from "./jobKinds"; // Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test jobs/jobKinds.test.ts // // Phase 1 of the queue refactor moved the drainable-kinds set and the label map // into one metadata table (jobKinds.ts). These snapshots pin the PRE-refactor // values verbatim, so the consolidation is provably behavior-preserving: every // kind that drained before still drains, every label is byte-identical, and // kinds that previously had no label still fall back to their raw kind. // Verbatim copy of the old DRAINABLE_KINDS set from registry.ts. const OLD_DRAINABLE = [ "whisper-all", "whisper-bucket-downloaded-no-transcript", "download-from-playlist", "download-missing", "download-missing-subs", "sync", "retry-bucket", "auto-transcribe", "auto-download", ]; // Verbatim copy of the old JOB_KIND_LABELS map from editor jobKindLabels.ts. const OLD_LABELS: Record = { "auto-transcribe": "Auto-transcribe", "auto-download": "Auto-download runner", "auto-download-unit": "Auto-download", "whisper-all": "Transcribe all", "whisper-bucket-downloaded-no-transcript": "Transcribe downloaded audio", "download-from-playlist": "Download from playlist", "download-missing": "Download missing", "download-missing-subs": "Download missing subs", "import-one": "Import video", "redownload-archive": "Archive source video", "retry-bucket": "Retry", "clean-audio-transcribed": "Clean audio", "check-kept-deleted": "Check kept videos", "persist-kept": "Persist kept videos", "backup-saved-videos": "Back up saved videos", "verify-saved-video-backup": "Verify saved-video backup", sync: "Sync", }; // Kinds added after the Phase 1 snapshot above. Pinned here so their label and // drainability are asserted too, without pretending they were part of the // original consolidation. const ADDED_KINDS: Record = { // Replace-auto-captions lane: a whisper batch (drains like every other batch) // plus the manual VTT purge (a fast file sweep — nothing to drain). "whisper-bucket-auto-subs": { label: "Replace auto-captions", drainable: true, }, "purge-superseded-auto-subs": { label: "Purge superseded auto-captions", drainable: false, }, // The two operation lanes' runners (phase 1 slice 1.2). Drainable like the // two runners they join — a drain stops picking and lets the in-flight unit // finish — and never replayable: a long-lived loop is not a unit of work, and // "retry" on one would mean "start a second dispatcher". "auto-digest": { label: "Auto-digest runner", drainable: true }, "auto-backfill": { label: "Auto-backfill runner", drainable: true }, // The hub's and the homepage's build and deploy (release 13 slice W1): /jobs // showed them by their raw kinds. One child process each — a cancel stops // it, there is nothing to drain. "build-hub": { label: "Build hub", drainable: false }, "deploy-hub": { label: "Deploy hub", drainable: false }, "build-deploy-hub": { label: "Build & deploy hub", drainable: false }, "build-homepage": { label: "Build homepage", drainable: false }, "deploy-homepage": { label: "Deploy homepage", drainable: false }, "build-deploy-homepage": { label: "Build & deploy homepage", drainable: false, }, // A list of videos persisted one at a time: a drain stops between them. "persist-videos": { label: "Persist videos", drainable: true }, // A report site's evidence media: one pass, cancelled rather than drained. "reports-prepare": { label: "Prepare report media", drainable: false }, // A report site's exports: one pass, cancelled rather than drained. "reports-export": { label: "Export reports", drainable: false }, // A batch of clip windows: stops between windows, a re-run resumes. "fetch-windows": { label: "Fetch windows", drainable: true }, // One video's metadata re-read: one spawn, cancelled rather than drained. "refresh-metadata": { label: "Refresh metadata", drainable: false }, // One file through a local worker (pnpm ops transcribe): one engine run. "transcribe-file": { label: "Transcribe file", drainable: false }, }; test("added kinds carry their pinned label and drainability", () => { for (const [kind, meta] of Object.entries(ADDED_KINDS)) { assert.equal(jobKindLabel(kind), meta.label, `${kind} label`); assert.equal(isDrainableKind(kind), meta.drainable, `${kind} drainability`); assert.notEqual(getJobKind(kind), undefined, `${kind} should be registered`); } }); test("drainable kinds match the old DRAINABLE_KINDS set exactly", () => { const drainSet = new Set(OLD_DRAINABLE); for (const kind of OLD_DRAINABLE) { assert.equal(isDrainableKind(kind), true, `${kind} should drain`); } // No kind in the table drains unless it was in the old set. for (const kind of Object.keys(OLD_LABELS)) { assert.equal( isDrainableKind(kind), drainSet.has(kind), `${kind} drainability changed`, ); } // Unknown kinds are not drainable. assert.equal(isDrainableKind("no-such-kind"), false); }); test("labels match the old JOB_KIND_LABELS map exactly", () => { for (const [kind, label] of Object.entries(OLD_LABELS)) { assert.equal(jobKindLabel(kind), label, `label for ${kind}`); } }); test("label-less and unknown kinds fall back to the raw kind", () => { // Replayable kinds that never had a label entry must still fall back. assert.equal(jobKindLabel("store-playlist"), "store-playlist"); assert.equal( jobKindLabel("clear-failed-transcriptions"), "clear-failed-transcriptions", ); // Genuinely unknown kind. assert.equal(jobKindLabel("totally-unknown"), "totally-unknown"); assert.equal(getJobKind("totally-unknown"), undefined); }); // RELEASE 17: `needsMedia` means "opens or writes the BIG file". These kinds // read only the text tier and flipped to `needsText`; the rest stay media. const TEXT_KINDS = [ "auto-digest", "digest-channel-local", "digest-channel-remote", "digest-share-cluster", "normalize-transcripts", "purge-superseded-auto-subs", "fetch-window", "fetch-windows", "evict-clips", "metadata-scan", "refresh-metadata", "download-missing-subs", "check-availability", "quick-availability-check", "check-maybe-missing", "check-kept-deleted", ]; const STILL_MEDIA = [ "auto-transcribe", "auto-download", "auto-download-unit", "auto-backfill", "whisper-all", "whisper-bucket-downloaded-no-transcript", "whisper-bucket-auto-subs", "sync", "import-one", "download-from-playlist", "download-missing", "redownload-archive", "redownload-incomplete-bucket", "retry-bucket", "persist-kept", "persist-videos", "whisper-video", "transcribe-one", "download-one-pipeline", "transcode-audio", "diarize-channel", "backfill-channel", "scan-media", "scan-media-channel", "clean-audio-transcribed", "clean-extra-audio-formats", "remove-wrong-format-audio", "reports-prepare", ]; test("release 17: the text kinds flipped from needsMedia to needsText", () => { for (const k of TEXT_KINDS) { assert.ok(getJobKind(k), `${k} is registered`); assert.equal(kindNeedsMedia(k), false, `${k} does not open a big file`); assert.equal(kindNeedsText(k), true, `${k} reads the text tier`); } }); test("release 17: every media kind keeps needsMedia, and none is also a text kind", () => { for (const k of STILL_MEDIA) { assert.ok(getJobKind(k), `${k} is registered`); assert.equal(kindNeedsMedia(k), true, k); assert.equal(kindNeedsText(k), false, k); } // The movers fix an unreachable channel and declare neither. for (const k of ["relocate-channel-media", "relocate-saved-videos", "repoint-storage-location"]) { assert.equal(kindNeedsMedia(k), false, k); assert.equal(kindNeedsText(k), false, k); } }); test("the feed metadata backfill reads and writes the text tier only", () => { assert.ok(getJobKind("feed-metadata"), "registered"); assert.equal(kindNeedsMedia("feed-metadata"), false); assert.equal(kindNeedsText("feed-metadata"), true); assert.equal(jobKindLabel("feed-metadata"), "Backfill feed metadata"); assert.equal(isDrainableKind("feed-metadata"), false); }); // RELEASE 18: the seven publish stages, the lane's runner, and the kinds the // stages replace (label-only: archived metas still read). const PUBLISH_LABELS: Record = { "publish-update-index": "Update the index", "publish-build-site": "Build site", "publish-deploy-site": "Deploy site", "publish-build-hub": "Build hub", "publish-deploy-hub": "Deploy hub", "publish-build-homepage": "Build homepage", "publish-deploy-homepage": "Deploy homepage", }; test("release 18: the seven publish stages are replayable, undrainable, labelled", () => { for (const [kind, label] of Object.entries(PUBLISH_LABELS)) { const meta = getJobKind(kind); assert.ok(meta, `${kind} is registered`); assert.equal(meta.label, label); assert.equal(meta.replayable, true, `${kind} is replayable`); assert.equal(isDrainableKind(kind), false, `${kind} is a child process: cancel, not drain`); assert.equal(kindNeedsMedia(kind), false); assert.equal(isPublishStageJobKind(kind), true); } assert.equal(isPublishStageJobKind("publish-nothing"), false); assert.equal(isPublishStageJobKind("build-hub"), false); assert.equal(isPublishStageJobKind(undefined), false); const runner = getJobKind("auto-publish"); assert.ok(runner); assert.equal(runner.label, "Auto-publish runner"); assert.equal(runner.drainable, true); assert.equal(runner.replayable, false); }); test("release 18: the kinds no longer created keep their labels (seven of them none)", () => { for (const kind of [ "build-index", "build-stats", "build-export", "build-deploy", "build-all", "build-deploy-all", "deploy-export", ]) { assert.ok(getJobKind(kind), `${kind} is registered`); // /jobs shows them by their raw kind, as it always has (jobs.spec reads it). assert.equal(jobKindLabel(kind), kind); assert.equal(getJobKind(kind)?.replayable, false); } assert.equal(jobKindLabel("build-hub"), "Build hub"); assert.equal(jobKindLabel("build-deploy-homepage"), "Build & deploy homepage"); }); test("release 18: every drainable kind but the lane runners and the clip-window batch is an ingest kind", () => { const runners = new Set(["auto-transcribe", "auto-download", "auto-digest", "auto-backfill", "auto-publish"]); // fetch-windows writes only `data//clips/`, a cache the index never reads. const notIngest = new Set(["fetch-windows"]); for (const kind of jobKindIds()) { if (!isDrainableKind(kind) || runners.has(kind) || notIngest.has(kind)) continue; assert.equal(isIngestKind(kind), true, `${kind} is drainable and per channel: an ingest kind`); } for (const kind of runners) assert.equal(isIngestKind(kind), false, kind); // The lane's own unit, and the single writers, count too. for (const kind of ["auto-download-unit", "import-one", "transcribe-one", "normalize-transcripts"]) { assert.equal(isIngestKind(kind), true, kind); } // Nothing that publishes, moves or only reads. for (const kind of ["publish-build-site", "build-export", "relocate-channel-media", "scan-media", "fetch-window", "fetch-windows"]) { assert.equal(isIngestKind(kind), false, kind); } assert.equal(isIngestKind("no-such-kind"), false); }); // RELEASE 21 D1: local media attached to held videos. It writes the big file // (the store), stops between videos, and its pointers and created records are // what the index reads — so media, drainable, and ingest. test("release 21: attach-media is a drainable media ingest kind, not replayable", () => { const meta = getJobKind("attach-media"); assert.ok(meta, "registered"); assert.equal(jobKindLabel("attach-media"), "Attach local media"); assert.equal(isDrainableKind("attach-media"), true); assert.equal(meta.replayable, false); assert.equal(kindNeedsMedia("attach-media"), true); assert.equal(kindNeedsText("attach-media"), false); assert.equal(isIngestKind("attach-media"), true); }); // RELEASE 21 D3: playable copies and torrents. Reads and writes big files, // stops between videos, and writes the manifest a site's build reads. test("release 21: prepare-playable is a drainable media ingest kind, not replayable", () => { const meta = getJobKind("prepare-playable"); assert.ok(meta, "registered"); assert.equal(jobKindLabel("prepare-playable"), "Prepare playable copies"); assert.equal(isDrainableKind("prepare-playable"), true); assert.equal(meta.replayable, false); assert.equal(kindNeedsMedia("prepare-playable"), true); assert.equal(isIngestKind("prepare-playable"), true); });