import { test } from "node:test"; import assert from "node:assert/strict"; import { type AutoQueueGroup, type AutoQueueLeaf, type AutoQueueRuntime, type ChannelWork, buildPendingByLeaf, bucketIdsFrom, bucketLaneWorkIds, bucketsForKind, defaultBucketsForPolicy, defaultDrawsForPolicy, isGroup, optInBucketsForKind, selectableBucketsForKind, emptyAutoQueueRuntime, flattenLeaves, policyDrawsBucket, selectNextWork, } from "./autoQueuePolicy"; // REPOINTED, NOT REWRITTEN (one-core phase 3 slice 4a). The defaults and the // sanitizer moved to lib/autoQueueSchema.ts; every assertion below is the one it // was, which is the point — this file is the proof that the move changed no // answer, from the `held` defaults to the tree normalisation. import { defaultAutoQueue, sanitizeAutoQueue, sanitizeAutoQueueOrder, } from "../lib/autoQueueSchema"; import { bucketLaneOperationId } from "../lib/operations"; // Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test common/jobs/autoQueuePolicy.test.ts // (or `node_modules/.bin/tsx --test common/jobs/autoQueuePolicy.test.ts` from the repo root) // Drain a policy: repeatedly selectNextWork, consuming the chosen video from // the pending map, until nothing is left. Returns the leaf ids in served order. function drain( root: AutoQueueGroup, pending: Record, runtime: AutoQueueRuntime = emptyAutoQueueRuntime(), max = 1000, ): string[] { const order: string[] = []; for (let i = 0; i < max; i++) { const pick = selectNextWork(root, pending, runtime); if (!pick) break; order.push(pick.leafId); const ids = pending[pick.leafId]; assert.equal(ids[0], pick.videoId, "pick should be head of leaf queue"); ids.shift(); } return order; } test("strict: drains highest-priority leaf first, then falls back", () => { const root: AutoQueueGroup = { id: "root", mode: "strict", children: [ { id: "corn", match: { type: "channel", value: "cornbreadman" } }, { id: "hasan", match: { type: "channel", value: "hasanabi" } }, ], }; const pending = { corn: ["c1", "c2"], hasan: ["h1", "h2", "h3"] }; assert.deepEqual(drain(root, pending), [ "corn", "corn", "hasan", "hasan", "hasan", ]); }); test("strict: a higher-priority arrival is served before lower (preemption-at-next-slot)", () => { const root: AutoQueueGroup = { id: "root", mode: "strict", children: [ { id: "corn", match: { type: "channel", value: "cornbreadman" } }, { id: "hasan", match: { type: "channel", value: "hasanabi" } }, ], }; const runtime = emptyAutoQueueRuntime(); // Mid-"hasanabi batch": only hasan has work. const pending: Record = { corn: [], hasan: ["h1", "h2"] }; let pick = selectNextWork(root, pending, runtime); assert.equal(pick?.leafId, "hasan"); pending.hasan.shift(); // cornbreadman gains a video — next selection jumps to it. pending.corn.push("c1"); pick = selectNextWork(root, pending, runtime); assert.equal(pick?.leafId, "corn"); }); test("round-robin: alternates A B A B between non-empty children", () => { const root: AutoQueueGroup = { id: "root", mode: "round-robin", children: [ { id: "a", match: { type: "channel", value: "a" } }, { id: "b", match: { type: "channel", value: "b" } }, ], }; const pending = { a: ["a1", "a2", "a3"], b: ["b1", "b2", "b3"] }; const order = drain(root, pending); // SWRR with equal weights interleaves; first two picks cover both children. assert.deepEqual(new Set(order.slice(0, 2)), new Set(["a", "b"])); assert.equal(order.filter((x) => x === "a").length, 3); assert.equal(order.filter((x) => x === "b").length, 3); // No three-in-a-row of the same leaf while both have work. assert.ok(!/(.)\1\1/.test(order.join(""))); }); test("round-robin: skips an empty child without burning its turn", () => { const root: AutoQueueGroup = { id: "root", mode: "round-robin", children: [ { id: "a", match: { type: "channel", value: "a" } }, { id: "b", match: { type: "channel", value: "b" } }, ], }; const pending = { a: ["a1", "a2"], b: [] }; assert.deepEqual(drain(root, pending), ["a", "a"]); }); test("weighted-fair: honors the 2:1 weight ratio", () => { const root: AutoQueueGroup = { id: "root", mode: "weighted-fair", children: [ { id: "a", match: { type: "channel", value: "a" }, weight: 2 }, { id: "b", match: { type: "channel", value: "b" }, weight: 1 }, ], }; const pending = { a: Array.from({ length: 20 }, (_, i) => `a${i}`), b: Array.from({ length: 20 }, (_, i) => `b${i}`), }; const order = drain(root, pending, emptyAutoQueueRuntime(), 30); // Over the first 30 picks, A should be served ~twice as often as B. const a = order.slice(0, 30).filter((x) => x === "a").length; const b = order.slice(0, 30).filter((x) => x === "b").length; assert.equal(a + b, 30); assert.ok(Math.abs(a - 20) <= 1, `expected ~20 A picks, got ${a}`); assert.ok(Math.abs(b - 10) <= 1, `expected ~10 B picks, got ${b}`); }); test("maxWorkers: a saturated leaf falls through to the next-priority sibling", () => { const root: AutoQueueGroup = { id: "root", mode: "strict", children: [ { id: "corn", match: { type: "channel", value: "corn" }, maxWorkers: 1 }, { id: "hasan", match: { type: "channel", value: "hasan" } }, ], }; const pending = { corn: ["c1", "c2"], hasan: ["h1"] }; // corn already has 1 in-flight worker -> saturated -> hasan wins. const pick = selectNextWork(root, pending, emptyAutoQueueRuntime(), { corn: 1 }); assert.equal(pick?.leafId, "hasan"); }); test("maxWorkers: a capped ancestor group blocks its whole subtree", () => { const root: AutoQueueGroup = { id: "root", mode: "strict", children: [ { id: "vips", mode: "strict", maxWorkers: 2, children: [ { id: "corn", match: { type: "channel", value: "corn" } }, { id: "hasan", match: { type: "channel", value: "hasan" } }, ], }, { id: "rest", match: { type: "all" } }, ], }; const pending = { corn: ["c1"], hasan: ["h1"], rest: ["r1"] }; // The vips group has 2 in-flight (at its cap) -> skip it -> rest wins. const pick = selectNextWork(root, pending, emptyAutoQueueRuntime(), { vips: 2 }); assert.equal(pick?.leafId, "rest"); // Path includes every ancestor id for the chosen leaf. assert.deepEqual(pick?.path, ["root", "rest"]); }); test("path: returns root..leaf ids for nested selection", () => { const root: AutoQueueGroup = { id: "root", mode: "strict", children: [ { id: "g", mode: "round-robin", children: [{ id: "corn", match: { type: "channel", value: "corn" } }], }, ], }; const pick = selectNextWork(root, { corn: ["c1"] }, emptyAutoQueueRuntime()); assert.deepEqual(pick?.path, ["root", "g", "corn"]); }); test("selectNextWork: returns null when everything is empty", () => { const root: AutoQueueGroup = { id: "root", mode: "strict", children: [{ id: "a", match: { type: "channel", value: "a" } }], }; assert.equal(selectNextWork(root, { a: [] }, emptyAutoQueueRuntime()), null); }); // --- buildPendingByLeaf ----------------------------------------------------- const CHANNELS: ChannelWork[] = [ { slug: "cornbreadman", platform: "youtube", buckets: { downloadedNoTranscript: ["c1", "c2"], failedListed: ["cf1"] }, }, { slug: "hasanabi", platform: "twitch", buckets: { downloadedNoTranscript: ["h1"], failedListed: [] }, }, ]; test("buildPendingByLeaf: channel + platform + all matchers", () => { const root: AutoQueueGroup = { id: "root", mode: "strict", children: [ { id: "corn", match: { type: "channel", value: "cornbreadman" } }, { id: "tw", match: { type: "platform", value: "twitch" } }, { id: "rest", match: { type: "all" } }, ], }; const pending = buildPendingByLeaf(root, CHANNELS, ["downloadedNoTranscript"]); assert.deepEqual(pending.corn, ["c1", "c2"]); assert.deepEqual(pending.tw, ["h1"]); // hasanabi is twitch assert.deepEqual(pending.rest, []); // both channels already claimed }); test("buildPendingByLeaf: first-match-wins prevents double-claiming a video", () => { const root: AutoQueueGroup = { id: "root", mode: "strict", children: [ { id: "all1", match: { type: "all" } }, { id: "corn", match: { type: "channel", value: "cornbreadman" } }, ], }; const pending = buildPendingByLeaf(root, CHANNELS, ["downloadedNoTranscript"]); assert.deepEqual(pending.all1.sort(), ["c1", "c2", "h1"]); assert.deepEqual(pending.corn, []); // already claimed by the catch-all above it }); test("buildPendingByLeaf: a bucket leaf draws from its named bucket", () => { const root: AutoQueueGroup = { id: "root", mode: "strict", children: [ { id: "retry", match: { type: "channel", value: "cornbreadman", bucket: "failedListed" }, }, { id: "corn", match: { type: "channel", value: "cornbreadman" } }, ], }; const pending = buildPendingByLeaf(root, CHANNELS, ["downloadedNoTranscript"]); assert.deepEqual(pending.retry, ["cf1"]); assert.deepEqual(pending.corn, ["c1", "c2"]); }); test("buildPendingByLeaf: a bucketless leaf unions all default buckets, deduped", () => { const root: AutoQueueGroup = { id: "root", mode: "strict", children: [{ id: "corn", match: { type: "channel", value: "cornbreadman" } }], }; // cornbreadman has c1,c2 in downloadedNoTranscript and cf1 in failedListed; a // bucketless leaf draws from both. const pending = buildPendingByLeaf(root, CHANNELS, [ "downloadedNoTranscript", "failedListed", ]); assert.deepEqual(pending.corn, ["c1", "c2", "cf1"]); }); test("buildPendingByLeaf: union follows default-bucket list order", () => { const channels: ChannelWork[] = [ { slug: "cornbreadman", platform: "odysee", // a4 is in BOTH buckets — it must be taken once, under the earlier bucket. buckets: { partialDownloads: ["p1", "a4"], undownloadedIds: ["a4", "u1"] }, }, ]; const root: AutoQueueGroup = { id: "root", mode: "strict", children: [{ id: "corn", match: { type: "channel", value: "cornbreadman" } }], }; // Partials first (DOWNLOAD_BUCKETS order), then the fresh ids, deduped. const pending = buildPendingByLeaf(root, channels, [ "partialDownloads", "undownloadedIds", ]); assert.deepEqual(pending.corn, ["p1", "a4", "u1"]); }); test("buildPendingByLeaf: partial-only channel is still picked (regression)", () => { // The cornbreadman bug: remaining work drifted entirely into partialDownloads, // leaving undownloadedIds empty — the old single-bucket default skipped it. const channels: ChannelWork[] = [ { slug: "cornbreadman", platform: "odysee", buckets: { undownloadedIds: [], partialDownloads: ["p1", "p2"] }, }, ]; const root: AutoQueueGroup = { id: "root", mode: "strict", children: [{ id: "corn", match: { type: "channel", value: "cornbreadman" } }], }; const pending = buildPendingByLeaf(root, channels, bucketsForKind("download")); assert.deepEqual(pending.corn, ["p1", "p2"]); }); test("bucketsForKind: per-kind ordered bucket lists", () => { assert.deepEqual( [...bucketsForKind("transcription")], ["downloadedNoTranscript", "failedListed"], ); // `chatOnlyPending` is LAST: a chat fetch is the cheapest work on the lane and // must never delay a real download. It is [] for every channel that has not // set downloadFilter.rejectedLivestreams, so the order below is unchanged for // all of them. assert.deepEqual( [...bucketsForKind("download")], ["partialDownloads", "undownloadedIds", "chatOnlyPending"], ); }); test("flattenLeaves: pre-order priority order", () => { const root: AutoQueueGroup = { id: "root", mode: "strict", children: [ { id: "g", mode: "strict", children: [ { id: "a", match: { type: "all" } }, { id: "b", match: { type: "all" } }, ], }, { id: "c", match: { type: "all" } }, ], }; assert.deepEqual( flattenLeaves(root).map((l) => l.id), ["a", "b", "c"], ); }); // --- sanitize --------------------------------------------------------------- test("sanitizeAutoQueue: empty/garbage -> defaults", () => { assert.deepEqual(sanitizeAutoQueue(undefined), defaultAutoQueue()); assert.deepEqual(sanitizeAutoQueue("nope"), defaultAutoQueue()); const def = defaultAutoQueue(); assert.equal(def.transcription.enabled, false); assert.equal(def.transcription.root.children.length, 0); }); // THE GATE'S DEFAULT, THROUGH THE PATH EVERY SETTINGS.JSON TAKES. // // `defaultAutoQueue()` is asserted elsewhere, but no file on disk is built from // it: `getSettings` runs `sanitizeAutoQueue` over whatever the file says, and // before S0-pause that sanitizer deliberately left `held` UNDEFINED so // `isGateHeld` could fall back to the four retired pause fields. The fallback // is gone, so this is now the only thing that decides what a lane naming no // gate reads as — and getting it wrong is silent in both directions: a missing // default would resume the backfill lane on every corpus, a blanket `true` // would hold all four. // // `defaultHeldFor` is not a new policy. It is the reading the retired fields // gave such a file, preserved: `transcriptionsPaused`, `downloadsPaused` and // `digest.digestsPaused` all defaulted false (free), and `backfill.enabled` // defaulted false and was read INVERTED — so the backfill lane has shipped held // since it existed. test("sanitizeAutoQueue: a lane naming no gate gets the default its retired field gave it", () => { // Lanes PRESENT but empty — the shape a hand-edited file has, and the one a // `value == null` shortcut to `defaultAutoQueuePolicy` would never reach. const bare = sanitizeAutoQueue({ backfill: {}, digest: {} }); assert.equal(bare.backfill.held, true, "backfill ships held"); assert.equal(bare.digest.held, false); // And the two lanes the object did not mention at all. assert.equal(bare.transcription.held, false); assert.equal(bare.download.held, false); // A PRE-`held` SETTINGS OBJECT: four real policies, every other field spelled, // and no `held` key anywhere — which is every settings.json written before // slice 1.4. It comes out at the defaults, the same answer the fallback used // to compute, and nothing else about it moves. const preHeld = { transcription: { enabled: true, maxWorkers: 2, order: "listed" }, download: { enabled: true, maxWorkers: 1, order: "newest" }, digest: { enabled: false, maxWorkers: null, order: "cheapest" }, backfill: { enabled: true, maxWorkers: 1, order: "listed" }, }; const out = sanitizeAutoQueue(preHeld); assert.deepEqual( [ out.transcription.held, out.download.held, out.digest.held, out.backfill.held, ], [false, false, false, true], ); // The gate is the only thing the sanitizer supplied: the arms and the orders // it was given survive, so this cannot pass by rebuilding the policies. assert.deepEqual( [out.transcription.enabled, out.download.enabled, out.digest.enabled, out.backfill.enabled], [true, true, false, true], ); assert.equal(out.download.order, "newest"); assert.equal(out.transcription.maxWorkers, 2); // IDEMPOTENT. Re-sanitizing the sanitized object is identity for the gate — // an explicit `false` on the backfill lane is a real answer, not an absence, // so a second pass must not hold it again. const twice = sanitizeAutoQueue({ ...out, backfill: { ...out.backfill, held: false } }); assert.equal(twice.backfill.held, false); assert.equal(sanitizeAutoQueue(out).backfill.held, true); }); test("sanitizeAutoQueue: coerces a hand-written tree, assigns missing ids", () => { const raw = { transcription: { enabled: true, maxWorkers: 3, root: { id: "root", mode: "round-robin", children: [ { match: { type: "channel", value: "corn" }, weight: 2 }, { mode: "bogus", children: [{ match: { type: "all" } }] }, ], }, }, }; const s = sanitizeAutoQueue(raw); assert.equal(s.transcription.enabled, true); assert.equal(s.transcription.maxWorkers, 3); assert.equal(s.transcription.root.mode, "round-robin"); const [leaf, group] = s.transcription.root.children; assert.ok(leaf.id, "missing id was assigned"); assert.equal((leaf as { weight?: number }).weight, 2); // An invalid mode falls back to strict. assert.equal((group as AutoQueueGroup).mode, "strict"); // download policy defaults in when absent. assert.equal(s.download.enabled, false); }); test("sanitizeAutoQueue: duplicate ids are de-duplicated", () => { const raw = { transcription: { root: { id: "dup", mode: "strict", children: [ { id: "dup", match: { type: "all" } }, { id: "dup", match: { type: "all" } }, ], }, }, }; const s = sanitizeAutoQueue(raw); const ids = [ s.transcription.root.id, ...s.transcription.root.children.map((c) => c.id), ]; assert.equal(new Set(ids).size, ids.length, "all ids unique after sanitize"); }); // --- Replace-auto-captions opt-in lane -------------------------------------- test("the default union is unchanged by the opt-in buckets", () => { // Every existing auto-queue setup must keep behaving exactly as before, so the // defaults stay byte-identical and the opt-in buckets live beside them. assert.deepEqual( [...bucketsForKind("transcription")], ["downloadedNoTranscript", "failedListed"], ); assert.deepEqual( [...bucketsForKind("download")], ["partialDownloads", "undownloadedIds", "chatOnlyPending"], ); assert.deepEqual( [...defaultBucketsForPolicy("transcription", { replaceAutoSubs: false })], [...bucketsForKind("transcription")], ); assert.deepEqual( [...defaultBucketsForPolicy("download", {})], [...bucketsForKind("download")], ); }); test("selectableBucketsForKind offers defaults plus the opt-in buckets", () => { assert.deepEqual( [...selectableBucketsForKind("transcription")], ["downloadedNoTranscript", "failedListed", "downloadedAutoSubsOnly"], ); assert.deepEqual( [...selectableBucketsForKind("download")], ["partialDownloads", "undownloadedIds", "chatOnlyPending", "autoSubsOnly"], ); }); test("replaceAutoSubs appends the opt-in bucket at the TAIL (lowest priority)", () => { const buckets = defaultBucketsForPolicy("transcription", { replaceAutoSubs: true, }); assert.equal(buckets[buckets.length - 1], "downloadedAutoSubsOnly"); const channels: ChannelWork[] = [ { slug: "cornbreadman", platform: "youtube", buckets: { downloadedNoTranscript: ["c1"], failedListed: ["cf1"], downloadedAutoSubsOnly: ["ca1"], }, }, ]; const root: AutoQueueGroup = { id: "root", mode: "strict", children: [{ id: "L", match: { type: "all" } }], }; // Real work is claimed first; the auto-caption candidate lands last. assert.deepEqual(buildPendingByLeaf(root, channels, buckets).L, [ "c1", "cf1", "ca1", ]); // With the lane off, the auto-caption candidate isn't picked up at all. assert.deepEqual( buildPendingByLeaf( root, channels, defaultBucketsForPolicy("transcription", { replaceAutoSubs: false }), ).L, ["c1", "cf1"], ); }); test("a leaf can target an opt-in bucket without the runner-wide switch", () => { const channels: ChannelWork[] = [ { slug: "cornbreadman", platform: "youtube", buckets: { downloadedNoTranscript: ["c1"], downloadedAutoSubsOnly: ["ca1"], }, }, { slug: "hasanabi", platform: "twitch", buckets: { downloadedNoTranscript: ["h1"], downloadedAutoSubsOnly: ["ha1"] }, }, ]; const root: AutoQueueGroup = { id: "root", mode: "strict", children: [ { id: "auto-subs-cornbreadman", match: { type: "channel", value: "cornbreadman", bucket: "downloadedAutoSubsOnly", }, }, { id: "rest", match: { type: "all" } }, ], }; const pending = buildPendingByLeaf( root, channels, defaultBucketsForPolicy("transcription", { replaceAutoSubs: false }), ); // Per-channel opt-in: only cornbreadman's candidate is claimed, and hasanabi's // never enters the queue. assert.deepEqual(pending["auto-subs-cornbreadman"], ["ca1"]); assert.deepEqual(pending.rest, ["c1", "h1"]); }); // --- the four lanes --------------------------------------------------------- test("sanitizeAutoQueue: every lane is keyed, even ones no file names", () => { // Every settings.json in existence carries exactly two lanes. const s = sanitizeAutoQueue({ transcription: { enabled: true }, download: { enabled: true }, }); assert.deepEqual(Object.keys(s).sort(), [ "backfill", "digest", "download", "transcription", ]); }); test("sanitizeAutoQueue: the new lanes default off, strict, one catch-all leaf", () => { for (const lane of ["digest", "backfill"] as const) { const p = sanitizeAutoQueue({})[lane]; assert.equal(p.enabled, false, `${lane} must default OFF`); assert.equal(p.root.mode, "strict"); assert.equal(p.root.children.length, 1); const leaf = p.root.children[0]; assert.ok(!isGroup(leaf)); assert.deepEqual((leaf as AutoQueueLeaf).match, { type: "all" }); } }); test("sanitizeAutoQueue: the runner lanes keep their EMPTY default root", () => { // A tree that dispatches nothing is what an absent runner policy has always // meant, and the new lanes' catch-all leaf must not leak into it. for (const lane of ["transcription", "download"] as const) { assert.equal(sanitizeAutoQueue({})[lane].root.children.length, 0); assert.equal(defaultAutoQueue()[lane].root.children.length, 0); } }); test("sanitizeAutoQueue: the digest lane defaults to cheapest, the rest to listed", () => { const s = sanitizeAutoQueue({}); assert.equal(s.digest.order, "cheapest"); assert.equal(s.backfill.order, "listed"); assert.equal(s.transcription.order, "listed"); assert.equal(s.download.order, "listed"); // An explicit stored order still wins over the lane default. assert.equal(sanitizeAutoQueue({ digest: { order: "newest" } }).digest.order, "newest"); // And a garbage one falls back to "listed", not to the lane default — // sanitizeAutoQueueOrder is the one coercion and it does not know lanes. assert.equal(sanitizeAutoQueue({ digest: { order: "soonest" } }).digest.order, "listed"); }); test("sanitizeAutoQueueOrder accepts cheapest", () => { assert.equal(sanitizeAutoQueueOrder("cheapest"), "cheapest"); assert.equal(sanitizeAutoQueueOrder("cheap"), "listed"); }); test("sanitizeAutoQueue: a stored root on a new lane replaces the default leaf", () => { const s = sanitizeAutoQueue({ digest: { enabled: true, root: { mode: "strict", children: [] } }, }); assert.equal(s.digest.enabled, true); assert.equal(s.digest.root.children.length, 0); }); test("bucketsForKind: the operation lanes have no buckets", () => { for (const lane of ["digest", "backfill"] as const) { assert.deepEqual([...bucketsForKind(lane)], []); assert.deepEqual([...optInBucketsForKind(lane)], []); assert.deepEqual([...selectableBucketsForKind(lane)], []); assert.deepEqual( [...defaultBucketsForPolicy(lane, { replaceAutoSubs: true })], [], ); } }); test("an operation leaf in a runner lane's tree draws NOTHING", () => { // THE SLICE'S HEADLINE INVARIANT, and the one retainLeaves("buckets") used to // write by zeroing the leaf after the fact. A leaf naming `digest` inside the // auto-transcribe tree is legal to write and always has been; getting this // wrong hands a digest candidate to whisper. // // It now holds BY CONSTRUCTION: the transcription lane projects no `digest` // list (operationsForLane("transcription") is empty, so autoRunner's // buildChannelWork never puts one in ChannelWork.operations), and a leaf with // no list to draw from comes back empty on its own. const root: AutoQueueGroup = { id: "root", mode: "strict", children: [ // FIRST in priority order, so if it could claim anything it would claim // it out from under the bucket leaves below. { id: "digest-leaf", match: { type: "all", operation: "digest" } }, { id: "cornbread", match: { type: "channel", value: "cornbreadman" } }, { id: "rest", match: { type: "all" } }, ], }; const channels: ChannelWork[] = [ { slug: "cornbreadman", platform: "youtube", buckets: { downloadedNoTranscript: ["c1"], failedListed: ["c2"] }, // Exactly what the transcription lane projects: no operations map at all. }, { slug: "hasanabi", platform: "youtube", buckets: { downloadedNoTranscript: ["h1"], failedListed: [] }, }, ]; const pending = buildPendingByLeaf( root, channels, bucketsForKind("transcription"), ); assert.deepEqual(pending["digest-leaf"], []); // ... and the bucket leaves are untouched: the operation leaf stole nothing. assert.deepEqual(pending.cornbread, ["c1", "c2"]); assert.deepEqual(pending.rest, ["h1"]); // The mirror image, so this is a statement about the PROJECTION and not about // operation leaves being inert: hand the same tree a lane that DOES project // `digest`, and the same leaf claims it. const withDigest: ChannelWork[] = channels.map((c) => ({ ...c, operations: { digest: [`${c.slug}-d`] }, })); const pending2 = buildPendingByLeaf( root, withDigest, bucketsForKind("transcription"), ); assert.deepEqual(pending2["digest-leaf"], ["cornbreadman-d", "hasanabi-d"]); }); test("sanitizeAutoQueue defaults replaceAutoSubs to false", () => { assert.equal(defaultAutoQueue().transcription.replaceAutoSubs, false); assert.equal(sanitizeAutoQueue({}).download.replaceAutoSubs, false); // Only an explicit `true` turns the lane on. assert.equal( sanitizeAutoQueue({ transcription: { replaceAutoSubs: "yes" } }) .transcription.replaceAutoSubs, false, ); assert.equal( sanitizeAutoQueue({ transcription: { replaceAutoSubs: true } }).transcription .replaceAutoSubs, true, ); }); // --- Ordering within a rule (policy.order) ---------------------------------- // The comparator the runner supplies is built from real upload dates; here a // literal date map stands in, so these tests exercise the ENGINE's contract — // sort each leaf's finished list, change nothing else — without any disk. function byDate( dates: Record, dir: "newest" | "oldest", ): (a: string, b: string) => number { const olderFirst = dir === "newest" ? 1 : -1; return (a, b) => { const ka = dates[a] ?? ""; const kb = dates[b] ?? ""; if (ka === kb) return 0; return ka < kb ? olderFirst : -olderFirst; }; } const ORDER_CHANNELS: ChannelWork[] = [ { slug: "cornbreadman", platform: "odysee", buckets: { downloadedNoTranscript: ["c_old", "c_new"], failedListed: [] }, }, { slug: "destiny", platform: "youtube", buckets: { downloadedNoTranscript: ["d_mid"], failedListed: [] }, }, ]; const ORDER_DATES = { c_old: "20240101", c_new: "20260812", d_mid: "20250601", }; const ORDER_ROOT: AutoQueueGroup = { id: "root", mode: "strict", children: [{ id: "all", match: { type: "all" } }], }; test("buildPendingByLeaf: no comparator reproduces today's order exactly", () => { // The whole compatibility claim for this feature in one assertion: an absent // `compare` must not merely produce a "similar" list, it must produce the // identical one, so every existing settings.json keeps its behaviour. const before = buildPendingByLeaf(ORDER_ROOT, ORDER_CHANNELS, [ "downloadedNoTranscript", "failedListed", ]); const after = buildPendingByLeaf( ORDER_ROOT, ORDER_CHANNELS, ["downloadedNoTranscript", "failedListed"], {}, ); assert.deepEqual(before.all, ["c_old", "c_new", "d_mid"]); assert.deepEqual(after, before); }); test("buildPendingByLeaf: a comparator reorders within a leaf, across channels", () => { const newest = buildPendingByLeaf( ORDER_ROOT, ORDER_CHANNELS, ["downloadedNoTranscript", "failedListed"], { compare: byDate(ORDER_DATES, "newest") }, ); assert.deepEqual(newest.all, ["c_new", "d_mid", "c_old"]); const oldest = buildPendingByLeaf( ORDER_ROOT, ORDER_CHANNELS, ["downloadedNoTranscript", "failedListed"], { compare: byDate(ORDER_DATES, "oldest") }, ); assert.deepEqual(oldest.all, ["c_old", "d_mid", "c_new"]); }); test("ordering sorts INSIDE a rule; rule order still wins", () => { // The documented semantic. `corn` is declared first, so its videos are served // first even though destiny's is newer than one of them — ordering never // promotes a video past a higher-priority RULE. const root: AutoQueueGroup = { id: "root", mode: "strict", children: [ { id: "corn", match: { type: "channel", value: "cornbreadman" } }, { id: "rest", match: { type: "all" } }, ], }; const pending = buildPendingByLeaf( root, ORDER_CHANNELS, ["downloadedNoTranscript", "failedListed"], { compare: byDate(ORDER_DATES, "newest") }, ); assert.deepEqual(pending.corn, ["c_new", "c_old"]); assert.deepEqual(pending.rest, ["d_mid"]); assert.deepEqual(drain(root, pending), ["corn", "corn", "rest"]); }); test("ordering does not change CLAIMING (bucket priority still wins)", () => { // A video in two buckets is still attributed once, to the higher-priority // bucket — sorting happens after claiming, never instead of it. const channels: ChannelWork[] = [ { slug: "cornbreadman", platform: "odysee", buckets: { partialDownloads: ["shared"], undownloadedIds: ["shared", "fresh"], }, }, ]; const root: AutoQueueGroup = { id: "root", mode: "strict", children: [ { id: "partials", match: { type: "all", bucket: "partialDownloads" } }, { id: "rest", match: { type: "all" } }, ], }; const pending = buildPendingByLeaf( root, channels, ["partialDownloads", "undownloadedIds"], { compare: byDate({ shared: "20200101", fresh: "20260101" }, "newest") }, ); assert.deepEqual(pending.partials, ["shared"]); assert.deepEqual(pending.rest, ["fresh"]); }); test("equal keys keep listed order (stable sort, no arbitrary tiebreak)", () => { const channels: ChannelWork[] = [ { slug: "cornbreadman", platform: "odysee", // Same upload date: whatever order they arrived in must survive, which is // what preserves playlist order for undownloadedIds. buckets: { undownloadedIds: ["zzz", "aaa", "mmm"] }, }, ]; const pending = buildPendingByLeaf( { id: "root", mode: "strict", children: [{ id: "all", match: { type: "all" } }] }, channels, ["undownloadedIds"], { compare: byDate({ zzz: "20260101", aaa: "20260101", mmm: "20260101" }, "newest") }, ); assert.deepEqual(pending.all, ["zzz", "aaa", "mmm"]); }); test("sanitize: order defaults to listed and rejects junk", () => { const s = sanitizeAutoQueue({ transcription: { enabled: true, order: "newest", root: {} }, download: { enabled: true, order: "sideways", root: {} }, }); assert.equal(s.transcription.order, "newest"); assert.equal(s.download.order, "listed"); // A settings.json written before the field existed reads as today's behaviour. assert.equal(sanitizeAutoQueue({}).transcription.order, "listed"); assert.equal(defaultAutoQueue().download.order, "listed"); }); test("sanitize: a lapsed snooze normalizes to null, a future one survives", () => { const future = Date.now() + 60_000; const s = sanitizeAutoQueue({ transcription: { snoozeUntil: future, root: {} }, download: { snoozeUntil: Date.now() - 60_000, root: {} }, }); assert.equal(s.transcription.snoozeUntil, future); assert.equal(s.download.snoozeUntil, null); assert.equal(sanitizeAutoQueue({}).transcription.snoozeUntil, null); assert.equal(sanitizeAutoQueue({ download: { snoozeUntil: "soon" } }).download.snoozeUntil, null); }); // --- Operations: a leaf can name one ----------------------------------------- const OP_CHANNELS: ChannelWork[] = [ { slug: "ch", platform: "youtube", buckets: { downloadedNoTranscript: ["v1", "v2"] }, // The SAME videos are pending for two different operations, which is the // normal case: a transcript can want a digest and its audio a diarization. operations: { digest: ["v1", "v3"], diarization: ["v1", "v4"] }, }, ]; test("a leaf naming an operation draws from operations, not buckets", () => { const pending = buildPendingByLeaf( { id: "root", mode: "strict", children: [ { id: "dig", match: { type: "all", operation: "digest" } }, { id: "rest", match: { type: "all" } }, ], }, OP_CHANNELS, ["downloadedNoTranscript"], ); assert.deepEqual(pending.dig, ["v1", "v3"]); // And the bucket leaf is UNAFFECTED: v1 being claimed for digest must not // remove it from the transcription queue, because they are different work. assert.deepEqual(pending.rest, ["v1", "v2"]); }); test("two operations claim the same video independently", () => { // The whole reason the claim key carries the operation. An id-keyed set would // let whichever leaf ran first take v1 and silently drop it from the other // operation's queue. const pending = buildPendingByLeaf( { id: "root", mode: "strict", children: [ { id: "dig", match: { type: "all", operation: "digest" } }, { id: "dia", match: { type: "all", operation: "diarization" } }, ], }, OP_CHANNELS, ["downloadedNoTranscript"], ); assert.deepEqual(pending.dig, ["v1", "v3"]); assert.deepEqual(pending.dia, ["v1", "v4"]); }); test("two leaves on the SAME operation still dedup, first-match-wins", () => { const pending = buildPendingByLeaf( { id: "root", mode: "strict", children: [ { id: "first", match: { type: "channel", value: "ch", operation: "digest" } }, { id: "second", match: { type: "all", operation: "digest" } }, ], }, OP_CHANNELS, ["downloadedNoTranscript"], ); assert.deepEqual(pending.first, ["v1", "v3"]); assert.deepEqual(pending.second, []); }); test("a projection with no operations map leaves an operation leaf empty", () => { // Every ChannelWork written before operations existed omits the map. The leaf // must find nothing rather than throw or fall back to the buckets. const pending = buildPendingByLeaf( { id: "root", mode: "strict", children: [{ id: "dig", match: { type: "all", operation: "digest" } }] }, [{ slug: "ch", platform: "youtube", buckets: { downloadedNoTranscript: ["v1"] } }], ["downloadedNoTranscript"], ); assert.deepEqual(pending.dig, []); }); test("a tree with no operation is byte-for-byte what it was", () => { // The compatibility claim: every settings.json in existence names no // operation, so every claim key is "\0"+id and the dedup is unchanged. const before = buildPendingByLeaf(ORDER_ROOT, ORDER_CHANNELS, [ "downloadedNoTranscript", "failedListed", ]); assert.deepEqual(before.all, ["c_old", "c_new", "d_mid"]); }); test("sanitize: operation wins and DROPS bucket, so the tree cannot be ambiguous", () => { const s = sanitizeAutoQueue({ transcription: { root: { id: "root", mode: "strict", children: [ { id: "a", match: { type: "all", operation: " digest ", bucket: "failedListed" } }, { id: "b", match: { type: "all", bucket: "failedListed" } }, { id: "c", match: { type: "all", operation: " " } }, ], }, }, download: { root: {} }, }); const leaves = flattenLeaves(s.transcription.root); assert.deepEqual(leaves[0].match, { type: "all", operation: "digest" }); // An untouched bucket leaf keeps working exactly as before. assert.deepEqual(leaves[1].match, { type: "all", bucket: "failedListed" }); // A blank operation is not an operation. assert.deepEqual(leaves[2].match, { type: "all" }); }); // --------------------------------------------------------------------------- // policyDrawsBucket — "would the runner actually pick this up?", asked from // outside the runner. The backfill hand-off (controller/backfillReacquire.ts) // keeps a re-acquired audio file only when the answer is yes, so a wrong answer // here either leaks a file onto a 97%-full disk or deletes one out from under a // running engine. const LEAF_ALL: AutoQueueGroup = { id: "root", mode: "strict", children: [{ id: "leaf-all", match: { type: "all" } }], }; test("policyDrawsBucket: an opt-in bucket needs replaceAutoSubs or an explicit leaf", () => { const off = { replaceAutoSubs: false, root: LEAF_ALL }; const on = { replaceAutoSubs: true, root: LEAF_ALL }; const ch = { slug: "chan", platform: "youtube" as const }; // A bucket-less catch-all draws the DEFAULT union, which excludes the opt-in // bucket until replaceAutoSubs appends it. assert.equal( policyDrawsBucket("transcription", off, ch, "downloadedAutoSubsOnly"), false, ); assert.equal( policyDrawsBucket("transcription", on, ch, "downloadedAutoSubsOnly"), true, ); // The always-drawn buckets are drawn either way. assert.equal( policyDrawsBucket("transcription", off, ch, "downloadedNoTranscript"), true, ); // The second way in: a leaf naming the bucket, with the flag still off. This // is the case that makes `replaceAutoSubs: false` NOT a refusal. const explicit = { replaceAutoSubs: false, root: { id: "root", mode: "strict" as const, children: [ { id: "leaf-bucket", match: { type: "all" as const, bucket: "downloadedAutoSubsOnly", }, }, ], }, }; assert.equal( policyDrawsBucket("transcription", explicit, ch, "downloadedAutoSubsOnly"), true, ); // …and it draws ONLY that bucket. assert.equal( policyDrawsBucket("transcription", explicit, ch, "downloadedNoTranscript"), false, ); }); test("policyDrawsBucket: channel and platform leaves cover only what they match", () => { const byChannel = { replaceAutoSubs: true, root: { id: "root", mode: "strict" as const, children: [ { id: "l", match: { type: "channel" as const, value: "mine" } }, ], }, }; assert.equal( policyDrawsBucket( "transcription", byChannel, { slug: "mine", platform: "youtube" }, "downloadedAutoSubsOnly", ), true, ); assert.equal( policyDrawsBucket( "transcription", byChannel, { slug: "other", platform: "youtube" }, "downloadedAutoSubsOnly", ), false, ); const byPlatform = { replaceAutoSubs: true, root: { id: "root", mode: "strict" as const, children: [ { id: "l", match: { type: "platform" as const, value: "rumble" } }, ], }, }; assert.equal( policyDrawsBucket( "transcription", byPlatform, { slug: "mine", platform: "rumble" }, "downloadedAutoSubsOnly", ), true, ); assert.equal( policyDrawsBucket( "transcription", byPlatform, { slug: "mine", platform: "youtube" }, "downloadedAutoSubsOnly", ), false, ); // An unknown platform is not a wildcard. assert.equal( policyDrawsBucket( "transcription", byPlatform, { slug: "mine", platform: null }, "downloadedAutoSubsOnly", ), false, ); }); test("policyDrawsBucket: an operation leaf draws no bucket at all", () => { const ops = { replaceAutoSubs: true, root: { id: "root", mode: "strict" as const, children: [ { id: "l", match: { type: "all" as const, operation: "diarization" }, }, ], }, }; assert.equal( policyDrawsBucket( "transcription", ops, { slug: "mine", platform: "youtube" }, "downloadedAutoSubsOnly", ), false, ); assert.equal( policyDrawsBucket( "transcription", ops, { slug: "mine", platform: "youtube" }, "downloadedNoTranscript", ), false, ); }); test("policyDrawsBucket: an empty tree draws nothing", () => { assert.equal( policyDrawsBucket( "transcription", defaultAutoQueue().transcription, { slug: "mine", platform: "youtube" }, "downloadedNoTranscript", ), false, ); }); // --------------------------------------------------------------------------- // Slice 1.5: one work list per lane. // // The bucket lanes' default draw stopped being "walk these two bucket names" // and became "draw the lane's work list", projected from // snapshot.backfill[op].ids with bucketLaneWorkIds as its fallback. The whole // claim of the change is that NOTHING A LEAF DRAWS MOVES, so these tests are // equivalence tests: the same tree, over the same corpus, under both // projections. // A snapshot-shaped corpus with every trap the two projections could differ on: // a video in two default buckets (dedup), an opt-in bucket that must stay at // the tail, and a channel whose undownloadedIds are NOT sorted (playlist order). const CORPUS = [ { slug: "alpha", platform: "youtube" as const, buckets: { partialDownloads: ["p1"], downloadedNoTranscript: ["t2", "t1"], failedListed: ["t1", "t3"], downloadedAutoSubsOnly: ["a1"], autoSubsOnly: ["a2"], }, undownloadedIds: ["zz9", "aa1", "mm5"], }, { slug: "beta", platform: "rumble" as const, buckets: { partialDownloads: [], downloadedNoTranscript: [], failedListed: ["t9"], downloadedAutoSubsOnly: [], autoSubsOnly: [], }, undownloadedIds: ["b1"], }, ]; // Every pick, as `/