commit 6bcce6ed122ec024bab6ba93b1e857a68fa6f2c0
parent 9fdd996f96979e768ba3761405c68e9e6045585e
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 7 Sep 2026 22:14:15 -0400
common: the sweeps' scope becomes a lane's tree, before anything is deleted
Slice 1.3 of plans/one-core-phase-1.md, first commit: the pure function that
carries ten retired settings fields onto the two lane trees, with its tests, and
nothing removed yet.
`migrateSweepsToLanes` runs on READ rather than as a one-shot script, because
the editor is not the only reader of settings.json — bin/ scripts, the MCP
server and the export build all call getSettings, and a migration that ran when
someone opened a page would give two readers two different answers about what is
armed. It applies only when `autoQueue.digest` / `.backfill` is ABSENT from the
file, which is what makes it idempotent by construction instead of by a version
marker nobody would maintain.
It never enables a lane the flag did not. That is the headline assertion of the
whole slice and it has its own test: the backfill lane as configured on this
corpus would re-fetch audio for ~66,540 videos, and an off-by-one in a migration
is not an acceptable way to find that out.
An empty channel list is ONE `{type:"all"}` leaf, not zero leaves and not a leaf
per channel, because that is what an empty `sweepChannels` meant:
`startDigestSweep` passes `scope.length > 0 ? scope : undefined` and an absent
scope is the whole corpus. A leaf per channel would have been the same thing
today and a different thing the day a channel is added.
`laneRootFromScope` is exported because the editor's arm action will call it
too: "arming a lane from the console produces the tree the migration would have
produced" is a property worth having, and the cheapest way to have it is for
there to be one leaf builder. A test asserts the equality so that a second
builder added later fails here rather than in production.
The module is in lib/ and its test is in jobs/ — architecture.test.ts forbids
lib/ importing jobs/, and the migration has to be asserted THROUGH
`sanitizeAutoQueue`, since it deliberately produces raw nodes for the sanitizer
to normalize rather than being a second implementation of it.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
2 files changed, 411 insertions(+), 0 deletions(-)
diff --git a/common/jobs/laneMigration.test.ts b/common/jobs/laneMigration.test.ts
@@ -0,0 +1,251 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { laneRootFromScope, migrateSweepsToLanes } from "../lib/laneMigration";
+import { sanitizeAutoQueue } from "./autoQueuePolicy";
+
+// The migration is asserted THROUGH THE SANITIZER, because that is how it is
+// used: it produces raw nodes and `sanitizeAutoQueue` normalizes them. Testing
+// the raw output alone would pass on a shape the sanitizer then coerced into
+// something else.
+//
+// IT LIVES IN jobs/ THOUGH THE MODULE LIVES IN lib/, and that is the layering
+// guard rather than an accident: architecture.test.ts forbids lib/ importing
+// jobs/, so a test beside the module could not reach the sanitizer it has to
+// assert through. jobs/ importing lib/ is the legal direction.
+const migrate = (file: unknown) => sanitizeAutoQueue(migrateSweepsToLanes(file));
+
+// The two lanes as the live settings.json carries them, minus the sweep fields
+// under test.
+const LIVE_LANES = {
+ transcription: {
+ enabled: true,
+ maxWorkers: null,
+ order: "listed",
+ root: {
+ id: "t-root",
+ mode: "strict",
+ children: [{ id: "t-all", match: { type: "all" } }],
+ },
+ },
+ download: {
+ enabled: true,
+ maxWorkers: null,
+ order: "listed",
+ root: {
+ id: "d-root",
+ mode: "strict",
+ children: [{ id: "d-all", match: { type: "all" } }],
+ },
+ },
+};
+
+test("the live shape: both sweeps disarmed, no channel scope", () => {
+ const out = migrate({
+ autoQueue: LIVE_LANES,
+ digest: {
+ sweepEnabled: false,
+ sweepChannels: [],
+ recencyOrder: "newest",
+ recencyReach: "corpus",
+ },
+ backfill: {
+ enabled: true,
+ sweepEnabled: false,
+ sweepKinds: [],
+ sweepChannels: [],
+ order: "newest",
+ reach: "corpus",
+ },
+ });
+
+ // THE HEADLINE ASSERTION OF THE WHOLE SLICE. A migration that enabled a lane
+ // the flag did not would arm ~66,540 audio re-downloads on this corpus.
+ assert.equal(out.digest.enabled, false);
+ assert.equal(out.backfill.enabled, false);
+
+ // The digest lane keeps `cheapest` — the composition — rather than taking
+ // `recencyOrder`, which is only its date half.
+ assert.equal(out.digest.order, "cheapest");
+ // The backfill lane's order had no second term, so it is carried verbatim.
+ assert.equal(out.backfill.order, "newest");
+
+ // An empty channel list is one catch-all leaf, which is what an empty
+ // `sweepChannels` meant to `startDigestSweep`.
+ for (const lane of ["digest", "backfill"] as const) {
+ assert.equal(out[lane].root.mode, "strict");
+ assert.deepEqual(
+ out[lane].root.children.map((c) => ("match" in c ? c.match : null)),
+ [{ type: "all" }],
+ );
+ }
+
+ // The two runner lanes are untouched by any of it.
+ assert.equal(out.transcription.enabled, true);
+ assert.equal(out.transcription.root.children.length, 1);
+ assert.equal(out.download.root.id, "d-root");
+});
+
+test("an armed digest sweep with three channels becomes three channel leaves", () => {
+ const out = migrate({
+ autoQueue: LIVE_LANES,
+ digest: {
+ sweepEnabled: true,
+ sweepChannels: ["teamrcn", "ObviousRises-rumble", "HasanAbiVODs3"],
+ },
+ });
+
+ assert.equal(out.digest.enabled, true);
+ assert.deepEqual(
+ out.digest.root.children.map((c) => ("match" in c ? c.match : null)),
+ [
+ { type: "channel", value: "teamrcn" },
+ { type: "channel", value: "ObviousRises-rumble" },
+ { type: "channel", value: "HasanAbiVODs3" },
+ ],
+ );
+ // Ids are derived from the scope, not generated, so re-arming the same scope
+ // keeps the fairness memory and pick log that leaf already had.
+ assert.deepEqual(
+ out.digest.root.children.map((c) => c.id),
+ ["digest-teamrcn", "digest-ObviousRises-rumble", "digest-HasanAbiVODs3"],
+ );
+ // Scope ORDER is priority: a strict root walks its children in order, and the
+ // sweep visited its scope in the order it was given.
+ assert.equal(out.digest.root.mode, "strict");
+});
+
+test("an armed backfill sweep with two kinds and one channel is a leaf per pair", () => {
+ const out = migrate({
+ autoQueue: LIVE_LANES,
+ backfill: {
+ sweepEnabled: true,
+ sweepKinds: ["diarization", "attribution-diarized"],
+ sweepChannels: ["teamrcn"],
+ order: "oldest",
+ },
+ });
+
+ assert.equal(out.backfill.enabled, true);
+ assert.equal(out.backfill.order, "oldest");
+ assert.deepEqual(
+ out.backfill.root.children.map((c) => ("match" in c ? c.match : null)),
+ [
+ { type: "channel", value: "teamrcn", operation: "diarization" },
+ { type: "channel", value: "teamrcn", operation: "attribution-diarized" },
+ ],
+ );
+ assert.deepEqual(
+ out.backfill.root.children.map((c) => c.id),
+ ["backfill-teamrcn-diarization", "backfill-teamrcn-attribution-diarized"],
+ );
+});
+
+test("an operation scope with no channel scope is a catch-all leaf per operation", () => {
+ const out = migrate({
+ backfill: { sweepEnabled: true, sweepKinds: ["diarization"] },
+ });
+ assert.deepEqual(
+ out.backfill.root.children.map((c) => ("match" in c ? c.match : null)),
+ [{ type: "all", operation: "diarization" }],
+ );
+});
+
+test("a file that already carries the blocks is untouched", () => {
+ const already = {
+ autoQueue: {
+ ...LIVE_LANES,
+ digest: {
+ enabled: true,
+ maxWorkers: 2,
+ order: "newest",
+ root: {
+ id: "kept",
+ mode: "round-robin",
+ children: [{ id: "kept-leaf", match: { type: "channel", value: "keep" } }],
+ },
+ },
+ backfill: {
+ enabled: false,
+ maxWorkers: null,
+ order: "listed",
+ root: { id: "kept-b", mode: "strict", children: [] },
+ },
+ },
+ // Sweep fields still on disk — tolerated, and deliberately NOT allowed to
+ // overwrite a lane block that exists. A file mid-migration has both.
+ digest: { sweepEnabled: false, sweepChannels: ["ignored"] },
+ backfill: { sweepEnabled: true, sweepChannels: ["ignored"], order: "newest" },
+ };
+ const out = migrate(already);
+
+ assert.equal(out.digest.enabled, true);
+ assert.equal(out.digest.maxWorkers, 2);
+ assert.equal(out.digest.order, "newest");
+ assert.equal(out.digest.root.mode, "round-robin");
+ assert.deepEqual(
+ out.digest.root.children.map((c) => ("match" in c ? c.match : null)),
+ [{ type: "channel", value: "keep" }],
+ );
+ // The armed sweep flag does NOT reach a lane that already has a block.
+ assert.equal(out.backfill.enabled, false);
+ assert.equal(out.backfill.order, "listed");
+ assert.deepEqual(out.backfill.root.children, []);
+
+ // Idempotent: running it over its own output changes nothing.
+ assert.deepEqual(migrate({ ...already, autoQueue: out }), out);
+});
+
+test("a settings file with no digest or backfill block at all migrates to the lane defaults", () => {
+ const out = migrate({});
+ assert.equal(out.digest.enabled, false);
+ assert.equal(out.digest.order, "cheapest");
+ assert.equal(out.backfill.enabled, false);
+ assert.equal(out.backfill.order, "listed");
+ for (const lane of ["digest", "backfill"] as const) {
+ assert.deepEqual(
+ out[lane].root.children.map((c) => ("match" in c ? c.match : null)),
+ [{ type: "all" }],
+ );
+ }
+});
+
+// THE CHECK THE SLICE PLAN ASKS FOR, as a unit test rather than as an e2e
+// assertion: arming a lane from the console with channels X must produce the
+// tree the migration would have produced from `sweepChannels: X`. It holds by
+// construction — both sides call `laneRootFromScope` — and this is what says so,
+// so that a second leaf builder added later fails here rather than in
+// production.
+test("arming a lane with a scope equals migrating the same scope", () => {
+ const channels = ["teamrcn", "ObviousRises-rumble"];
+ const armed = laneRootFromScope("digest", { channels });
+ const migrated = migrateSweepsToLanes({
+ digest: { sweepEnabled: true, sweepChannels: channels },
+ }).digest as { root: unknown };
+ assert.deepEqual(armed, migrated.root);
+
+ const armedBackfill = laneRootFromScope("backfill", {
+ channels,
+ operations: ["diarization"],
+ });
+ const migratedBackfill = migrateSweepsToLanes({
+ backfill: {
+ sweepEnabled: true,
+ sweepChannels: channels,
+ sweepKinds: ["diarization"],
+ },
+ }).backfill as { root: unknown };
+ assert.deepEqual(armedBackfill, migratedBackfill.root);
+});
+
+test("blank and duplicate scope entries are dropped rather than becoming leaves", () => {
+ const root = laneRootFromScope("digest", {
+ channels: ["a", " ", "a", " b "],
+ });
+ assert.deepEqual(
+ root.children.map((c) => ("match" in c ? c.match : null)),
+ [
+ { type: "channel", value: "a" },
+ { type: "channel", value: "b" },
+ ],
+ );
+});
diff --git a/common/lib/laneMigration.ts b/common/lib/laneMigration.ts
@@ -0,0 +1,160 @@
+// THE SWEEPS' LAST ACT: their persisted scope becomes a lane's tree.
+//
+// Two sweeps armed themselves through ten fields on `settings.digest` and
+// `settings.backfill` — a flag, a channel list, an operation list and an
+// order apiece. Slice 1.3 deletes all of that: a lane is armed by
+// `autoQueue[lane].enabled` and scoped by `autoQueue[lane].root`, the same two
+// things that have armed and scoped the transcription and download lanes since
+// the auto-queue existed.
+//
+// So the fields have to land somewhere on the way out, and this is the pure
+// function that lands them. It runs on READ (getSettings), not as a one-shot
+// script, for the reason every migration in this file's neighbourhood does: the
+// editor is not the only reader of settings.json — bin/ scripts, the MCP server
+// and the export build all call getSettings — and a migration that only ran
+// when someone opened a page would give two readers two different answers about
+// what is armed.
+//
+// THREE RULES, each of which is a bug it is written to avoid:
+//
+// 1. IT NEVER ENABLES A LANE THE FLAG DID NOT. `sweepEnabled: false` becomes
+// `enabled: false`. The backfill lane as configured on the live corpus
+// would re-fetch audio for ~66,540 videos; an off-by-one in a migration is
+// not an acceptable way to find that out.
+// 2. IT ONLY APPLIES WHEN THE LANE BLOCK IS ABSENT FROM THE FILE. A file that
+// already carries `autoQueue.digest` has been through here (or has been
+// hand-edited, which is the same claim) and is returned untouched — so the
+// function is idempotent by construction rather than by a version marker
+// nobody would maintain.
+// 3. AN EMPTY CHANNEL LIST IS ONE `{type:"all"}` LEAF, because that is what an
+// empty `sweepChannels` meant: `startDigestSweep` passes
+// `scope.length > 0 ? scope : undefined` and an absent scope is the whole
+// corpus. A leaf per channel would have been the same thing today and a
+// different thing the day a channel is added.
+//
+// It lives in lib/ and imports nothing from jobs/: it produces RAW nodes, which
+// `sanitizeAutoQueue` then normalizes exactly as it normalizes a hand-edited
+// file. That is deliberate — a migration that built already-sanitized objects
+// would be a second implementation of the sanitizer, and the two would drift.
+
+import type { AutoQueueGroup, AutoQueueLeaf, AutoQueueMatch } from "./autoQueueTypes";
+
+// A lane's scope in the terms the sweeps used: some channels, some operations.
+// Both empty means "everything", which is what an unscoped sweep meant.
+export type LaneScope = {
+ channels?: readonly string[] | undefined;
+ operations?: readonly string[] | undefined;
+};
+
+function slugs(value: unknown): string[] {
+ if (!Array.isArray(value)) return [];
+ const out: string[] = [];
+ for (const v of value) {
+ if (typeof v !== "string") continue;
+ const s = v.trim();
+ if (s && !out.includes(s)) out.push(s);
+ }
+ return out;
+}
+
+// THE ONE LEAF BUILDER, shared by the migration below and by the editor's
+// `armLaneAction`. That sharing is the point: "arming a lane from the console
+// produces the tree the migration would have produced" is a property worth
+// having, and the cheapest way to have it is for there to be one function.
+//
+// Ids are DERIVED, not generated: `digest-teamrcn`, `backfill-all-diarization`.
+// The sanitizer would happily assign `node-7`, but a lane's fairness memory and
+// its pick log are keyed by leaf id, so a stable id means re-arming the same
+// scope keeps the ledger it had.
+export function laneRootFromScope(
+ lane: string,
+ scope: LaneScope,
+): AutoQueueGroup {
+ const channels = slugs(scope.channels);
+ const operations = slugs(scope.operations);
+ const targets: AutoQueueMatch[] =
+ channels.length > 0
+ ? channels.map((value) => ({ type: "channel" as const, value }))
+ : [{ type: "all" as const }];
+ const leaf = (match: AutoQueueMatch): AutoQueueLeaf => ({
+ id: [lane, match.value ?? "all", match.operation].filter(Boolean).join("-"),
+ match,
+ weight: 1,
+ maxWorkers: null,
+ });
+ const children: AutoQueueLeaf[] =
+ operations.length > 0
+ ? targets.flatMap((match) =>
+ operations.map((operation) => leaf({ ...match, operation })),
+ )
+ : targets.map((match) => leaf(match));
+ return {
+ id: `${lane}-root`,
+ // STRICT, like every tree an operator has ever been handed: the sweeps
+ // walked their scope in order and never interleaved, and round-robin would
+ // be a dispatch change dressed up as a migration.
+ mode: "strict",
+ weight: 1,
+ maxWorkers: null,
+ children,
+ };
+}
+
+type RawRecord = Record<string, unknown>;
+
+function asRecord(value: unknown): RawRecord {
+ return value && typeof value === "object" && !Array.isArray(value)
+ ? (value as RawRecord)
+ : {};
+}
+
+// The `autoQueue` object to hand `sanitizeAutoQueue`, with the two operation
+// lanes filled in from the retired sweep fields when — and only when — the file
+// does not already spell them.
+//
+// Takes the PARSED FILE, not the merged settings object, because "absent from
+// the file" is the whole trigger and a merged object has already had the
+// defaults folded in. Returns a new object; the input is never mutated.
+export function migrateSweepsToLanes(parsed: unknown): RawRecord {
+ const file = asRecord(parsed);
+ const autoQueue = { ...asRecord(file.autoQueue) };
+
+ if (autoQueue.digest === undefined) {
+ const digest = asRecord(file.digest);
+ autoQueue.digest = {
+ enabled: digest.sweepEnabled === true,
+ // NO `order`. The lane's default is `cheapest`, which is the composition
+ // the digest batch actually ran — duration inside a day, newest day
+ // first — and `digest.recencyOrder` was only ever the DATE HALF of it.
+ // Migrating it onto `order` would replace the composition with one of its
+ // two terms and silently drop the other; the runner fixes the date half
+ // at newest-first (the live value) instead. A digest lane that wants pure
+ // recency sets `order: "newest"` and gets no duration term at all.
+ root: laneRootFromScope("digest", {
+ channels: slugs(digest.sweepChannels),
+ }),
+ };
+ }
+
+ if (autoQueue.backfill === undefined) {
+ const backfill = asRecord(file.backfill);
+ const order = backfill.order;
+ autoQueue.backfill = {
+ enabled: backfill.sweepEnabled === true,
+ // Carried, unlike digest's: the backfill lane's order was a plain
+ // recency order with no second term to lose, and the runner reads the
+ // same enum. An unrecognised value is the sanitizer's problem.
+ ...(order === undefined ? {} : { order }),
+ root: laneRootFromScope("backfill", {
+ channels: slugs(backfill.sweepChannels),
+ // `sweepKinds` was the operation scope, and empty meant "every enabled
+ // operation on the lane" — which is exactly what a leaf naming no
+ // operation draws. So an empty list produces plain channel leaves and a
+ // non-empty one produces a leaf per (channel, operation).
+ operations: slugs(backfill.sweepKinds),
+ }),
+ };
+ }
+
+ return autoQueue;
+}
