commit a1d081dd366dd337b5927e117d11386e4c4716a8
parent 39714d6514f7243a687670f4bce499a3f3569ccd
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 11 Sep 2026 12:06:17 -0400
Merge channel-priority/s2 — the sync slice
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
5 files changed, 356 insertions(+), 11 deletions(-)
diff --git a/common/jobs/syncScheduler.test.ts b/common/jobs/syncScheduler.test.ts
@@ -0,0 +1,255 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import type { ChannelConfig } from "../lib/channelConfig";
+import {
+ sanitizeChannelPriority,
+ type ChannelPriority,
+} from "../lib/channelPriority";
+import { defaultSyncScheduler } from "../lib/settings";
+import type { SyncSchedulerSettings } from "../lib/settings";
+import { emptySchedulerState } from "./syncSchedulerState";
+import {
+ buildScheduleView,
+ selectDueChannels,
+ type ChannelEntry,
+} from "./syncScheduler";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test jobs/syncScheduler.test.ts
+//
+// THE SYNC HALF OF plans/channel-priority.md (S2). `selectDueChannels` and
+// `buildScheduleView` are the two places the model reaches the cron sync, and
+// the pair must agree: the console cannot show a channel as eligible that the
+// scheduler will not schedule.
+//
+// Everything here is pure — no server, no settings file. The model is built
+// through `sanitizeChannelPriority` rather than by hand so a test can only
+// assert over a document the writer could actually produce.
+
+const HOUR = 60 * 60_000;
+const DAY = 24 * HOUR;
+const NOW = Date.parse("2026-09-11T12:00:00.000Z");
+
+function scheduler(
+ over: Partial<SyncSchedulerSettings> = {},
+): SyncSchedulerSettings {
+ return {
+ ...defaultSyncScheduler(),
+ enabled: true,
+ defaultIntervalMinutes: 60,
+ quietHoursStart: null,
+ quietHoursEnd: null,
+ ...over,
+ };
+}
+
+// A channel that IS due: last synced `overdueMs` past its 60-minute interval.
+function channel(
+ slug: string,
+ overdueMs: number,
+ over: Partial<ChannelConfig> = {},
+): ChannelEntry {
+ return {
+ slug,
+ config: {
+ handling: "youtube",
+ name: slug,
+ url: `https://www.youtube.com/@${slug}/videos`,
+ lastSyncedAt: new Date(NOW - HOUR - overdueMs).toISOString(),
+ ...over,
+ },
+ };
+}
+
+function model(doc: unknown): ChannelPriority {
+ return sanitizeChannelPriority(doc);
+}
+
+const NO_MODEL = sanitizeChannelPriority(undefined);
+
+function due(
+ channels: ChannelEntry[],
+ priority: ChannelPriority = NO_MODEL,
+ focusSlugs: string[] = [],
+) {
+ return selectDueChannels({
+ channels,
+ scheduler: scheduler(),
+ state: emptySchedulerState(),
+ activeSlugs: new Set<string>(),
+ now: NOW,
+ priority,
+ focusSlugs,
+ });
+}
+
+test("with no priority document the order is most-overdue-first, as before", () => {
+ const r = due([
+ channel("a", 1 * 60_000),
+ channel("b", 1 * DAY),
+ channel("c", 1 * HOUR),
+ ]);
+ assert.deepEqual(r.due, ["b", "c", "a"]);
+ assert.deepEqual(r.skipped, []);
+});
+
+test("a focus channel one minute overdue outranks a low channel a day overdue", () => {
+ // Plan test 6. The concurrency cap slices this list from the front, so the
+ // ordering IS the policy: whoever is first gets the tick's slots.
+ const r = due(
+ [channel("low-one", 1 * DAY), channel("focused", 1 * 60_000)],
+ model({ channels: { "low-one": { tier: "low" } } }),
+ ["focused"],
+ );
+ assert.deepEqual(r.due, ["focused", "low-one"]);
+});
+
+test("tier orders before overdue, and rank orders inside a tier", () => {
+ const r = due(
+ [
+ channel("low-stale", 10 * DAY),
+ channel("normal-fresh", 1 * 60_000),
+ channel("focus-rank-2", 1 * 60_000),
+ channel("focus-rank-1", 1 * 60_000),
+ channel("focus-unranked", 5 * DAY),
+ ],
+ model({
+ channels: {
+ "low-stale": { tier: "low" },
+ "focus-rank-1": { tier: "normal", rank: 1 },
+ "focus-rank-2": { tier: "normal", rank: 2 },
+ },
+ }),
+ ["focus-rank-1", "focus-rank-2", "focus-unranked"],
+ );
+ // focus (rank 1, rank 2, then unranked however stale) -> normal -> low.
+ assert.deepEqual(r.due, [
+ "focus-rank-1",
+ "focus-rank-2",
+ "focus-unranked",
+ "normal-fresh",
+ "low-stale",
+ ]);
+});
+
+test("most-overdue-first survives WITHIN a tier", () => {
+ const r = due(
+ [channel("n-fresh", 1 * 60_000), channel("n-stale", 1 * DAY)],
+ model({ channels: {} }),
+ );
+ assert.deepEqual(r.due, ["n-stale", "n-fresh"]);
+});
+
+test("ties all the way down keep the input order", () => {
+ // `listChannelConfigs` hands this function slug order and the sort is stable,
+ // so the pre-model tiebreak (alphabetical) is preserved by not touching it.
+ const r = due([channel("b", 1 * HOUR), channel("a", 1 * HOUR)]);
+ assert.deepEqual(r.due, ["b", "a"]);
+});
+
+test("a sync override of paused skips while the base tier stays normal", () => {
+ // The migrated `excludeFromSync` shape: stop syncing, keep every other lane.
+ const priority = model({
+ channels: { parked: { tier: "normal", overrides: { sync: "paused" } } },
+ });
+ assert.equal(priority.channels.parked.tier, "normal");
+ const r = due([channel("parked", 1 * DAY), channel("live", 1 * HOUR)], priority);
+ assert.deepEqual(r.due, ["live"]);
+ // Silently, like every other configuration gate in the loop — the run log is
+ // for holds worth reading (backoff, already running), not for "this channel
+ // does not auto-sync".
+ assert.deepEqual(r.skipped, []);
+});
+
+test("a base-paused channel with a sync override of normal still syncs", () => {
+ // The "sync only" preset (the operator's omnibased case): paused everywhere
+ // the auto lanes look, still keeping its playlist and metadata current.
+ const priority = model({
+ channels: { omnibased: { tier: "paused", overrides: { sync: "normal" } } },
+ });
+ assert.equal(priority.channels.omnibased.overrides?.sync, "normal");
+ const r = due([channel("omnibased", 1 * HOUR)], priority);
+ assert.deepEqual(r.due, ["omnibased"]);
+});
+
+test("a base-paused channel with no sync override never becomes due", () => {
+ const r = due(
+ [channel("off", 10 * DAY)],
+ model({ channels: { off: { tier: "paused" } } }),
+ );
+ assert.deepEqual(r.due, []);
+ assert.deepEqual(r.skipped, []);
+});
+
+test("focus does not rescue a channel paused for sync", () => {
+ // Focus wins over the stored tier; paused wins over focus, per operation.
+ const r = due(
+ [channel("held", 1 * DAY), channel("other", 1 * HOUR)],
+ model({ channels: { held: { tier: "paused" } } }),
+ ["held"],
+ );
+ assert.deepEqual(r.due, ["other"]);
+});
+
+test("the legacy excludeFromSync flag still skips, beside the new predicate", () => {
+ // S5 deletes this flag. Until its migration has run, it is the only thing
+ // some channels carry, so both skips have to stand.
+ const r = due(
+ [
+ channel("legacy", 1 * DAY, { excludeFromSync: true }),
+ channel("modern", 1 * HOUR),
+ ],
+ model({ channels: {} }),
+ );
+ assert.deepEqual(r.due, ["modern"]);
+ assert.deepEqual(r.skipped, []);
+});
+
+test("the schedule projection agrees with the scheduler about who is skipped", () => {
+ const channels = [
+ channel("normal", 1 * HOUR),
+ channel("sync-paused", 1 * DAY),
+ channel("base-paused", 1 * DAY),
+ channel("sync-only", 1 * HOUR),
+ channel("legacy", 1 * DAY, { excludeFromSync: true }),
+ channel("no-url", 1 * DAY, { url: undefined }),
+ channel("interval-off", 1 * DAY, { syncIntervalMinutes: 0 }),
+ ];
+ const priority = model({
+ channels: {
+ "sync-paused": { tier: "normal", overrides: { sync: "paused" } },
+ "base-paused": { tier: "paused" },
+ "sync-only": { tier: "paused", overrides: { sync: "normal" } },
+ },
+ });
+ const eligible = buildScheduleView({
+ channels,
+ scheduler: scheduler(),
+ state: emptySchedulerState(),
+ now: NOW,
+ priority,
+ })
+ .filter((v) => v.autoSyncEligible)
+ .map((v) => v.slug);
+ assert.deepEqual(eligible, ["normal", "sync-only"]);
+ // Every channel here is overdue if it is eligible at all, so the two answers
+ // are the same set — which is the invariant the console depends on.
+ assert.deepEqual([...due(channels, priority).due].sort(), [...eligible].sort());
+});
+
+test("the projection is unchanged by an empty priority document", () => {
+ const view = buildScheduleView({
+ channels: [channel("a", 1 * HOUR), channel("b", 1 * DAY, { excludeFromSync: true })],
+ scheduler: scheduler(),
+ state: emptySchedulerState(),
+ now: NOW,
+ priority: NO_MODEL,
+ });
+ assert.deepEqual(
+ view.map((v) => [v.slug, v.autoSyncEligible]),
+ [
+ ["a", true],
+ ["b", false],
+ ],
+ );
+});
diff --git a/common/jobs/syncScheduler.ts b/common/jobs/syncScheduler.ts
@@ -1,5 +1,12 @@
import type { ChannelConfig } from "../lib/channelConfig";
import type { SyncSchedulerSettings } from "../lib/settings";
+import {
+ type ChannelPriority,
+ effectiveTier,
+ isChannelPaused,
+ rankOf,
+ tierOrder,
+} from "../lib/channelPriority";
import { resolveFullSweepIntervalMinutes } from "./deepSync";
import type { SchedulerSkip, SchedulerState } from "./syncSchedulerState";
@@ -20,10 +27,23 @@ export type SelectDueInput = {
// Slugs that already have a running or queued sync job (from the registry).
activeSlugs: ReadonlySet<string>;
now: number;
+ // THE CHANNEL PRIORITY MODEL, asked for the "sync" operation and nothing
+ // else. It decides two things here and only two: which channels are skipped
+ // (effective tier `paused`) and what order the survivors come back in.
+ // Required rather than optional so a new caller cannot silently schedule a
+ // paused channel; `getSettings().channelPriority` always exists and is
+ // already sanitized (lib/settings.ts).
+ priority: ChannelPriority;
+ // The RESOLVED focus set. Focus is a compiled POSITION, never a stored tier,
+ // so it cannot come out of the model alone — a `{kind:"site"}` focus resolves
+ // against `transcripts/sites/*/site.json`, which is I/O this pure module must
+ // not do. The caller runs `resolveFocusSlugs` and hands the answer in.
+ focusSlugs?: readonly string[];
};
export type SelectDueResult = {
- // Slugs that should be synced now, ordered most-overdue first.
+ // Slugs that should be synced now, ordered focus first, then by tier, then
+ // by rank, then most-overdue first.
due: string[];
// Channels deliberately held back, with a human reason (for the run log).
// The common "not yet due" case is intentionally omitted to keep the log
@@ -79,11 +99,14 @@ export function nextEligibleAfterFailure(
return now + backoffMinutes(failures, scheduler) * 60_000;
}
-// Core selection. Evaluates each channel against the config gates, the elapsed
-// interval, the backoff window and the active-job set, then orders the winners
-// most-overdue first so a concurrency-capped tick services the stalest channels.
+// Core selection. Evaluates each channel against the config gates, the channel
+// priority model, the elapsed interval, the backoff window and the active-job
+// set, then orders the winners focus first, then by tier, then by rank, then
+// most-overdue first — so a concurrency-capped tick spends its slots on the
+// focused channels and services the stalest of them first.
export function selectDueChannels(input: SelectDueInput): SelectDueResult {
- const { channels, scheduler, state, activeSlugs, now } = input;
+ const { channels, scheduler, state, activeSlugs, now, priority } = input;
+ const focus = new Set(input.focusSlugs ?? []);
if (!scheduler.enabled) return { due: [], skipped: [] };
if (
isInQuietHours(now, scheduler.quietHoursStart, scheduler.quietHoursEnd)
@@ -92,11 +115,23 @@ export function selectDueChannels(input: SelectDueInput): SelectDueResult {
}
const skipped: SchedulerSkip[] = [];
- const due: { slug: string; overdueMs: number }[] = [];
+ const due: {
+ slug: string;
+ overdueMs: number;
+ tier: number;
+ rank: number;
+ }[] = [];
for (const { slug, config } of channels) {
if (!config.url) continue; // not auto-sync material; no noise in the log
+ // BOTH SKIPS STAND until S5 deletes the legacy one. `excludeFromSync` is
+ // the flag the priority model replaces; the migration turns each of them
+ // into `overrides: {sync: "paused"}`, and until it has run the flag is
+ // still the only thing some channels carry. Silent `continue` either way —
+ // "this channel does not auto-sync" is configuration, not a hold worth a
+ // line in the run log.
if (config.excludeFromSync) continue;
+ if (isChannelPaused(priority, slug, "sync")) continue;
const interval = resolveIntervalMinutes(config, scheduler);
if (interval <= 0) continue; // per-channel disabled
@@ -119,10 +154,33 @@ export function selectDueChannels(input: SelectDueInput): SelectDueResult {
const overdueMs = overdueAmount(config.lastSyncedAt, interval, now);
if (overdueMs === null) continue; // not yet due
- due.push({ slug, overdueMs });
+ due.push({
+ slug,
+ overdueMs,
+ // Focus outranks the stored tier; paused already left the loop above, so
+ // "focus wins over the stored tier, paused wins over focus" holds here
+ // by construction.
+ tier: tierOrder(
+ focus.has(slug) ? "focus" : effectiveTier(priority, slug, "sync"),
+ ),
+ // Unranked sorts last inside its tier, which is what an absent `rank`
+ // means everywhere else in the model.
+ rank: rankOf(priority, slug) ?? Number.POSITIVE_INFINITY,
+ });
}
- due.sort((a, b) => b.overdueMs - a.overdueMs);
+ // TIER, THEN RANK, THEN MOST-OVERDUE-FIRST. Most-overdue-first survives
+ // *within* a tier, so a focus channel due by a minute outranks a low channel
+ // due by a day and the tick's `maxConcurrentSyncs` cap
+ // (editor/app/scheduler/runTick.ts) spends its slots on focus first. Ties all
+ // the way down keep the input order — `listChannelConfigs` returns slug
+ // order and Array.prototype.sort is stable — which is the order this
+ // function returned before the model existed.
+ due.sort((a, b) => {
+ if (a.tier !== b.tier) return a.tier - b.tier;
+ if (a.rank !== b.rank) return a.rank < b.rank ? -1 : 1;
+ return b.overdueMs - a.overdueMs;
+ });
return { due: due.map((d) => d.slug), skipped };
}
@@ -132,7 +190,10 @@ export type ChannelScheduleView = {
slug: string;
name: string | null;
// True when this channel is eligible for auto-sync (scheduler on, has a url,
- // not excluded, and a positive resolved interval).
+ // not excluded, not paused for sync by the channel priority model, and a
+ // positive resolved interval). THE SAME PREDICATE `selectDueChannels` skips
+ // on, so the sync console can never show a channel as eligible that the
+ // scheduler will not schedule.
autoSyncEligible: boolean;
intervalMinutes: number; // resolved; 0 = disabled
inheritsInterval: boolean; // using the global default vs a per-channel value
@@ -164,8 +225,12 @@ export function buildScheduleView(input: {
scheduler: SyncSchedulerSettings;
state: SchedulerState;
now: number;
+ // The same model `selectDueChannels` takes, for the same reason: the
+ // projection must agree with the scheduler about who is skipped. Focus is
+ // not needed — it changes the ORDER, not who is eligible.
+ priority: ChannelPriority;
}): ChannelScheduleView[] {
- const { channels, scheduler, state, now } = input;
+ const { channels, scheduler, state, now, priority } = input;
return channels.map(({ slug, config }) => {
const interval = resolveIntervalMinutes(config, scheduler);
const sweepInterval = resolveFullSweepIntervalMinutes(config, scheduler);
@@ -194,6 +259,7 @@ export function buildScheduleView(input: {
scheduler.enabled &&
!!config.url &&
!config.excludeFromSync &&
+ !isChannelPaused(priority, slug, "sync") &&
interval > 0,
intervalMinutes: interval,
inheritsInterval: config.syncIntervalMinutes === undefined,
diff --git a/editor/app/api/widget/sync/route.ts b/editor/app/api/widget/sync/route.ts
@@ -187,6 +187,7 @@ export async function buildWidgetSyncPayload(): Promise<WidgetSyncPayload> {
scheduler: settings.syncScheduler,
state,
now,
+ priority: settings.channelPriority,
});
const eligible = view.filter((v) => v.autoSyncEligible);
let nextRunAt: number | null = null;
diff --git a/editor/app/scheduler/runTick.ts b/editor/app/scheduler/runTick.ts
@@ -24,6 +24,8 @@ import {
type SchedulerState,
} from "yt-dlp-transcript-common/jobs/syncSchedulerState";
import { isSocialChannel } from "yt-dlp-transcript-common/lib/channelConfig";
+import { resolveFocusSlugs } from "yt-dlp-transcript-common/lib/channelPriority";
+import { listSites } from "yt-dlp-transcript-common/lib/site";
import { syncAction } from "../channels/[slug]/pipelineActions";
import { fetchPostsAction } from "../channels/[slug]/socialActions";
// STORAGE CHORES riding this heartbeat because it is the one timer the editor
@@ -114,12 +116,31 @@ export async function runSchedulerTick(): Promise<SchedulerTickResult> {
slug: c.slug,
config: c.config,
}));
+ // The focus set is the only half of the priority model that costs I/O, and
+ // ONLY a `{kind:"site"}` focus pays it: `resolveFocusSlugs` reads a site's
+ // `channels[]` so a focus on a site tracks its membership instead of
+ // freezing a list. `{kind:"none"}` and `{kind:"channels"}` resolve from the
+ // document alone. No cache here — the tick runs on the heartbeat, not per
+ // grant, so one `listSites()` per tick is not a cost worth memoizing.
+ const priority = settings.channelPriority;
+ const siteChannels: Record<string, string[]> = {};
+ if (priority.focus.kind === "site") {
+ for (const site of listSites(paths)) {
+ siteChannels[site.siteId] = site.channels.map((c) => c.slug);
+ }
+ }
const { due, skipped } = selectDueChannels({
channels,
scheduler,
state,
activeSlugs,
now,
+ priority,
+ focusSlugs: resolveFocusSlugs(
+ priority,
+ siteChannels,
+ channels.map((c) => c.slug),
+ ),
});
// Concurrency cap doubles as the stagger: queue at most (cap - running)
@@ -134,7 +155,8 @@ export async function runSchedulerTick(): Promise<SchedulerTickResult> {
const bySlug = new Map(channels.map((c) => [c.slug, c.config]));
for (const slug of toQueue) {
// The scheduler's ELIGIBILITY rules are source-agnostic (url +
- // excludeFromSync + interval + lastSyncedAt), but the dispatch is not: a
+ // excludeFromSync + sync tier + interval + lastSyncedAt), but the
+ // dispatch is not: a
// social channel must run a post fetch, not a yt-dlp video sync against
// its profile URL.
const result = isSocialChannel(bySlug.get(slug))
diff --git a/editor/app/scheduler/status.ts b/editor/app/scheduler/status.ts
@@ -41,6 +41,7 @@ export async function buildSchedulerStatusPayload(): Promise<SchedulerStatusPayl
scheduler: settings.syncScheduler,
state,
now,
+ priority: settings.channelPriority,
});
return {
now,
