commit 7dcaea61ed552981295cf907cdab8cc934c98aff
parent 8d46c5578545b91b181d8d919a3e0b99cfc419a2
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 6 Oct 2026 11:13:44 -0400
jobs: the seven publish-* stage kinds, the auto-publish runner, label-only entries for the kinds release 18 stops creating; publish stages read "run <id> · <target>" on /jobs and are never re-queued at boot
- jobKinds: publish-update-index … publish-deploy-homepage (replayable, not
drainable), auto-publish (drainable runner), and build-index, build-stats,
build-export, build-deploy, build-all, build-deploy-all, deploy-export kept
known with NO label (their raw kind, as /jobs has always shown them)
- isIngestKind: the kinds whose `done` can change what the index reads —
every drainable per-channel kind plus the single writers and the
auto-download unit (tested against the drainable set)
- jobSpecDetail: `run <last six of runId> · <target>` for a publish stage
- bootQueuedJobs: a queued publish stage is cancelled as `publish`, "the
publish lane re-derives stages from on-disk state", never re-queued
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
6 files changed, 354 insertions(+), 3 deletions(-)
diff --git a/common/jobs/bootQueuedJobs.test.ts b/common/jobs/bootQueuedJobs.test.ts
@@ -8,6 +8,7 @@ import type { JobMeta } from "./jobMeta";
import type { JobSpec } from "./jobSpec";
import {
INTERRUPTED_REASON,
+ PUBLISH_RESTART_REASON,
STORAGE_PASS_WAIT_MS,
processIsAlive,
settleAfterStoragePass,
@@ -555,3 +556,42 @@ test("the queued pass leaves a queued meta whose writer is still alive", async (
await rm(f.root, { recursive: true, force: true });
}
});
+
+// RELEASE 18: a queued publish stage is never re-queued. The lane and Publish
+// now re-derive stages from the stamps; a stage re-queued with its run's
+// preconditions would wait on a run that is gone.
+test("a queued publish stage is cancelled as `publish`, never re-queued, even with a fresh spec", async () => {
+ const stage: JobSpec = {
+ kind: "publish-build-site",
+ slug: "jeralyzer",
+ params: { kind: "build-site", target: "jeralyzer", runId: "r1", indexAfter: BOOT - HOUR },
+ };
+ const f = await fixture([
+ { id: "P1", kind: "publish-build-site", queueKey: "publish", spec: stage },
+ { id: "P2", kind: "publish-update-index", queueKey: "publish", spec: { kind: "publish-update-index", slug: "_index" } },
+ { id: "F1", spec: SPEC },
+ ]);
+ try {
+ const rq = recordingRequeue();
+ const lines: string[] = [];
+ const res = await settleQueuedJobMetas({
+ paths: f.paths,
+ requeue: rq.fn,
+ bootedAt: BOOT,
+ log: (l) => lines.push(l),
+ });
+ assert.deepEqual(rq.calls, [SPEC]);
+ assert.deepEqual(
+ res.cancelled.map((c) => [c.id, c.category]).sort(),
+ [
+ ["P1", "publish"],
+ ["P2", "publish"],
+ ],
+ );
+ assert.equal((await f.read("P1")).cancelReason, PUBLISH_RESTART_REASON);
+ assert.match(PUBLISH_RESTART_REASON, /the publish lane re-derives stages from on-disk state/);
+ assert.match(lines.at(-1) ?? "", /cancelled 2 \(publish 2\)/);
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+});
diff --git a/common/jobs/bootQueuedJobs.ts b/common/jobs/bootQueuedJobs.ts
@@ -4,6 +4,7 @@ import { writeFileAtomic } from "../lib/jsonFile-server";
import type { Paths } from "../lib/paths";
import { metaPath, readJobMeta, type JobMeta } from "./jobMeta";
import type { JobSpec } from "./jobSpec";
+import { isPublishStageJobKind } from "./jobKinds";
// THE BOOT PASS OVER STALE `queued` METAS (release 9, B4b).
//
@@ -23,7 +24,10 @@ import type { JobSpec } from "./jobSpec";
// 1. an idle boot (`requeue: null`) cancels everything — it must not resume
// work;
// 2. `sync` is never re-queued: the heartbeat re-derives due syncs itself,
-// paced;
+// paced; nor is a publish stage (release 18, `publish-*`): the publish
+// lane and Publish now re-derive stages from the stamps on disk, and a
+// stage re-queued with its run's preconditions would wait on a run that
+// is gone;
// 3. a meta queued more than REQUEUE_MAX_AGE_MS before this boot is stale;
// 4. a meta with no replay spec cannot be re-queued;
// 5. of what is left, only the NEWEST meta per kind + channel + params is
@@ -48,7 +52,7 @@ export type RequeueFn = (
) => Promise<{ ok: true; jobId: string } | { ok: false; error: string }>;
export type CancelCategory =
- "idle" | "sync" | "stale" | "no-spec" | "superseded" | "refused";
+ "idle" | "sync" | "publish" | "stale" | "no-spec" | "superseded" | "refused";
export type BootQueuedResult = {
requeued: { id: string; newId: string }[];
@@ -56,6 +60,8 @@ export type BootQueuedResult = {
};
export const RESTART_REASON = "server restarted before it ran";
+export const PUBLISH_RESTART_REASON =
+ "server restarted; the publish lane re-derives stages from on-disk state";
export const REQUEUE_MAX_AGE_MS = 24 * 60 * 60 * 1000;
// Key a spec by what it would DO: kind + channel + bucket + params, with the
@@ -283,6 +289,8 @@ export async function settleQueuedJobMetas(
"sync",
"server restarted; the scheduler re-derives syncs",
);
+ } else if (isPublishStageJobKind(meta.kind)) {
+ await cancel(meta, "publish", PUBLISH_RESTART_REASON);
} else if (
typeof meta.queuedAt !== "number" ||
opts.bootedAt - meta.queuedAt > maxAgeMs
diff --git a/common/jobs/jobDetail.test.ts b/common/jobs/jobDetail.test.ts
@@ -0,0 +1,42 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { jobSpecDetail } from "./jobDetail";
+
+// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test jobs/jobDetail.test.ts
+
+test("a publish stage reads `run <last six of the run id> · <target>`", () => {
+ assert.equal(
+ jobSpecDetail("publish-build-site", {
+ kind: "publish-build-site",
+ slug: "jeralyzer",
+ params: { runId: "run-0000abc12-9f8e7d6c", kind: "build-site", target: "jeralyzer" },
+ }),
+ "run 8e7d6c · jeralyzer",
+ );
+ assert.equal(
+ jobSpecDetail("publish-update-index", {
+ kind: "publish-update-index",
+ slug: "_index",
+ params: { runId: "cli-abcdef" },
+ }),
+ "run abcdef · _index",
+ );
+});
+
+test("a publish stage with no run id (or no spec) adds nothing", () => {
+ assert.equal(jobSpecDetail("publish-deploy-hub", { kind: "publish-deploy-hub", slug: "_hub" }), undefined);
+ assert.equal(jobSpecDetail("publish-deploy-hub", undefined), undefined);
+});
+
+test("every other kind is unchanged: fetch-window keeps its phrase, the rest nothing", () => {
+ assert.equal(
+ jobSpecDetail("fetch-window", {
+ kind: "fetch-window",
+ slug: "c",
+ params: { requestedBy: "umtool", manifest: "m", clipId: "c1" },
+ }),
+ "umtool · m/c1",
+ );
+ assert.equal(jobSpecDetail("sync", { kind: "sync", slug: "c", params: { runId: "zzzzzzzz" } }), undefined);
+ assert.equal(jobSpecDetail(undefined, undefined), undefined);
+});
diff --git a/common/jobs/jobDetail.ts b/common/jobs/jobDetail.ts
@@ -7,16 +7,27 @@ import type { JobSpec } from "./jobSpec";
// for what. The answer is already on the job — the replay spec's params — so
// this reads it rather than adding a second place to store it.
//
+// A PUBLISH STAGE (release 18, `publish-<stage>`) reads `run <last six of the
+// run id> · <target>`: every stage one Publish now, one lane pass or one CLI
+// command enqueued carries the same run id, so a run reads as a group on
+// /jobs. The target is the spec's slug ("_index", a site id, "_hub", …).
+//
// Pure and total: a kind with nothing to add returns undefined and the row
// renders exactly as it did.
export function jobSpecDetail(
kind: string | undefined,
spec: JobSpec | null | undefined,
): string | undefined {
- if (kind !== "fetch-window") return undefined;
const p = spec?.params ?? {};
const s = (v: unknown): string | null =>
typeof v === "string" && v.trim() !== "" ? v.trim() : null;
+ if (typeof kind === "string" && kind.startsWith("publish-")) {
+ const runId = s(p.runId);
+ const target = s(spec?.slug);
+ if (!runId || !target) return undefined;
+ return `run ${runId.slice(-6)} · ${target}`;
+ }
+ if (kind !== "fetch-window") return undefined;
const by = s(p.requestedBy);
if (!by) return undefined;
const what = [s(p.manifest), s(p.clipId)].filter(Boolean).join("/");
diff --git a/common/jobs/jobKinds.test.ts b/common/jobs/jobKinds.test.ts
@@ -6,6 +6,9 @@ import {
getJobKind,
kindNeedsMedia,
kindNeedsText,
+ isIngestKind,
+ isPublishStageJobKind,
+ jobKindIds,
} from "./jobKinds";
// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test jobs/jobKinds.test.ts
@@ -211,3 +214,72 @@ test("the feed metadata backfill reads and writes the text tier only", () => {
assert.equal(jobKindLabel("feed-metadata"), "Backfill feed metadata");
assert.equal(isDrainableKind("feed-metadata"), false);
});
+
+// RELEASE 18: the seven publish stages, the lane's runner, and the kinds the
+// stages replace (label-only: archived metas still read).
+const PUBLISH_LABELS: Record<string, string> = {
+ "publish-update-index": "Update the index",
+ "publish-build-site": "Build site",
+ "publish-deploy-site": "Deploy site",
+ "publish-build-hub": "Build hub",
+ "publish-deploy-hub": "Deploy hub",
+ "publish-build-homepage": "Build homepage",
+ "publish-deploy-homepage": "Deploy homepage",
+};
+
+test("release 18: the seven publish stages are replayable, undrainable, labelled", () => {
+ for (const [kind, label] of Object.entries(PUBLISH_LABELS)) {
+ const meta = getJobKind(kind);
+ assert.ok(meta, `${kind} is registered`);
+ assert.equal(meta.label, label);
+ assert.equal(meta.replayable, true, `${kind} is replayable`);
+ assert.equal(isDrainableKind(kind), false, `${kind} is a child process: cancel, not drain`);
+ assert.equal(kindNeedsMedia(kind), false);
+ assert.equal(isPublishStageJobKind(kind), true);
+ }
+ assert.equal(isPublishStageJobKind("publish-nothing"), false);
+ assert.equal(isPublishStageJobKind("build-hub"), false);
+ assert.equal(isPublishStageJobKind(undefined), false);
+ const runner = getJobKind("auto-publish");
+ assert.ok(runner);
+ assert.equal(runner.label, "Auto-publish runner");
+ assert.equal(runner.drainable, true);
+ assert.equal(runner.replayable, false);
+});
+
+test("release 18: the kinds no longer created keep their labels (seven of them none)", () => {
+ for (const kind of [
+ "build-index",
+ "build-stats",
+ "build-export",
+ "build-deploy",
+ "build-all",
+ "build-deploy-all",
+ "deploy-export",
+ ]) {
+ assert.ok(getJobKind(kind), `${kind} is registered`);
+ // /jobs shows them by their raw kind, as it always has (jobs.spec reads it).
+ assert.equal(jobKindLabel(kind), kind);
+ assert.equal(getJobKind(kind)?.replayable, false);
+ }
+ assert.equal(jobKindLabel("build-hub"), "Build hub");
+ assert.equal(jobKindLabel("build-deploy-homepage"), "Build & deploy homepage");
+});
+
+test("release 18: every drainable kind but the lane runners is an ingest kind", () => {
+ const runners = new Set(["auto-transcribe", "auto-download", "auto-digest", "auto-backfill", "auto-publish"]);
+ for (const kind of jobKindIds()) {
+ if (!isDrainableKind(kind) || runners.has(kind)) continue;
+ assert.equal(isIngestKind(kind), true, `${kind} is drainable and per channel: an ingest kind`);
+ }
+ for (const kind of runners) assert.equal(isIngestKind(kind), false, kind);
+ // The lane's own unit, and the single writers, count too.
+ for (const kind of ["auto-download-unit", "import-one", "transcribe-one", "normalize-transcripts"]) {
+ assert.equal(isIngestKind(kind), true, kind);
+ }
+ // Nothing that publishes, moves or only reads.
+ for (const kind of ["publish-build-site", "build-export", "relocate-channel-media", "scan-media", "fetch-window"]) {
+ assert.equal(isIngestKind(kind), false, kind);
+ }
+ assert.equal(isIngestKind("no-such-kind"), false);
+});
diff --git a/common/jobs/jobKinds.ts b/common/jobs/jobKinds.ts
@@ -674,6 +674,121 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
replayable: false,
queueKeyStrategy: "custom",
},
+ // THE PUBLISH STAGES (release 18): one job per stage on the `publish` queue,
+ // each a child process (`archilyzer stage <kind> <target>`, publish/
+ // stageRun.ts) the editor spawns through runManagedCommand — so Cancel kills
+ // it, and there is nothing to drain. Replayable: the spec's params ARE the
+ // StageRequest (publish/publishStages.ts), and a replay re-asks the stage's
+ // preconditions on disk. The labels are the stages' own (publish/stages.ts
+ // STAGES[kind].label; jobKinds.test.ts holds them equal). No channelSlug and
+ // no `needsMedia`: a stage reads the index and the bundles, and the update-
+ // index child guards its own channels (the index build's hold).
+ "publish-update-index": {
+ kind: "publish-update-index",
+ label: "Update the index",
+ drainable: false,
+ replayable: true,
+ queueKeyStrategy: "custom",
+ },
+ "publish-build-site": {
+ kind: "publish-build-site",
+ label: "Build site",
+ drainable: false,
+ replayable: true,
+ queueKeyStrategy: "custom",
+ },
+ "publish-deploy-site": {
+ kind: "publish-deploy-site",
+ label: "Deploy site",
+ drainable: false,
+ replayable: true,
+ queueKeyStrategy: "custom",
+ },
+ "publish-build-hub": {
+ kind: "publish-build-hub",
+ label: "Build hub",
+ drainable: false,
+ replayable: true,
+ queueKeyStrategy: "custom",
+ },
+ "publish-deploy-hub": {
+ kind: "publish-deploy-hub",
+ label: "Deploy hub",
+ drainable: false,
+ replayable: true,
+ queueKeyStrategy: "custom",
+ },
+ "publish-build-homepage": {
+ kind: "publish-build-homepage",
+ label: "Build homepage",
+ drainable: false,
+ replayable: true,
+ queueKeyStrategy: "custom",
+ },
+ "publish-deploy-homepage": {
+ kind: "publish-deploy-homepage",
+ label: "Deploy homepage",
+ drainable: false,
+ replayable: true,
+ queueKeyStrategy: "custom",
+ },
+ // THE PUBLISH LANE'S RUNNER (publish/publishRunner.ts): one long-lived job on
+ // queueKey "", like the four auto-queue runners. Drainable — a drain lets the
+ // stage in flight finish and dispatches no more — and never replayable: a
+ // loop is not a unit of work.
+ "auto-publish": {
+ kind: "auto-publish",
+ label: "Auto-publish runner",
+ drainable: true,
+ replayable: false,
+ queueKeyStrategy: "parallel",
+ },
+ // THE KINDS RELEASE 18 NO LONGER CREATES. Their archived metas still read,
+ // so they stay known, labels exactly as they were: the six the hub and the
+ // homepage have carried since release 13 above, and these seven with NONE
+ // (/jobs has always shown them by their raw kind, and e2e reads that text).
+ "build-index": {
+ kind: "build-index",
+ drainable: false,
+ replayable: false,
+ queueKeyStrategy: "custom",
+ },
+ "build-stats": {
+ kind: "build-stats",
+ drainable: false,
+ replayable: false,
+ queueKeyStrategy: "custom",
+ },
+ "build-export": {
+ kind: "build-export",
+ drainable: false,
+ replayable: false,
+ queueKeyStrategy: "custom",
+ },
+ "build-deploy": {
+ kind: "build-deploy",
+ drainable: false,
+ replayable: false,
+ queueKeyStrategy: "custom",
+ },
+ "build-all": {
+ kind: "build-all",
+ drainable: false,
+ replayable: false,
+ queueKeyStrategy: "custom",
+ },
+ "build-deploy-all": {
+ kind: "build-deploy-all",
+ drainable: false,
+ replayable: false,
+ queueKeyStrategy: "custom",
+ },
+ "deploy-export": {
+ kind: "deploy-export",
+ drainable: false,
+ replayable: false,
+ queueKeyStrategy: "custom",
+ },
// A REPORT SITE'S EVIDENCE MEDIA (publish/reportMedia.ts): every clip its
// published reports cite, cut from the media on disk, and every cited post
// capture copied, into the site's report-media cache before its build.
@@ -808,6 +923,69 @@ export function jobKindLabel(kind: string): string {
return JOB_KINDS[kind]?.label ?? kind;
}
+// THE PUBLISH STAGES' JOB KINDS (release 18): `publish-<stage kind>`.
+export function isPublishStageJobKind(kind: string | undefined): boolean {
+ return typeof kind === "string" && kind.startsWith("publish-") && JOB_KINDS[kind] !== undefined;
+}
+
+// THE KINDS WHOSE `done` CAN CHANGE WHAT THE INDEX READS (release 18): what
+// makes the index stale, and a site's channels "changed", to the publish
+// status (publish/publishState.ts). The plan's words are "the drainable
+// kinds", and every drainable kind that runs per channel is here
+// (jobKinds.test.ts holds that) — plus the per-video and one-shot writers of
+// the same text and posts that are not drainable: the auto-download lane's
+// unit, a single import, transcription or download, the availability checks,
+// the forum import, the feed backfill and the cues sweep. The lane RUNNERS are
+// not: their meta is `running` for as long as the lane is, and their units
+// either have their own job (auto-download-unit) or none at all
+// (transcription, digest and backfill units — publishState.ts reads the
+// channel's report regeneration for those).
+const INGEST_KINDS: ReadonlySet<string> = new Set([
+ "whisper-all",
+ "whisper-bucket-downloaded-no-transcript",
+ "whisper-bucket-auto-subs",
+ "purge-superseded-auto-subs",
+ "digest-channel-local",
+ "digest-channel-remote",
+ "digest-share-cluster",
+ "normalize-transcripts",
+ "redownload-incomplete-bucket",
+ "download-from-playlist",
+ "download-missing",
+ "download-missing-subs",
+ "import-one",
+ "import-archive-org",
+ "redownload-archive",
+ "retry-bucket",
+ "diarize-channel",
+ "backfill-channel",
+ "persist-videos",
+ "persist-kept",
+ "fetch-posts",
+ "import-forum-pages",
+ "capture-posts",
+ "check-post-availability",
+ "sync",
+ "metadata-scan",
+ "feed-metadata",
+ "auto-download-unit",
+ "whisper-video",
+ "transcribe-one",
+ "download-one-pipeline",
+ "check-availability",
+ "quick-availability-check",
+ "check-maybe-missing",
+]);
+
+export function isIngestKind(kind: string | undefined): boolean {
+ return typeof kind === "string" && INGEST_KINDS.has(kind);
+}
+
+// Every registered kind (for the tests that hold one table to another).
+export function jobKindIds(): string[] {
+ return Object.keys(JOB_KINDS);
+}
+
export function isDrainableKind(kind: string): boolean {
return JOB_KINDS[kind]?.drainable ?? false;
}
