// The publish stages (release 18) — the ONLY module that knows all of them. // // Publishing is seven independent, queueable stages driven by on-disk state // (publish/stamps.ts), like the ingest lanes: one index build shared by every // site build, then builds and deploys one at a time. Each stage is: // // needs(input, req) PURE: is the target fresh, stale (and why), or blocked? // argv(req) the child argv the editor spawns: ["stage", kind, target, …] // run(ctx, req) the body, in that child or in the CLI's own process // // `needs()` reads a `NeedsInput` — the minimal PublishStatus-shaped input // defined here. S3's status view (publish/publishPlan.ts, re-exported by // views/publishStatus.ts) satisfies it // from the stamps plus the job metas (`changedChannels`) and config mtimes; the // stage child builds one from disk alone (`readNeedsInput`, stageBodies.ts), // because ordering is enforced ON DISK: a stage whose precondition is not met // when it starts exits 3, whatever the queue believed when it enqueued it. // // Nothing here loads LMDB, next or the AWS SDK: the bodies are imported lazily. import type { Paths } from "../lib/paths"; import { ALL_TARGET, HOMEPAGE_TARGET, HUB_TARGET, INDEX_TARGET, deployRecordFor, type BuiltStamp, type DeployKind, type DeployedFile, type IndexStamp, } from "./stamps"; export type StageKind = | "update-index" | "build-site" | "deploy-site" | "build-hub" | "deploy-hub" | "build-homepage" | "deploy-homepage"; export const STAGE_KINDS: readonly StageKind[] = [ "update-index", "build-site", "deploy-site", "build-hub", "deploy-hub", "build-homepage", "deploy-homepage", ]; export function isStageKind(v: unknown): v is StageKind { return typeof v === "string" && (STAGE_KINDS as readonly string[]).includes(v); } export type StageRequest = { kind: StageKind; // "_index" | siteId | "_all" | "_hub" | "_homepage" target: string; runId: string; preview?: string; to?: "pages" | "local"; runner?: "local" | "docker"; force?: boolean; skipArchives?: boolean; // On-disk preconditions of a run: the index stamp (for a build) or the // target's built stamp (for a deploy) must be at least this new (ms). indexAfter?: number; builtAfter?: number; // `build site --allow-missing-media` (a report citation with no prepared // media is let through compose). Not part of the plan's shape; optional. allowMissingMedia?: boolean; }; export type Freshness = | { state: "fresh" } | { state: "stale"; reason: string } | { state: "blocked"; reason: string }; export type StageOutcome = { status: "ran" | "noop"; stamp: string; summary: string }; export type StageContext = { paths: Paths; onLog: (line: string) => void; signal: AbortSignal; }; export type Stage = { kind: StageKind; label: string; jobKind: `publish-${StageKind}`; queueKey: "publish"; needs(s: NeedsInput, r: StageRequest): Freshness; argv(r: StageRequest): string[]; run(ctx: StageContext, r: StageRequest): Promise; }; // --------------------------------------------------------------------------- // The input needs() reads (S3's PublishStatus satisfies it) // --------------------------------------------------------------------------- export type TargetState = { built: BuiltStamp | null; deployed: DeployedFile | null; // Member channels (a site's, or every listed site's for the hub) with an // ingest job ended `done` after `builtCheckedAt(built)` — the later of // `builtAt` and `checkedAt`, so a no-op build clears the chip. changedChannels: string[]; // The newest mtime (ms) of a config file this target's build reads (its // site.json, tags.json, search-aliases.json, duplicates*.json), or null. configChangedAt: number | null; // What is wrong with the bundle on disk (builtBundleProblem / builtHubProblem // / builtHomepageProblem), or null. Only asked when `built` is set. bundleProblem: string | null; // Why it is never deployed anywhere (a private site), or null. deployProblem?: string | null; // Why it cannot go to Cloudflare Pages (no project), or null. pagesProblem?: string | null; }; export type NeedsInput = { index: { stamp: IndexStamp | null; // When the newest drainable ingest job ended `done` (ms), or null. lastIngestDoneAt: number | null; // The newest mtime (ms) of an index input config file (tags.json, // search-aliases.json, duplicates*.json, sites/*/site.json, // homepage.json, the charts config), or null. configChangedAt: number | null; // indexSettingsSig over the settings as they are now; compared with the // stamp's. Absent: not judged (a caller that did not read settings). settingsSig?: string; }; sites: Record; hub: TargetState; homepage: TargetState & { // `main`'s HEAD where a repository is reachable, else null. mainHead: string | null; }; }; // --------------------------------------------------------------------------- // needs() // --------------------------------------------------------------------------- const FRESH: Freshness = { state: "fresh" }; const stale = (reason: string): Freshness => ({ state: "stale", reason }); const blocked = (reason: string): Freshness => ({ state: "blocked", reason }); export const UPDATE_INDEX_FIRST = "update the index first"; /** The deploy kind a request names: --to local, --preview , else production. */ export function deployKindOf(r: Pick): DeployKind { if (r.to === "local") return "local"; return r.preview ? "preview" : "production"; } function namesList(slugs: string[], max = 4): string { const shown = slugs.slice(0, max).join(", "); return slugs.length > max ? `${shown}, …` : shown; } function needsIndex(s: NeedsInput, r: StageRequest): Freshness { const stamp = s.index.stamp; if (!stamp) return stale("no index stamp yet"); if (r.force) return stale("forced"); if (s.index.lastIngestDoneAt !== null && s.index.lastIngestDoneAt > stamp.scannedAt) { return stale("new data since the last index"); } if (s.index.configChangedAt !== null && s.index.configChangedAt > stamp.scannedAt) { return stale("a config file changed since the last index"); } if (s.index.settingsSig !== undefined && s.index.settingsSig !== stamp.settingsSig) { return stale("the settings the index reads changed"); } return FRESH; } // The stamp a build needs, or why it is blocked. function indexGate(s: NeedsInput, r: StageRequest): Freshness | IndexStamp { const stamp = s.index.stamp; if (!stamp) return blocked(UPDATE_INDEX_FIRST); if (r.indexAfter !== undefined && stamp.builtAt < r.indexAfter) { return blocked("waiting for the index update this run started"); } return stamp; } // Stale reasons shared by the three builds, after the target's own signature. /** * When the bundle was last known to match its inputs: built, or found fresh * by a later no-op build (`checkedAt`). What `changedChannels` and a config * change are measured against. */ export function builtCheckedAt(built: BuiltStamp): number { return Math.max(built.builtAt, built.checkedAt ?? 0); } function builtStale(t: TargetState, sigMatches: boolean, sigReason: string): Freshness { const built = t.built!; if (t.changedChannels.length > 0) { const n = t.changedChannels.length; return stale(`${n} channel${n === 1 ? "" : "s"} changed (${namesList(t.changedChannels)})`); } if (t.configChangedAt !== null && t.configChangedAt > builtCheckedAt(built)) return stale("config changed"); if (!sigMatches) return stale(sigReason); if (t.bundleProblem) return stale(t.bundleProblem); return FRESH; } function needsBuildSite(s: NeedsInput, r: StageRequest): Freshness { const gate = indexGate(s, r); if ("state" in gate) return gate; if (r.target === ALL_TARGET) { const ids = Object.keys(s.sites).sort(); const staleIds = ids.filter((id) => needsBuildSite(s, { ...r, target: id }).state !== "fresh"); if (staleIds.length === 0) return FRESH; return stale(`${staleIds.length} of ${ids.length} sites to build (${namesList(staleIds)})`); } const t = s.sites[r.target]; if (!t) return blocked(`no site "${r.target}"`); const entry = gate.sites[r.target]; if (!entry) return blocked(`the index has not seen site "${r.target}" — ${UPDATE_INDEX_FIRST}`); if (r.force) return stale("forced"); if (!t.built) return stale("never built"); return builtStale(t, t.built.inputSig === entry.inputSig, "data changed"); } function needsBuildHub(s: NeedsInput, r: StageRequest): Freshness { const gate = indexGate(s, r); if ("state" in gate) return gate; if (r.force) return stale("forced"); if (!s.hub.built) return stale("never built"); return builtStale(s.hub, s.hub.built.inputSig === gate.hubSig, "the index or the pool it lists changed"); } function needsBuildHomepage(s: NeedsInput, r: StageRequest): Freshness { const gate = indexGate(s, r); if ("state" in gate) return gate; const h = s.homepage; if (r.force) return stale("forced"); if (!h.built) return stale("never built"); if (h.built.indexStampId !== gate.stampId) return stale("the index was updated"); if (h.mainHead !== null && (h.built.sourceCommit ?? null) !== h.mainHead) { return stale("main has moved since the source was published"); } if (h.bundleProblem) return stale(h.bundleProblem); return FRESH; } // Is `built` what the CURRENT index would build? (Same inputs, same bundle.) function builtFromCurrentIndex(s: NeedsInput, kind: "site" | "hub" | "homepage", id: string): boolean { const stamp = s.index.stamp; const built = kind === "site" ? s.sites[id]?.built : kind === "hub" ? s.hub.built : s.homepage.built; if (!stamp || !built) return false; if (kind === "site") return stamp.sites[id] !== undefined && built.inputSig === stamp.sites[id].inputSig; if (kind === "hub") return built.inputSig === stamp.hubSig; return built.indexStampId === stamp.stampId; } function needsDeploy( t: TargetState | undefined, name: string, buildCmd: string, r: StageRequest, // The bundle matches the current index, and that index ran at or after // `builtAfter` (a run's no-op build: nothing was rebuilt because nothing // needed to be). currentSince: (after: number) => boolean, ): Freshness { if (!t) return blocked(`no site "${name}"`); const kind = deployKindOf(r); const never = t.deployProblem ?? (kind === "local" ? null : (t.pagesProblem ?? null)); if (never) return blocked(never); const built = t.built; if (!built) return blocked(`no build of ${name} — ${buildCmd}`); if ( r.builtAfter !== undefined && builtCheckedAt(built) < r.builtAfter && !currentSince(r.builtAfter) ) { return blocked(`waiting for the build of ${name} this run started`); } if (t.bundleProblem) return blocked(t.bundleProblem); // Production ships only a build of main. A null branch (a detached HEAD, or // an image built without ARCHILYZER_BRANCH) is refused the same way. if (kind === "production" && built.branch !== "main") { return blocked( built.branch === null ? `${name} was built with no branch recorded (a detached HEAD, or an image built without ARCHILYZER_BRANCH); production ships only a build of main (deploy it as a preview)` : `${name} was built from branch "${built.branch}"; production ships only a build of main (deploy it as a preview)`, ); } if (r.force) return stale("forced"); const rec = deployRecordFor(t.deployed, kind, r.preview); if (!rec) return stale(kind === "preview" ? `never deployed to preview "${r.preview}"` : `never deployed (${kind})`); if (rec.builtStampId !== built.stampId) return stale("a newer build is not deployed"); return FRESH; } // --------------------------------------------------------------------------- // argv (the child's command line) and its parser // --------------------------------------------------------------------------- export function stageArgv(r: StageRequest): string[] { const out = ["stage", r.kind, r.target, "--run-id", r.runId]; if (r.preview) out.push("--preview", r.preview); if (r.to) out.push("--to", r.to); if (r.runner) out.push("--runner", r.runner); if (r.force) out.push("--force"); if (r.skipArchives) out.push("--skip-archives"); if (r.allowMissingMedia) out.push("--allow-missing-media"); if (r.indexAfter !== undefined) out.push("--index-after", String(r.indexAfter)); if (r.builtAfter !== undefined) out.push("--built-after", String(r.builtAfter)); return out; } /** The flags the `stage` row accepts (archilyzer.ts), by kind. */ export const STAGE_FLAGS = { "run-id": "string", preview: "string", to: "string", runner: "string", force: "boolean", "skip-archives": "boolean", "allow-missing-media": "boolean", "index-after": "string", "built-after": "string", } as const; const TARGET_RE = /^(?:_index|_all|_hub|_homepage|[a-z0-9][a-z0-9-]*)$/; /** The default target of a kind that has only one. */ export function fixedTarget(kind: StageKind): string | null { if (kind === "update-index") return INDEX_TARGET; if (kind === "build-hub" || kind === "deploy-hub") return HUB_TARGET; if (kind === "build-homepage" || kind === "deploy-homepage") return HOMEPAGE_TARGET; return null; } /** * The StageRequest a `stage [flags]` command line names, or * the usage problem. The inverse of `stageArgv`. */ export function parseStageArgs( positionals: string[], flags: Record, ): StageRequest | { error: string } { const [kind, target] = positionals; if (!isStageKind(kind)) return { error: `stage: which stage? one of ${STAGE_KINDS.join(", ")}` }; if (!target || !TARGET_RE.test(target)) return { error: `stage ${kind}: which target?` }; const fixed = fixedTarget(kind); if (fixed !== null && target !== fixed) return { error: `stage ${kind}: the target is ${fixed}` }; if (fixed === null && (target.startsWith("_") && !(kind === "build-site" && target === ALL_TARGET))) { return { error: `stage ${kind}: the target is a site id` }; } const runId = typeof flags["run-id"] === "string" ? flags["run-id"] : ""; if (!runId) return { error: `stage ${kind}: --run-id is required` }; const r: StageRequest = { kind, target, runId }; if (typeof flags.preview === "string") r.preview = flags.preview; if (flags.to !== undefined) { if (flags.to !== "pages" && flags.to !== "local") return { error: `stage ${kind}: --to is pages or local` }; r.to = flags.to; } if (flags.runner !== undefined) { if (flags.runner !== "local" && flags.runner !== "docker") return { error: `stage ${kind}: --runner is local or docker` }; r.runner = flags.runner; } if (flags.force === true) r.force = true; if (flags["skip-archives"] === true) r.skipArchives = true; if (flags["allow-missing-media"] === true) r.allowMissingMedia = true; for (const [flag, key] of [ ["index-after", "indexAfter"], ["built-after", "builtAfter"], ] as const) { const v = flags[flag]; if (v === undefined) continue; const n = typeof v === "string" ? Number(v) : NaN; if (!Number.isFinite(n)) return { error: `stage ${kind}: --${flag} is a time in ms` }; r[key] = n; } if (r.preview && r.to === "local") return { error: `stage ${kind}: --preview and --to local are two different deploys` }; return r; } // --------------------------------------------------------------------------- // The table // --------------------------------------------------------------------------- function stage( kind: StageKind, label: string, needs: (s: NeedsInput, r: StageRequest) => Freshness, ): Stage { return { kind, label, jobKind: `publish-${kind}`, queueKey: "publish", needs, argv: stageArgv, run: async (ctx, r) => (await import("./stageBodies")).runStageBody(ctx, r), }; } export const STAGES: Record = { "update-index": stage("update-index", "Update the index", needsIndex), "build-site": stage("build-site", "Build site", needsBuildSite), "deploy-site": stage("deploy-site", "Deploy site", (s, r) => needsDeploy(s.sites[r.target], r.target, `archilyzer publish build ${r.target}`, r, (after) => builtFromCurrentIndex(s, "site", r.target) && (s.index.stamp?.builtAt ?? 0) >= after)), "build-hub": stage("build-hub", "Build hub", needsBuildHub), "deploy-hub": stage("deploy-hub", "Deploy hub", (s, r) => needsDeploy(s.hub, "the hub", "archilyzer publish hub", r, (after) => builtFromCurrentIndex(s, "hub", "_hub") && (s.index.stamp?.builtAt ?? 0) >= after)), "build-homepage": stage("build-homepage", "Build homepage", needsBuildHomepage), "deploy-homepage": stage("deploy-homepage", "Deploy homepage", (s, r) => needsDeploy(s.homepage, "the homepage", "archilyzer publish homepage", r, (after) => builtFromCurrentIndex(s, "homepage", "_homepage") && (s.index.stamp?.builtAt ?? 0) >= after)), }; /** The job kind a stage runs as on the editor's `publish` queue. */ export function stageJobKind(kind: StageKind): `publish-${StageKind}` { return STAGES[kind].jobKind; }