// The publish stages' bodies (release 18): what `run()` does, in the stage // child the editor spawns (`archilyzer stage …`) or in the // CLI's own process (`archilyzer publish …`), always under the publish lock // (stageRun.ts takes it). // // Every body but update-index first asks its stage's `needs()` over the state // ON DISK (`readNeedsInput`): blocked → StageFailure exit 3 (the precondition // is not met — "update the index first", "no build of X"), fresh and not // forced → a no-op. Ordering between the stages of one run is enforced here, // not in anybody's memory. // // The three deploy bodies are release 18 S2's deploy stage (deployStage.ts): // pinned wrangler, credential preflight, the live check, deployed.json. import { execFile } from "node:child_process"; import { readdir } from "node:fs/promises"; import { promisify } from "node:util"; import { builtAudienceProblem, builtBundleProblem, builtHomepageProblem, builtHubProblem, siteDeployProblem, } from "../lib/builtExport"; import { getHomepageConfig } from "../lib/homepage"; import { previewBranchProblem } from "../lib/pagesDeploy"; import type { Paths } from "../lib/paths"; import { getSettings } from "../lib/settings"; import { listSites, type Site } from "../lib/site"; import { ALL_TARGET, HOMEPAGE_TARGET, HUB_TARGET, imageBuildFacts, newStampId, readBuiltStamp, readDeployedFile, readIndexStamp, writeBuiltStamp, type BuiltKind, type BuiltStamp, type IndexStamp, type Runner, } from "./stamps"; import { STAGES, type NeedsInput, type StageContext, type StageOutcome, type StageRequest, type TargetState, } from "./stages"; /** A stage that did not run to the end: its exit code says why (stageRun.ts). */ export class StageFailure extends Error { constructor( message: string, readonly exitCode: 1 | 2 | 3, ) { super(message); this.name = "StageFailure"; } } export class StageCancelled extends Error { constructor() { super("cancelled"); this.name = "StageCancelled"; } } function checkCancel(signal: AbortSignal): void { if (signal.aborted) throw new StageCancelled(); } // --------------------------------------------------------------------------- // git facts for the stamps // --------------------------------------------------------------------------- const exec = promisify(execFile); async function git(cwd: string, args: string[]): Promise { try { const { stdout } = await exec("git", args, { cwd, timeout: 10_000 }); const out = stdout.trim(); return out || null; } catch { return null; } } /** * The commit and branch a stamp records. `ARCHILYZER_COMMIT` / * `ARCHILYZER_BRANCH`, when set, WIN over git, each on its own: the runtime * image bakes them (it has no .git), and a test server sets * `ARCHILYZER_BRANCH=main` because a worktree's branch is never `main`. Else * the checkout's HEAD and branch; a detached HEAD records `branch: null`, * which a production deploy refuses like any branch but `main`. */ export async function checkoutInfo( paths: Pick, env: NodeJS.ProcessEnv = process.env, ): Promise<{ commit: string | null; branch: string | null }> { const facts = imageBuildFacts(env); const commit = facts.commit ?? (await git(paths.monorepoRoot, ["rev-parse", "HEAD"])); if (facts.branch !== null) return { commit, branch: facts.branch }; const branch = await git(paths.monorepoRoot, ["rev-parse", "--abbrev-ref", "HEAD"]); return { commit, branch: branch === null || branch === "HEAD" ? null : branch }; } /** `main`'s HEAD where a repository is reachable, else null. */ export async function mainHeadOf(paths: Pick): Promise { return git(paths.monorepoRoot, ["rev-parse", "--verify", "--quiet", "refs/heads/main^{commit}"]); } // --------------------------------------------------------------------------- // The state needs() reads, from disk alone // --------------------------------------------------------------------------- function sitePagesProblem(site: Site): string | null { return site.cloudflareProject?.trim() ? null : `Site "${site.siteId}" has no Cloudflare Pages project configured`; } /** * The NeedsInput a stage child can build from disk alone: the stamps and the * bundles. It does not read job metas or config mtimes (`changedChannels` is * empty, `configChangedAt` null): a child judges by signatures, and the * signature in the index stamp is the authority once the index has run. */ export async function readNeedsInput( paths: Paths, opts: { mainHead?: boolean } = {}, ): Promise { const { bundleDir, homepageOutDir, hubProjectProblem } = await import("./build"); const stamp = await readIndexStamp(paths); const sites: Record = {}; for (const site of listSites(paths)) { const built = await readBuiltStamp(paths, site.siteId); sites[site.siteId] = { built, deployed: await readDeployedFile(paths, site.siteId), changedChannels: [], configChangedAt: null, bundleProblem: built ? builtBundleProblem(bundleDir(paths, site.siteId), site.siteId) : null, deployProblem: siteDeployProblem(site), pagesProblem: sitePagesProblem(site), }; } const hubBuilt = await readBuiltStamp(paths, HUB_TARGET); const homeBuilt = await readBuiltStamp(paths, HOMEPAGE_TARGET); return { index: { stamp, lastIngestDoneAt: null, configChangedAt: null }, sites, hub: { built: hubBuilt, deployed: await readDeployedFile(paths, HUB_TARGET), changedChannels: [], configChangedAt: null, bundleProblem: hubBuilt ? builtHubProblem(bundleDir(paths, HUB_TARGET)) : null, pagesProblem: hubProjectProblem(getHomepageConfig(paths).cloudflareProject), }, homepage: { built: homeBuilt, deployed: await readDeployedFile(paths, HOMEPAGE_TARGET), changedChannels: [], configChangedAt: null, bundleProblem: homeBuilt ? builtHomepageProblem(homepageOutDir(paths)) : null, mainHead: opts.mainHead ? await mainHeadOf(paths) : null, }, }; } // --------------------------------------------------------------------------- // Stamping a build // --------------------------------------------------------------------------- async function stampBuilt( paths: Paths, outDir: string, s: { target: string; kind: BuiltKind; indexStampId: string | null; inputSig: string; runner: Runner; archivesStaged: number; sourceCommit?: string | null; }, ): Promise { const { bundleCounts, corpusGeneratedAtIn } = await import("./build"); const { commit, branch } = await checkoutInfo(paths); const counts = await bundleCounts(outDir); const built: BuiltStamp = { v: 1, stampId: newStampId(), target: s.target, kind: s.kind, indexStampId: s.indexStampId, inputSig: s.inputSig, builtAt: Date.now(), commit, branch, runner: s.runner, audience: builtAudienceProblem(outDir) ? "private" : "public", corpusGeneratedAt: await corpusGeneratedAtIn(outDir), files: counts.files, bytes: counts.bytes, archivesStaged: s.archivesStaged, ...(s.sourceCommit !== undefined ? { sourceCommit: s.sourceCommit } : {}), }; await writeBuiltStamp(paths, built); return built; } function plural(n: number, word: string): string { return `${n} ${word}${n === 1 ? "" : "s"}`; } // --------------------------------------------------------------------------- // update-index // --------------------------------------------------------------------------- async function runUpdateIndex(ctx: StageContext): Promise { const { paths, onLog, signal } = ctx; const { settingsFromFile } = await import("../lib/settings"); const { applyHealthTimings } = await import("../lib/storageHealth"); const { buildIndex } = await import("../controller/buildIndex"); const { buildStats } = await import("../controller/buildStats"); const { syncTemplatesToExport } = await import("../lib/chartsStore"); const { hubInputSig, indexSettingsSig, readIndexMeta, readMemberConfigs, siteInputSig } = await import("./inputSig"); // A CLI process has no health pass: the drive-health timings the builds' // watchdog runs on are applied here, once (bin/build-index.ts does the same). applyHealthTimings(settingsFromFile(paths.settingsFile).storage.health); const log = (m: string) => onLog(m.endsWith("\n") ? m : `${m}\n`); onLog("=== update-index: the LMDB index ===\n"); const idx = await buildIndex({ paths, onLog: log }); checkCancel(signal); onLog("=== update-index: the stats datasets ===\n"); const st = await buildStats({ paths, onLog: log, signal }); checkCancel(signal); onLog("=== update-index: chart templates ===\n"); const sites = listSites(paths); for (const site of sites) syncTemplatesToExport(paths, site.siteId); const templatesAt = Date.now(); checkCancel(signal); onLog("=== update-index: signatures ===\n"); const meta = readIndexMeta( paths, sites.map((s) => s.siteId), ); const settings = getSettings(); const configs = await readMemberConfigs(paths, sites); const cache = new Map(); const siteEntries: IndexStamp["sites"] = {}; for (const site of sites) { siteEntries[site.siteId] = { siteFp: meta.siteFp[site.siteId] ?? null, statsFp: meta.statsFp[site.siteId] ?? null, inputSig: await siteInputSig({ paths, site, settings, sites, configOf: (slug) => configs.get(slug), cache, }), }; } // The same index as the last stamp (nothing rebuilt, every signature the // same) keeps its stamp id, so the builds made from it stay current. const prev = await readIndexStamp(paths); let stampId = newStampId(); let reused = false; if ( prev && idx.shortCircuited && st.shortCircuited && prev.generation === meta.generation && sameSites(prev.sites, siteEntries) && (await hubInputSig(paths, prev.stampId, sites)) === prev.hubSig ) { stampId = prev.stampId; reused = true; } const { commit } = await checkoutInfo(paths); const stamp: IndexStamp = { v: 1, stampId, generation: meta.generation, scannedAt: meta.scannedAt ?? Date.now(), builtAt: Date.now(), templatesAt, commit, index: { shortCircuited: idx.shortCircuited, added: idx.added, changed: idx.changed, removed: idx.removed, heldChannels: idx.heldChannels, }, stats: { shortCircuited: st.shortCircuited, notIndexedYet: st.notIndexedYet, notIndexable: st.notIndexable, }, sites: siteEntries, hubSig: await hubInputSig(paths, stampId, sites), settingsSig: indexSettingsSig(settings), }; const { writeIndexStamp } = await import("./stamps"); await writeIndexStamp(paths, stamp); const summary = `index +${idx.added} ~${idx.changed} -${idx.removed}` + (idx.heldChannels.length ? ` (held: ${idx.heldChannels.join(", ")})` : "") + `; ${plural(sites.length, "site")} signed` + (reused ? "; nothing changed — the stamp stands" : ""); return { status: reused ? "noop" : "ran", stamp: stampId, summary }; } function sameSites(a: IndexStamp["sites"], b: IndexStamp["sites"]): boolean { const ka = Object.keys(a).sort(); const kb = Object.keys(b).sort(); if (ka.join("\n") !== kb.join("\n")) return false; return ka.every((k) => a[k].inputSig === b[k].inputSig); } // --------------------------------------------------------------------------- // build-site (one site, every site locally, every site in containers) // --------------------------------------------------------------------------- // Inside the docker per-site build container (docker/build-site.sh sets // ARCHIVES_READONLY=1): the build stays in export/out for the container to hand // back, and the HOST stamps it — nothing is installed or stamped here. function inBuildContainer(): boolean { return process.env.ARCHIVES_READONLY === "1"; } // What a container build reads where the host wrote no stamp (the editor's // Build all before release 18's surfaces): nothing is stamped in a container. const CONTAINER_NO_STAMP: IndexStamp = { v: 1, stampId: "", generation: 0, scannedAt: 0, builtAt: 0, templatesAt: 0, commit: null, index: { shortCircuited: true, added: 0, changed: 0, removed: 0, heldChannels: [] }, stats: { shortCircuited: true, notIndexedYet: 0, notIndexable: 0 }, sites: {}, hubSig: "", }; async function buildOneSite(ctx: StageContext, r: StageRequest, stamp: IndexStamp): Promise { const { paths, onLog, signal } = ctx; const { buildSiteBundle, bundleDir } = await import("./build"); const siteId = r.target; const inPlace = inBuildContainer(); const res = await buildSiteBundle(siteId, { paths, onLog, signal, skipArchives: r.skipArchives, allowMissingMedia: r.allowMissingMedia, inPlace, }); checkCancel(signal); if (res.code !== 0) throw new StageFailure(`build of ${siteId} failed (exit ${res.code})`, 1); if (inPlace) return { status: "ran", stamp: "", summary: `${siteId} built in place (build container)` }; const built = await stampBuilt(paths, bundleDir(paths, siteId), { target: siteId, kind: "site", indexStampId: stamp.stampId, inputSig: stamp.sites[siteId].inputSig, runner: "local", archivesStaged: res.archivesStaged, }); return { status: "ran", stamp: built.stampId, summary: `${siteId} built: ${plural(built.files, "file")}, ${(built.bytes / 1e6).toFixed(1)} MB`, }; } // The sites `_all` builds: each stale one (every one with --force). // (A fresh one is a no-op build: its `checkedAt` is moved on.) async function sitesToBuild( paths: Paths, input: NeedsInput, r: StageRequest, onLog: (l: string) => void, ): Promise { const ids: string[] = []; for (const id of Object.keys(input.sites)) { const f = STAGES["build-site"].needs(input, { ...r, target: id }); if (f.state === "fresh") { onLog(`[publish] ${id}: fresh — skipped\n`); await markChecked(paths, input.sites[id].built); } else if (f.state === "blocked") onLog(`[publish] ${id}: blocked — ${f.reason}\n`); else ids.push(id); } return ids; } /** * A no-op build: the bundle still matches its inputs, as of now. Recorded as * `checkedAt`, so the "channels changed" signal (measured against * max(builtAt, checkedAt)) clears, without pretending a build happened. */ async function markChecked(paths: Paths, built: BuiltStamp | null | undefined): Promise { if (built) await writeBuiltStamp(paths, { ...built, checkedAt: Date.now() }); } async function buildAllLocal(ctx: StageContext, r: StageRequest, input: NeedsInput): Promise { const stamp = input.index.stamp!; const ids = await sitesToBuild(ctx.paths, input, r, ctx.onLog); const failed: string[] = []; let built = 0; for (const [i, id] of ids.entries()) { checkCancel(ctx.signal); ctx.onLog(`\n=== Build ${id} (${i + 1}/${ids.length}) ===\n`); try { await buildOneSite(ctx, { ...r, target: id }, stamp); built++; } catch (err) { if (err instanceof StageCancelled) throw err; failed.push(id); ctx.onLog(`[publish] ${id}: ${(err as Error).message}\n`); } } const summary = `${built}/${ids.length} built` + (failed.length ? `; failed: ${failed.join(", ")}` : ""); if (failed.length) throw new StageFailure(summary, 1); return { status: built ? "ran" : "noop", stamp: stamp.stampId, summary }; } export const NO_ENGINE = "the docker runner needs an engine on this host"; async function buildAllDocker(ctx: StageContext, r: StageRequest, input: NeedsInput): Promise { const { paths, onLog, signal } = ctx; const b = await import("./build"); if (!(await b.dockerAvailable(signal))) throw new StageFailure(NO_ENGINE, 3); const stamp = input.index.stamp!; const ids = await sitesToBuild(paths, input, r, onLog); if (ids.length === 0) return { status: "noop", stamp: stamp.stampId, summary: "every site is fresh" }; if (!r.skipArchives) { onLog("=== the archive cache (host) ===\n"); const code = await b.runHostScript(onLog, signal, paths, "build:archives"); checkCancel(signal); if (code !== 0) throw new StageFailure(`warming the archive cache failed (exit ${code})`, 1); } const img = await b.ensureBuildImage(onLog, signal, paths); checkCancel(signal); if (img !== 0) throw new StageFailure(`the build image failed (exit ${img})`, 1); const { maxParallelBuilds } = getSettings().buildPipeline; onLog(`=== building ${plural(ids.length, "site")} in containers, up to ${maxParallelBuilds} at once ===\n`); const outcomes = await b.runWithConcurrency(ids, maxParallelBuilds, async (id) => { if (signal.aborted) return { id, ok: false }; const code = await b.runDockerBuildOne(onLog, signal, id, paths, { skipArchives: r.skipArchives }); const out = b.bundleDir(paths, id); const problem = code === 0 ? builtBundleProblem(out, id) : null; if (code !== 0 || problem) { onLog(`[${id}] build FAILED — ${problem ?? `exit ${code}`}\n`); return { id, ok: false }; } const staged = await readdir(b.dockerSiteStagingDir(paths, id)).catch(() => [] as string[]); await stampBuilt(paths, out, { target: id, kind: "site", indexStampId: stamp.stampId, inputSig: stamp.sites[id].inputSig, runner: "docker", archivesStaged: staged.filter((f) => !f.startsWith(".")).length, }); onLog(`[${id}] build ok\n`); return { id, ok: true }; }); checkCancel(signal); const failed = outcomes.filter((o) => !o.ok).map((o) => o.id); const summary = `${ids.length - failed.length}/${ids.length} built in containers` + (failed.length ? `; failed: ${failed.join(", ")}` : ""); if (failed.length) throw new StageFailure(summary, 1); return { status: "ran", stamp: stamp.stampId, summary }; } // --------------------------------------------------------------------------- // hub and homepage builds // --------------------------------------------------------------------------- async function buildHubStage(ctx: StageContext, stamp: IndexStamp): Promise { const { buildHubBundle, bundleDir } = await import("./build"); const code = await buildHubBundle(ctx); checkCancel(ctx.signal); if (code !== 0) throw new StageFailure(`the hub build failed (exit ${code})`, 1); const built = await stampBuilt(ctx.paths, bundleDir(ctx.paths, HUB_TARGET), { target: HUB_TARGET, kind: "hub", indexStampId: stamp.stampId, inputSig: stamp.hubSig, runner: "local", archivesStaged: 0, }); return { status: "ran", stamp: built.stampId, summary: `hub built: ${plural(built.files, "file")}` }; } async function buildHomepageStage(ctx: StageContext, stamp: IndexStamp): Promise { const { buildHomepage, homepageOutDir } = await import("./build"); const { readPublishedManifest } = await import("./source"); const code = await buildHomepage(ctx); checkCancel(ctx.signal); if (code !== 0) throw new StageFailure(`the homepage build failed (exit ${code})`, 1); const out = homepageOutDir(ctx.paths); const manifest = await readPublishedManifest(out); const built = await stampBuilt(ctx.paths, out, { target: HOMEPAGE_TARGET, kind: "homepage", indexStampId: stamp.stampId, inputSig: stamp.stampId, runner: "local", archivesStaged: 0, sourceCommit: manifest?.sourceCommit ?? null, }); return { status: "ran", stamp: built.stampId, summary: `homepage built: ${plural(built.files, "file")}` }; } // --------------------------------------------------------------------------- // deploys // --------------------------------------------------------------------------- // The three deploy bodies are ONE function, release 18 S2's runDeployStage // (publish/deployStage.ts): the bundle guards, the credential preflight, the // archives to R2, the pinned wrangler (wranglerBin), the live check and the // `deployed.json` record — or `--to local` into ARCHILYZER_SITE_OUT / // ARCHILYZER_HOMEPAGE_OUT. Its refusals and failures are DeployStageErrors // carrying the stage's exit code; it logs each sentence itself (stageRun.ts // does not say it again). // --------------------------------------------------------------------------- // The dispatcher // --------------------------------------------------------------------------- /** Run the stage `r` names (the body of every Stage's `run`). */ export async function runStageBody(ctx: StageContext, r: StageRequest): Promise { const { paths } = ctx; if (r.kind === "update-index") return runUpdateIndex(ctx); // Usage before state: a bad preview name is said as such, whatever is built. if (r.kind.startsWith("deploy-")) { if (r.preview !== undefined) { const problem = previewBranchProblem(r.preview); if (problem) throw new StageFailure(problem, 2); } if (r.preview && r.to === "local") { throw new StageFailure("--preview and --to local are two different deploys", 2); } } // The docker per-site container builds what the host's fan-out decided to // build: no stamps of its own to judge by (its builds dir is scratch). if (r.kind === "build-site" && inBuildContainer()) { return buildOneSite(ctx, r, (await readIndexStamp(paths)) ?? CONTAINER_NO_STAMP); } const input = await readNeedsInput(paths, { mainHead: r.kind === "build-homepage" }); const f = STAGES[r.kind].needs(input, r); if (f.state === "blocked") throw new StageFailure(f.reason, 3); if (f.state === "fresh") { const stampOf = r.kind.startsWith("deploy-") || r.target === ALL_TARGET ? (input.index.stamp?.stampId ?? "") : ""; const built = r.kind === "build-site" || r.kind === "deploy-site" ? input.sites[r.target]?.built : r.kind.endsWith("-hub") ? input.hub.built : input.homepage.built; if (r.kind.startsWith("build-")) { if (r.target === ALL_TARGET) { for (const t of Object.values(input.sites)) await markChecked(paths, t.built); } else { await markChecked(paths, built); } } return { status: "noop", stamp: built?.stampId ?? stampOf, summary: `${r.target}: fresh — nothing to do (--force runs it anyway)`, }; } ctx.onLog(`[publish] ${STAGES[r.kind].label} ${r.target}: ${f.reason}\n`); const stamp = input.index.stamp; switch (r.kind) { case "build-site": if (r.target === ALL_TARGET) { return r.runner === "docker" ? buildAllDocker(ctx, r, input) : buildAllLocal(ctx, r, input); } return buildOneSite(ctx, r, stamp!); case "build-hub": return buildHubStage(ctx, stamp!); case "build-homepage": return buildHomepageStage(ctx, stamp!); case "deploy-site": case "deploy-hub": case "deploy-homepage": { const { runDeployStage } = await import("./deployStage"); return runDeployStage( { paths, onLog: ctx.onLog, signal: ctx.signal }, { kind: r.kind, target: r.target, preview: r.preview, to: r.to, force: r.force }, ); } } }