Archilyzer · Source

archilyzer

Archilyzer
git clone https://archilyzer.pages.dev/source/archilyzer.git
Log | Files | Refs | README | LICENSE

commit a335cdb3f1f210e376af5c03224a77ed55e494e1
parent 5c0f1bf848b802c7c754b1f775602203b2180660
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date:   Tue,  6 Oct 2026 12:36:09 -0400

publish: build.ts loses the pre-stage entry points (buildSite, deploySite, buildAll, deployHub, deployHomepage, the docker all-phases)

Nothing calls them since the surfaces and the CLI run stages: deleted with
what only they used (runDeployIntoLog, runPagesDeployIntoLog,
homepageDeployArgs, the two outcome types) and their tests. The refusals they
tested are the deploy stage's (deployStage.test.ts) and builtExport's /
source's own tests; comments that named them now name the stage.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Diffstat:
Mcommon/lib/builtExport.ts | 14+++++++-------
Mcommon/lib/homepage.ts | 2+-
Mcommon/lib/settingsSchema.ts | 2+-
Mcommon/publish/build.test.ts | 305+------------------------------------------------------------------------------
Mcommon/publish/build.ts | 537++++---------------------------------------------------------------------------
Mcommon/publish/source.ts | 4++--
6 files changed, 37 insertions(+), 827 deletions(-)

diff --git a/common/lib/builtExport.ts b/common/lib/builtExport.ts @@ -58,7 +58,7 @@ export function builtSiteIdIn(outDir: string): string | null { * larger surprise than a refusal naming the fix. * * It refuses exactly what builtBundleProblem refuses — the check the deploy - * itself makes before wrangler (publish/build.ts, runDeployIntoLog) — in the + * itself makes before wrangler (publish/deployStage.ts, runDeployStage) — in the * operator's words, so an action's fast answer and the deploy's last word * never disagree about a bundle. */ @@ -116,9 +116,9 @@ export function builtBundleProblem(outDir: string, siteId: string): string | nul * reading copy: it may carry what the public may not (X posts while * `social.x.visibility` is "private"), so no deploy path ships it. Every * deploy path asks this BEFORE ANY UPLOAD — the R2 archive push included — in - * the place it asks builtBundleProblem: runDeployIntoLog, the container deploy - * phase, deploySite, and the editor's Build & deploy, Deploy and Build & - * deploy all. A build without a deploy is untouched. + * the place it asks builtBundleProblem: the deploy stage (publish/deployStage.ts) + * and its request checks, which every deploy surface goes through. A build + * without a deploy is untouched. */ export function siteDeployProblem(site: { siteId: string; @@ -174,8 +174,8 @@ export function deployAudienceProblem( // may hold only what the list below names. Anything else (a stale summaries // tree, a transcripts shard, an archive, a service worker) refuses the build // and every deploy of it: builtSiteProblem and builtBundleProblem ask it, so -// runDeployIntoLog, the container deploy phase, deploySite, the editor's -// deploy actions and docker/build-site.sh all refuse what it refuses, and +// the deploy stage (publish/deployStage.ts), its request checks and +// docker/build-site.sh all refuse what it refuses, and // `archilyzer build site` fails on it (publish/build.ts runBuildPhase). // // The bundle names its own scope — corpus.json's `site.scope: "cited"`, which @@ -576,7 +576,7 @@ export function hubReportDataIn(outDir: string, limit = 5): string[] { * * Unlike export/out nothing else ever builds into homepage/out, so the only * question is whether a build is there at all. `index.html` is the file - * deployHomepage checks inside its job, so the refusal an operator gets before + * the deploy stage checks inside its job, so the refusal an operator gets before * the job and the one the job would give agree on what "built" means. */ export function builtHomepageProblem(outDir: string): string | null { diff --git a/common/lib/homepage.ts b/common/lib/homepage.ts @@ -31,7 +31,7 @@ export type HomepageConfig = { // Absolute public URL of the deployed hub, e.g. "https://archilyzer-hub.pages.dev". siteUrl?: string; // Cloudflare Pages project the hub deploys to (e.g. "archilyzer-hub"; never - // "archilyzer", the homepage's — deployHub refuses it). + // "archilyzer", the homepage's — the hub's deploy refuses it, hubProjectProblem). cloudflareProject?: string; // The transcript modal's per-video export controls (Download menu, Copy MD; // the yt-dlp clip command shows either way) on the hub's Browse and Ask pages — the same switch diff --git a/common/lib/settingsSchema.ts b/common/lib/settingsSchema.ts @@ -459,7 +459,7 @@ export const DIGEST_SETTINGS_FIELD_DOCS: FieldDocs<DigestSettings> = { }; // Build all / Build & deploy all (editor buildAction.ts, `archilyzer build all`) -// fan out in containers (publish/build.ts, runDockerBuildAllPhase) whenever +// fan out in containers (publish/stageBodies.ts, buildAllDocker) whenever // `docker version` answers, and build serially on the host otherwise. There is // no mode switch: the `mode` key ("basic" | "docker") was a label nothing read, // and it was dropped on 2026-09-28 (release 11, follow-up O6c). A settings.json diff --git a/common/publish/build.test.ts b/common/publish/build.test.ts @@ -5,30 +5,22 @@ import { existsSync, mkdirSync, mkdtempSync, - readFileSync, rmSync, writeFileSync, } from "node:fs"; import os from "node:os"; import path from "node:path"; -import { runChildIntoLog } from "../jobs/runChild"; import type { Paths } from "../lib/paths"; -import type { Site } from "../lib/site"; import { HOMEPAGE_PAGES_PROJECT, buildHomepage, buildHubSteps, buildSiteSteps, hubProjectProblem, - homepageDeployArgs, homepageOutDir, - deployHomepage, - deploySite, dockerSiteOutDir, dockerSiteStagingDir, resolveOutDir, - runDeployIntoLog, - runDockerDeployAllPhase, } from "./build"; // Run with: @@ -157,15 +149,8 @@ test("hubProjectProblem refuses a missing project and the homepage's", () => { assert.equal(hubProjectProblem("archilyzer-hub"), null); }); -test("homepageDeployArgs: production is branch main; a preview is its own branch and never main", () => { - const base = ["pages", "deploy", "/repo/homepage/out", "--project-name", "archilyzer"]; - assert.deepEqual(homepageDeployArgs("/repo/homepage/out"), [...base, "--branch", "main"]); - assert.deepEqual(homepageDeployArgs("/repo/homepage/out", " "), [...base, "--branch", "main"]); - assert.deepEqual(homepageDeployArgs("/repo/homepage/out", " r8-home "), [...base, "--branch", "r8-home"]); -}); - -// The editor's homepage actions judge "is there a build?" on this path before -// any job, so it must be the directory deployHomepage ships. +// The build-homepage stage writes and the deploy stage ships this path, and +// both judge "is there a build?" on it. test("homepageOutDir is homepage/out of the checkout", () => { assert.equal(homepageOutDir({ monorepoRoot: "/repo" } as Paths), "/repo/homepage/out"); }); @@ -245,289 +230,3 @@ test("buildHomepage: the source step sits between compose and next build; a refu rmSync(root, { recursive: true, force: true }); } }); - -// M1: a deploy-only ships homepage/out as the last build left it. Its source -// ships only with a record that it was audited under today's rules (the -// positive case is source.test.ts' round trip, over publishedSourceProblem). -test("deployHomepage refuses an out/ whose source has no record of the rules it was audited under", async () => { - const root = mkdtempSync(path.join(os.tmpdir(), "deploy-homepage-")); - try { - const out = path.join(root, "homepage", "out"); - mkdirSync(path.join(out, "source"), { recursive: true }); - writeFileSync(path.join(out, "index.html"), "<p>home</p>"); - writeFileSync(path.join(out, "source", "index.html"), "<p>the /source page</p>"); - writeFileSync( - path.join(out, "source", "manifest.json"), - JSON.stringify({ - version: 1, generatedAt: "2026-09-28T12:00:00.000Z", branch: "main", sourceCommit: "1".repeat(40), - mirrorHead: "2".repeat(40), subject: "s", files: 1, bytes: 1, mirror: { files: 1, bytes: 1, packs: 1 }, - tree: { files: 1, dirs: 1, bytes: 1 }, tarball: { href: "/downloads/archilyzer-source.tar.gz", bytes: 1, sha256: "3".repeat(64) }, - audit: { objects: 1, commits: 1, gitleaks: "clean" }, tools: {}, - }), - ); - const p = { monorepoRoot: root } as Paths; - await assert.rejects( - deployHomepage({ paths: p, previewBranch: "r12-source" }), - /homepage\/out's source has no record of the rules it was audited under — run `archilyzer build homepage`/, - ); - // A half-removed source (no manifest) refuses too… - rmSync(path.join(out, "source", "manifest.json")); - mkdirSync(path.join(out, "source", "archilyzer.git")); - await assert.rejects(deployHomepage({ paths: p }), /without a valid manifest — run `archilyzer build homepage`/); - // …and so does a build whose source step refused (buildHomepage took - // out/source away, the page with it). - rmSync(path.join(out, "source"), { recursive: true }); - await assert.rejects(deployHomepage({ paths: p }), /has no \/source page \(its source step refused/); - } finally { - rmSync(root, { recursive: true, force: true }); - } -}); - -test("deployHomepage refuses a bad preview branch before it looks for a build", async () => { - const noBuild = { monorepoRoot: "/nonexistent-repo" } as Paths; - await assert.rejects( - deployHomepage({ paths: noBuild, previewBranch: "main" }), - /"main" is the production branch/, - ); - await assert.rejects( - deployHomepage({ paths: noBuild, previewBranch: "Not_Valid" }), - /not a valid preview branch name/, - ); - await assert.rejects( - deployHomepage({ paths: noBuild, previewBranch: "r8-home" }), - /homepage\/out holds no build/, - ); -}); - -// Phase C ships each per-site out/ a container wrote. It must be THAT site's -// bundle: a container once published the public/ baked into its image, so an -// out/ could carry another site's data. The check comes first. These sites have -// NO Cloudflare project, so were it ever removed they would be "skipped", and -// this test could never reach a real upload or deploy. -test("runDockerDeployAllPhase refuses a per-site out/ that is not the site's own bundle", async () => { - const root = mkdtempSync(path.join(os.tmpdir(), "deploy-all-")); - try { - const outFor = (id: string) => path.join(root, id, "out"); - mkdirSync(outFor("anilyzer"), { recursive: true }); - writeFileSync(path.join(outFor("anilyzer"), "site.json"), JSON.stringify({ siteId: "jeralyzer" })); - writeFileSync(path.join(outFor("anilyzer"), "corpus.json"), JSON.stringify({ site: { id: "jeralyzer" } })); - mkdirSync(outFor("bonnellyzer"), { recursive: true }); // built, but no bundle in it - const log: string[] = []; - const sites = [ - { siteId: "anilyzer" }, - { siteId: "bonnellyzer" }, - ] as Site[]; - const outcomes = await runDockerDeployAllPhase( - (l) => log.push(l), - new AbortController().signal, - sites, - new Set(["anilyzer", "bonnellyzer"]), - { ...paths, exportBuildsDir: root } as Paths, - outFor, - ); - assert.deepEqual(outcomes.map((o) => [o.siteId, o.status]), [["anilyzer", "failed"], ["bonnellyzer", "failed"]]); - assert.match(outcomes[0].reason!, /holds a build of "jeralyzer", not "anilyzer"/); - assert.match(outcomes[1].reason!, /has no site\.json naming a site/); - assert.ok(log.some((l) => l.startsWith("[anilyzer] deploy REFUSED — ")), log.join("\n")); - assert.ok(!log.some((l) => l.startsWith("=== Deploy")), "nothing reached the deploy"); - } finally { - rmSync(root, { recursive: true, force: true }); - } -}); - -function writeBundle(dir: string, siteId: string | null, corpusId: string | null): void { - mkdirSync(dir, { recursive: true }); - if (siteId !== null) writeFileSync(path.join(dir, "site.json"), JSON.stringify({ siteId })); - if (corpusId !== null) writeFileSync(path.join(dir, "corpus.json"), JSON.stringify({ site: { id: corpusId } })); - writeFileSync(path.join(dir, "index.html"), "<!doctype html>"); -} - -// runDeployIntoLog is the one door every SITE deploy goes through — the -// container Phase C, deploySite, the Publish tab's Build & deploy, the host -// Build & deploy all — and the build queue can rewrite export/out between a -// site's build and its deploy. So it refuses a bundle that is not the site's -// own right before wrangler. PATH here holds only a fake `pnpm` that records -// its argv: with the check or without it, no run of this test can reach a real -// wrangler, and the fake is proved to answer before anything is deployed. -test("runDeployIntoLog refuses a bundle that is not the site's own before wrangler, and ships the site's own", async () => { - const root = mkdtempSync(path.join(os.tmpdir(), "deploy-guard-")); - const bin = path.join(root, "bin"); - const argvFile = path.join(root, "pnpm-argv"); - mkdirSync(bin); - writeFileSync( - path.join(bin, "pnpm"), - `#!/bin/sh\nprintf '%s\\n' "$@" >> '${argvFile}'\necho "Take a peek over at https://abc123.w3c-never-real.pages.dev"\n`, - ); - chmodSync(path.join(bin, "pnpm"), 0o755); - const savedPath = process.env.PATH; - process.env.PATH = bin; - // The deploy spawns wranglerBin(paths): WRANGLER_BIN, here the same fake. - const savedWrangler = process.env.WRANGLER_BIN; - process.env.WRANGLER_BIN = path.join(bin, "pnpm"); - const signal = new AbortController().signal; - const site = { siteId: "anilyzer", cloudflareProject: "w3c-never-real" } as Site; - const testPaths = { ...paths, exportDir: root } as Paths; - try { - // The fake answers the same spawn the deploy makes, or nothing below runs. - await runChildIntoLog(() => {}, signal, { command: "pnpm", args: ["--fake?"], cwd: root, env: { ...process.env } }); - assert.equal(readFileSync(argvFile, "utf8"), "--fake?\n"); - rmSync(argvFile); - - const refusals: [string, string | null, string | null, RegExp][] = [ - ["another site's bundle", "jeralyzer", "jeralyzer", /holds a build of "jeralyzer", not "anilyzer" \(site\.json\)/], - ["the hub's bundle (no site.json)", null, null, /has no site\.json naming a site/], - ["a torn bundle", "anilyzer", "jeralyzer", /describes "jeralyzer", not "anilyzer" \(corpus\.json\)/], - ["a bundle with no corpus.json", "anilyzer", null, /has no corpus\.json naming a site/], - ]; - for (const [name, siteId, corpusId, why] of refusals) { - const out = path.join(root, name.replace(/\W+/g, "-"), "out"); - writeBundle(out, siteId, corpusId); - const log: string[] = []; - const code = await runDeployIntoLog((l) => log.push(l), signal, site, out, testPaths); - assert.equal(code, 1, name); - assert.equal(log.length, 1, `${name}: ${log.join("")}`); - assert.match(log[0], /^\[deploy\] REFUSED — /, name); - assert.match(log[0], why, name); - assert.match(log[0], /Nothing was sent to Cloudflare Pages; build anilyzer again, then deploy\.\n$/, name); - assert.equal(existsSync(argvFile), false, `${name}: pnpm was spawned`); - } - - // A legitimately built site is never refused: the same argv as ever. - const good = path.join(root, "good", "out"); - writeBundle(good, "anilyzer", "anilyzer"); - const log: string[] = []; - assert.equal(await runDeployIntoLog((l) => log.push(l), signal, site, good, testPaths), 0, log.join("\n")); - assert.equal( - readFileSync(argvFile, "utf8"), - ["pages", "deploy", good, "--project-name", "w3c-never-real", "--branch", "main", ""].join("\n"), - ); - assert.ok(log.includes("[deployed] https://abc123.w3c-never-real.pages.dev\n"), log.join("\n")); - } finally { - process.env.PATH = savedPath; - if (savedWrangler === undefined) delete process.env.WRANGLER_BIN; - else process.env.WRANGLER_BIN = savedWrangler; - rmSync(root, { recursive: true, force: true }); - } -}); - -// A PRIVATE site (site.json `audience: "private"`, release 17 slice XP) is never -// deployed, and neither is a bundle built private (its corpus.json says so): -// refused at the same door as the wrong-site bundle, before wrangler, in words -// naming the audience. The fake `pnpm` is the test above's. -function writePrivateBundle(dir: string, siteId: string): void { - writeBundle(dir, siteId, siteId); - writeFileSync( - path.join(dir, "corpus.json"), - JSON.stringify({ site: { id: siteId, audience: "private" } }), - ); -} - -test("runDeployIntoLog refuses a private site and a private build before wrangler", async () => { - const root = mkdtempSync(path.join(os.tmpdir(), "deploy-private-")); - const bin = path.join(root, "bin"); - const argvFile = path.join(root, "pnpm-argv"); - mkdirSync(bin); - writeFileSync(path.join(bin, "pnpm"), `#!/bin/sh\nprintf '%s\\n' "$@" >> '${argvFile}'\n`); - chmodSync(path.join(bin, "pnpm"), 0o755); - const savedPath = process.env.PATH; - process.env.PATH = bin; - // The deploy spawns wranglerBin(paths): WRANGLER_BIN, here the same fake. - const savedWrangler = process.env.WRANGLER_BIN; - process.env.WRANGLER_BIN = path.join(bin, "pnpm"); - const signal = new AbortController().signal; - const testPaths = { ...paths, exportDir: root } as Paths; - try { - await runChildIntoLog(() => {}, signal, { command: "pnpm", args: ["--fake?"], cwd: root, env: { ...process.env } }); - assert.equal(readFileSync(argvFile, "utf8"), "--fake?\n"); - rmSync(argvFile); - - // The site is private: its own, well-formed bundle is still refused. - const own = path.join(root, "own", "out"); - writeBundle(own, "mine", "mine"); - const priv = { siteId: "mine", cloudflareProject: "w3c-never-real", audience: "private" } as Site; - let log: string[] = []; - assert.equal(await runDeployIntoLog((l) => log.push(l), signal, priv, own, testPaths), 1); - assert.equal(log.length, 1, log.join("")); - assert.match( - log[0], - /^\[deploy\] REFUSED — Site "mine" is private \(audience: private\): it is built for reading on this machine and is never deployed\./, - ); - assert.match(log[0], /Nothing was sent to Cloudflare Pages\.\n$/); - - // The site is public now, but the bundle was built private. - const built = path.join(root, "built", "out"); - writePrivateBundle(built, "mine"); - log = []; - const pub = { siteId: "mine", cloudflareProject: "w3c-never-real" } as Site; - assert.equal(await runDeployIntoLog((l) => log.push(l), signal, pub, built, testPaths), 1); - assert.match(log[0], /holds a private build of "mine" \(its corpus\.json says "audience": "private"\)/); - assert.equal(existsSync(argvFile), false, "pnpm was spawned"); - } finally { - process.env.PATH = savedPath; - if (savedWrangler === undefined) delete process.env.WRANGLER_BIN; - else process.env.WRANGLER_BIN = savedWrangler; - rmSync(root, { recursive: true, force: true }); - } -}); - -test("runDockerDeployAllPhase skips a private site before the upload, in the audience's words", async () => { - const root = mkdtempSync(path.join(os.tmpdir(), "deploy-all-private-")); - try { - const outFor = (id: string) => path.join(root, id, "out"); - writeBundle(outFor("mine"), "mine", "mine"); - writePrivateBundle(outFor("built"), "built"); - const log: string[] = []; - // A Cloudflare project on each: without the audience check the run would - // reach the upload, which the log would show. - const sites = [ - { siteId: "mine", audience: "private", cloudflareProject: "w3c-never-real" }, - { siteId: "built", cloudflareProject: "w3c-never-real" }, - ] as Site[]; - const outcomes = await runDockerDeployAllPhase( - (l) => log.push(l), - new AbortController().signal, - sites, - new Set(["mine", "built"]), - { ...paths, exportBuildsDir: root } as Paths, - outFor, - ); - assert.deepEqual(outcomes.map((o) => [o.siteId, o.status]), [["mine", "skipped"], ["built", "skipped"]]); - assert.match(outcomes[0].reason!, /^Site "mine" is private \(audience: private\)/); - assert.match(outcomes[1].reason!, /private build of "built"/); - assert.ok(log.some((l) => l.startsWith("[mine] deploy skipped — Site \"mine\" is private")), log.join("\n")); - assert.ok(!log.some((l) => l.startsWith("=== Deploy")), "nothing reached the deploy"); - } finally { - rmSync(root, { recursive: true, force: true }); - } -}); - -test("deploySite (archilyzer deploy site) refuses a private site before anything, and a private build before the upload", async () => { - const root = mkdtempSync(path.join(os.tmpdir(), "deploy-site-private-")); - try { - const sitesDir = path.join(root, "sites"); - const site = (id: string, extra: Record<string, unknown> = {}) => { - mkdirSync(path.join(sitesDir, id), { recursive: true }); - writeFileSync( - path.join(sitesDir, id, "site.json"), - JSON.stringify({ siteId: id, cloudflareProject: "w3c-never-real", ...extra }), - ); - }; - site("mine", { audience: "private" }); - site("other"); - const testPaths = { ...paths, exportDir: root, sitesDir } as Paths; - const log: string[] = []; - await assert.rejects( - deploySite("mine", { paths: testPaths, onLog: (l) => log.push(l) }), - /^Error: Site "mine" is private \(audience: private\): it is built for reading on this machine and is never deployed\. Build it without deploying, or set its audience to public on its Settings tab\.$/, - ); - // Public, but export/out holds a private build of it. - writePrivateBundle(path.join(root, "out"), "other"); - await assert.rejects( - deploySite("other", { paths: testPaths, onLog: (l) => log.push(l) }), - /private build of "other".*Build other again, then deploy\.$/, - ); - assert.deepEqual(log, [], "nothing was logged: no upload, no deploy"); - } finally { - rmSync(root, { recursive: true, force: true }); - } -}); diff --git a/common/publish/build.ts b/common/publish/build.ts @@ -1,8 +1,7 @@ -// The publish layer's build/deploy primitives: one site's host build, the -// docker per-site fan-out, the R2 archive upload and the Pages deploy. Moved -// here from the editor (`editor/app/sites/lib/buildDeployCore.ts`) in one-core -// Phase 4 slice 1, unchanged; the editor's build/deploy server actions -// (`editor/app/sites/lib/{buildAction,deployAction}.ts`) call them as jobs. +// The publish layer's build primitives: one site's host build, the docker +// per-site container, the R2 archive upload, the hub and homepage builds and the +// per-target bundles. The publish stages (stageBodies.ts, deployStage.ts) are +// their callers; the deploy itself is the deploy stage's (deployStage.ts). // // These take an `onLog` callback and an AbortSignal, so they CANNOT live in a // "use server" module (every export there becomes a server action, which @@ -15,26 +14,14 @@ import { S3Client, HeadObjectCommand } from "@aws-sdk/client-s3"; import { Upload } from "@aws-sdk/lib-storage"; import { runChildIntoLog } from "../jobs/runChild"; import { - builtAudienceProblem, builtBundleProblem, builtHubProblem, builtScopeProblem, - builtSiteProblem, citedBuildProblem, - deployAudienceProblem, - siteDeployProblem, } from "../lib/builtExport"; -import { getHomepageConfig } from "../lib/homepage"; -import { - deploymentUrlIn, - pagesDeployArgs, - previewAliasUrl, - previewBranchProblem, - wranglerBin, -} from "../lib/pagesDeploy"; import { getPaths, type Paths } from "../lib/paths"; import { getSettings } from "../lib/settings"; -import { getSite, listSites, type Site } from "../lib/site"; +import { getSite, type Site } from "../lib/site"; import { builtStampPath } from "./stamps"; // Where the basic (host) build writes the static bundle to deploy: the fixed @@ -133,10 +120,8 @@ async function runSteps( // Run the basic (host) build phase for one site, streaming into `onLog`, // returning the exit code: buildSiteSteps, in export/ (serialized upstream on -// the build queue, since the export/ tree is shared). This is the single-site -// build for basic mode, and the fallback the all-sites docker action drops to -// when no container engine is available. The parallel per-site container path -// lives in runDockerBuildAllPhase. +// the build queue, since the export/ tree is shared). buildSiteBundle runs it, +// for the local runner and inside the docker per-site container alike. export async function runBuildPhase( onLog: (line: string) => void, signal: AbortSignal, @@ -238,8 +223,8 @@ export async function runArchiveUploadIntoLog( site: Site, paths: Paths, // Where the oversize archives were staged. Defaults to the basic host location; - // the docker deploy phase passes dockerSiteStagingDir since its container wrote - // .r2-staging under exportBuildsDir/<siteId> instead. + // the deploy stage passes dockerSiteStagingDir, where every runner's bundle + // keeps them (stageSiteArchives). stagingDirOverride?: string, ): Promise<number> { const bucket = getSettings().archiveStorage?.bucket?.trim(); @@ -340,108 +325,14 @@ export async function runArchiveUploadIntoLog( } } -// Deploy a previously-built static bundle (`outDir`) to the site's Cloudflare -// Pages project, streaming into `onLog`, returning the exit code. Runs on the -// host with the host's Cloudflare credentials (process.env) — deploy never runs -// inside a container, so container wrangler auth is never needed. -// -// `opts.previewBranch` makes it a PREVIEW deploy: Cloudflare treats a deploy to -// any branch but the project's production branch as a preview, reachable at the -// branch alias. Omitting it deploys production, `--branch main` (release 18: -// the branch is named, never inferred from the checkout, where a "production" -// deploy from a non-main checkout used to become a preview silently). The -// binary is the pinned wrangler (wranglerBin), not `pnpm dlx`. -// -// Either way, the URL wrangler prints ("Take a peek over at …") earns one -// terminal line of its own, because the streamed log scrolls and an operator -// who looked away has nowhere else to find it. A preview also gets the stable -// branch alias, which is knowable without reading the log at all. -export async function runDeployIntoLog( - onLog: (line: string) => void, - signal: AbortSignal, - site: Site, - outDir: string, - paths: Paths, - opts?: { previewBranch?: string }, -): Promise<number> { - // The last word before wrangler: the bundle must be this site's own - // (site.json AND corpus.json name it — builtBundleProblem). Every SITE deploy - // comes through here — the container Phase C, deploySite, the Publish tab's - // Build & deploy and the host Build & deploy all — and the deploy queue runs - // beside the build queue, so between a site's build and this line another - // job (a build of another site, the hub) can rewrite export/out. Nothing runs - // between this check and the spawn. The hub and the homepage deploy through - // runPagesDeployIntoLog and never come here. - // - // A PRIVATE site, or a bundle built private, is refused first: it is never - // deployed, whatever the bundle's identity (deployAudienceProblem). - const audienceProblem = deployAudienceProblem(site, outDir); - if (audienceProblem) { - onLog( - `[deploy] REFUSED — ${audienceProblem}. Nothing was sent to Cloudflare Pages.\n`, - ); - return 1; - } - const bundleProblem = builtBundleProblem(outDir, site.siteId); - if (bundleProblem) { - onLog( - `[deploy] REFUSED — ${bundleProblem}. Nothing was sent to Cloudflare Pages; ` + - `build ${site.siteId} again, then deploy.\n`, - ); - return 1; - } - const project = site.cloudflareProject as string; - const previewBranch = opts?.previewBranch?.trim() || undefined; - - // Spot the deployment URL as it streams past rather than re-reading the - // finished log file: the log is the operator's too, and buffering it a second - // time to grep it would double a big deploy's memory for one line of output. - let deploymentUrl: string | null = null; - const watch = (line: string) => { - if (deploymentUrl === null) deploymentUrl = deploymentUrlIn(line, project); - onLog(line); - }; - - const code = await runChildIntoLog(watch, signal, { - command: wranglerBin(paths), - args: pagesDeployArgs({ outDir, project, previewBranch }), - cwd: paths.exportDir, - env: { - ...process.env, - NODE_ENV: "production", - TRANSCRIPTS_DIR: paths.transcriptsDir, - EXPORT_PUBLIC_DIR: paths.exportPublicDir, - SITE_ID: site.siteId, - }, - }); - - // Only on success. A URL scraped out of a failed run points at nothing — or - // worse, at the deployment that is still live. - if (code === 0) { - if (previewBranch) { - const alias = previewAliasUrl(project, previewBranch); - onLog( - `[preview] ${alias}` + - (deploymentUrl ? ` (this deployment: ${deploymentUrl})` : "") + - "\n", - ); - } else if (deploymentUrl) { - onLog(`[deployed] ${deploymentUrl}\n`); - } - } - return code; -} - // --------------------------------------------------------------------------- -// Docker export pipeline (Build all, whenever a container engine answers) -// -// Three ordered phases (see PUBLISH.md, "Building every site in containers"): -// A) HOST, serial: build:data (shared LMDB + .export-index) then build:archives -// (warm the shared archive cache). One writer of the shared state. -// B) CONTAINERS, parallel (cap maxParallelBuilds): each site's compose + next -// build in its own container, read-only over the shared caches, writing only -// its per-site out/ under exportBuildsDir/<siteId>. -// C) HOST, serial: deploy each built site (handled by runDockerDeployAllPhase). +// Docker export pipeline: the parts of the build-all stage's docker runner +// (stageBodies.ts, buildAllDocker — see PUBLISH.md, "Building every site in +// containers"): the host's build:archives warm (runHostScript), the image +// (ensureBuildImage), then each site's compose + next build in its own +// container (runDockerBuildOne), up to maxParallelBuilds at once +// (runWithConcurrency), read-only over the shared caches and writing only its +// per-site out/ under exportBuildsDir/<siteId>. // --------------------------------------------------------------------------- // The container engine binary. Defaults to `docker`; podman is a CLI drop-in @@ -461,13 +352,6 @@ export function buildImageArgs(pipeline: { return ["build", "-f", pipeline.dockerfile, "-t", pipeline.dockerImage, "."]; } -export type SiteBuildOutcome = { siteId: string; code: number }; -export type SiteDeployOutcome = { - siteId: string; - status: "deployed" | "skipped" | "failed"; - reason?: string; -}; - // Cheap probe: is the container engine installed and its daemon reachable? Used // to fall back to serial host builds when docker isn't available. export async function dockerAvailable(signal: AbortSignal): Promise<boolean> { @@ -479,7 +363,7 @@ export async function dockerAvailable(signal: AbortSignal): Promise<boolean> { return code === 0; } -// Run a host-side export pnpm script (build:data / build:archives) for Phase A. +// Run a host-side export pnpm script (build:data / build:archives) before the fan-out. // These are pool-wide: no SITE_ID, and no EXPORT_PUBLIC_DIR override so the // shared index/staging land at their canonical export/.export-index location // (exactly what the fan-out containers mount read-only). @@ -504,7 +388,7 @@ export async function runHostScript( } // Build (or reuse cached layers of) the per-site build image. Runs before every -// Phase B, so a fan-out never meets an image older than the checkout: a code +// fan-out, so a fan-out never meets an image older than the checkout: a code // change re-runs only `COPY . .` onward, but a Dockerfile or lockfile change // re-installs every dependency first, inside this job. `archilyzer doctor` // warns ahead of that when the image is absent or older than the Dockerfile. @@ -588,139 +472,6 @@ export async function runDockerBuildOne( }); } -// Phases A + B. Returns each site's build exit code (0 = ok). Throws only on an -// infrastructure failure (data phase / archive warm / image build) that aborts -// the whole run before any site could build; per-site build failures are -// returned, not thrown, so one bad site never blocks the rest. -export async function runDockerBuildAllPhase( - onLog: (line: string) => void, - signal: AbortSignal, - sites: Site[], - paths: Paths, - opts?: { skipArchives?: boolean }, -): Promise<SiteBuildOutcome[]> { - const { maxParallelBuilds } = getSettings().buildPipeline; - - // --- Phase A: shared data + archive cache (host, serial) --- - onLog("=== Phase A: shared data + archive cache (host) ==="); - const dataCode = await runHostScript(onLog, signal, paths, "build:data"); - if (signal.aborted) return []; - if (dataCode !== 0) throw new Error(`Data phase failed (exit ${dataCode}).`); - if (!opts?.skipArchives) { - const archCode = await runHostScript(onLog, signal, paths, "build:archives"); - if (signal.aborted) return []; - if (archCode !== 0) throw new Error(`Archive cache warm failed (exit ${archCode}).`); - } - - const imgCode = await ensureBuildImage(onLog, signal, paths); - if (signal.aborted) return []; - if (imgCode !== 0) throw new Error(`Docker image build failed (exit ${imgCode}).`); - - // --- Phase B: per-site fan-out (containers, parallel) --- - onLog( - `=== Phase B: building ${sites.length} site(s), up to ${maxParallelBuilds} in parallel ===`, - ); - return runWithConcurrency(sites, maxParallelBuilds, async (site) => { - if (signal.aborted) return { siteId: site.siteId, code: 1 }; - const code = await runDockerBuildOne(onLog, signal, site.siteId, paths, opts); - onLog(`[${site.siteId}] build ${code === 0 ? "ok" : `FAILED (exit ${code})`}`); - return { siteId: site.siteId, code }; - }); -} - -// Phase C: deploy each built site SERIALLY on the host, after the build barrier. -// Partial-failure tolerant — a site that fails to upload/deploy is recorded and -// the loop continues. Sites that failed to build, or have no Cloudflare project, -// are skipped; a bundle that is not the site's own is refused (builtBundleProblem). -// `outDirFor` resolves each site's built bundle (docker: per-site; -// basic fallback: export/out). -export async function runDockerDeployAllPhase( - onLog: (line: string) => void, - signal: AbortSignal, - sites: Site[], - builtOk: Set<string>, - paths: Paths, - outDirFor: (siteId: string) => string, -): Promise<SiteDeployOutcome[]> { - const outcomes: SiteDeployOutcome[] = []; - for (const site of sites) { - if (signal.aborted) break; - if (!builtOk.has(site.siteId)) { - onLog(`[${site.siteId}] deploy skipped — build failed`); - outcomes.push({ siteId: site.siteId, status: "skipped", reason: "build failed" }); - continue; - } - // A private site (or a private build) is not deployed at all: skipped, in - // its own words, so a family with one private site does not fail every run. - // Asked first; a per-site dir holding another site's bundle built private - // therefore reads "skipped" rather than "REFUSED" — never shipped either way. - // - // Then the bundle must be this site's own — the check build-site.sh makes - // before it hands the bundle back, made again over whatever the per-site dir - // holds now — before anything past it (the R2 upload, the Pages deploy) is - // reached with another site's data. - const audienceProblem = deployAudienceProblem(site, outDirFor(site.siteId)); - if (audienceProblem) { - onLog(`[${site.siteId}] deploy skipped — ${audienceProblem}`); - outcomes.push({ siteId: site.siteId, status: "skipped", reason: audienceProblem }); - continue; - } - const bundleProblem = builtBundleProblem(outDirFor(site.siteId), site.siteId); - if (bundleProblem) { - onLog(`[${site.siteId}] deploy REFUSED — ${bundleProblem}`); - outcomes.push({ siteId: site.siteId, status: "failed", reason: bundleProblem }); - continue; - } - if (!site.cloudflareProject) { - onLog(`[${site.siteId}] deploy skipped — no Cloudflare project configured`); - outcomes.push({ - siteId: site.siteId, - status: "skipped", - reason: "no cloudflareProject", - }); - continue; - } - onLog(`=== Deploy ${site.siteId} ===`); - const uploadCode = await runArchiveUploadIntoLog( - onLog, - signal, - site, - paths, - dockerSiteStagingDir(paths, site.siteId), - ); - if (signal.aborted) break; - if (uploadCode !== 0) { - onLog(`[${site.siteId}] deploy FAILED — R2 upload exit ${uploadCode}`); - outcomes.push({ - siteId: site.siteId, - status: "failed", - reason: `R2 upload exit ${uploadCode}`, - }); - continue; - } - const deployCode = await runDeployIntoLog( - onLog, - signal, - site, - outDirFor(site.siteId), - paths, - ); - if (signal.aborted) break; - if (deployCode !== 0) { - onLog(`[${site.siteId}] deploy FAILED — exit ${deployCode}`); - outcomes.push({ - siteId: site.siteId, - status: "failed", - reason: `deploy exit ${deployCode}`, - }); - continue; - } - onLog(`[${site.siteId}] deployed.`); - outcomes.push({ siteId: site.siteId, status: "deployed" }); - } - return outcomes; -} - // Bounded-concurrency map over a fixed work set, preserving input order in the // results. No external dep; a fresh worker pulls the next index until exhausted. export async function runWithConcurrency<T, R>( @@ -743,11 +494,8 @@ export async function runWithConcurrency<T, R>( } // --------------------------------------------------------------------------- -// Named entry points (one-core Phase 4 slice 2): one call per thing an operator -// publishes, over the run* phases above. The editor's actions wrap these in -// runManagedFunction (a job, a queue, a log); the archilyzer CLI calls them -// straight, logging to the terminal. Every run* export stays — they are still -// the parts, and build.test.ts pins some of them. +// Named entry points: one call per thing the stages build, over the parts +// above. Their options default to the terminal and a never-aborted signal. // --------------------------------------------------------------------------- export type PublishOpts = { @@ -776,120 +524,6 @@ function resolved(opts: PublishOpts): { }; } -/** Build one site into export/out (basic/host mode). Returns the exit code. */ -export async function buildSite( - siteId: string, - opts: PublishOpts & { skipData?: boolean; skipArchives?: boolean; allowMissingMedia?: boolean } = {}, -): Promise<number> { - const { paths, onLog, signal } = resolved(opts); - return runBuildPhase(onLog, signal, siteId.trim(), paths, { - skipData: opts.skipData, - skipArchives: opts.skipArchives, - allowMissingMedia: opts.allowMissingMedia, - }); -} - -/** - * Deploy the site already built in export/out: its oversize archives to R2, - * then the bundle to its Pages project (a PREVIEW with `previewBranch`). - * THROWS on every refusal and failure, with the sentences the editor's deploy - * job has always ended on; returns quietly on a cancel. - * - * The refusals are checked here even though the editor action checks them - * before it starts the job, because a queued deploy can start after another - * site's build has replaced export/out — the action's check is the fast answer, - * this one is the last word. - */ -export async function deploySite( - siteId: string, - opts: PublishOpts & { - previewBranch?: string; - // The bundle to ship and where its oversize archives were staged. Default: - // export/out and the host staging dir (the editor's deploy action); the - // deploy-site stage passes the site's own bundle under exportBuildsDir. - outDir?: string; - stagingDir?: string; - } = {}, -): Promise<void> { - const { paths, onLog, signal } = resolved(opts); - if (opts.previewBranch !== undefined) { - const problem = previewBranchProblem(opts.previewBranch); - if (problem) throw new Error(problem); - } - const branch = opts.previewBranch?.trim() || undefined; - const site = getSite(siteId.trim(), paths); - // Before anything else is asked of a private site: it is never deployed. - const privateProblem = siteDeployProblem(site); - if (privateProblem) throw new Error(`${privateProblem}.`); - if (!site.cloudflareProject) { - throw new Error( - `Site "${site.siteId}" has no Cloudflare Pages project configured.`, - ); - } - const outDir = opts.outDir ?? resolveOutDir(site.siteId, paths); - const builtProblem = builtSiteProblem(outDir, site.siteId); - if (builtProblem) throw new Error(builtProblem); - // Before the R2 upload below: a bundle built private is never deployed. - const builtPrivate = builtAudienceProblem(outDir); - if (builtPrivate) throw new Error(`${builtPrivate}. Build ${site.siteId} again, then deploy.`); - // …nor a full build of a site that now publishes only its reports. - const builtScope = builtScopeProblem(site, outDir); - if (builtScope) throw new Error(`${builtScope}.`); - // The production path logs no banner and gains none here: its log has - // always opened on wrangler's own first line. - if (branch) { - onLog(`=== Deploy (preview "${branch}") ===\n`); - onLog(PREVIEW_SHARES_ARCHIVES_NOTICE); - } - // Push oversize archives to R2 first, so the manifest URLs the Pages deploy - // publishes resolve immediately. No-op when R2 isn't configured. - const uploadCode = await runArchiveUploadIntoLog(onLog, signal, site, paths, opts.stagingDir); - if (signal.aborted) return; - if (uploadCode !== 0) { - throw new Error(`Archive R2 upload failed (exit ${uploadCode}).`); - } - const code = await runDeployIntoLog(onLog, signal, site, outDir, paths, { - previewBranch: branch, - }); - if (signal.aborted) return; - if (code !== 0) throw new Error(`Deploy failed (exit ${code}).`); -} - -/** - * Build every configured site (or `sites`), without deploying. "docker" is the - * parallel per-site container fan-out (runDockerBuildAllPhase, which throws on - * an infrastructure failure); "basic" is a serial host loop whose shared - * export/out is overwritten per site, so only the last survives. The pool-wide - * data phase runs once either way. Per-site failures are returned, not thrown. - */ -export async function buildAll( - opts: PublishOpts & { - mode: "docker" | "basic"; - skipArchives?: boolean; - sites?: Site[]; - }, -): Promise<SiteBuildOutcome[]> { - const { paths, onLog, signal } = resolved(opts); - const sites = opts.sites ?? listSites(paths); - if (opts.mode === "docker") { - return runDockerBuildAllPhase(onLog, signal, sites, paths, { - skipArchives: opts.skipArchives, - }); - } - const outcomes: SiteBuildOutcome[] = []; - for (let i = 0; i < sites.length; i++) { - if (signal.aborted) break; - const site = sites[i]; - onLog(`\n=== Build ${site.siteId} (${i + 1}/${sites.length}) ===`); - const code = await runBuildPhase(onLog, signal, site.siteId, paths, { - skipData: i > 0, - skipArchives: opts.skipArchives, - }); - outcomes.push({ siteId: site.siteId, code }); - } - return outcomes; -} - // --------------------------------------------------------------------------- // The hub and the homepage — two apps, two Pages projects (decision // 2026-09-25). @@ -976,92 +610,14 @@ export async function buildHub(opts: PublishOpts = {}): Promise<number> { return runSteps(onLog, signal, buildHubSteps({ paths })); } -// One Pages deploy of `outDir` to `project`, streaming into onLog, with the -// deployment URL (or the preview alias) repeated as the last line on success -// — runDeployIntoLog's shape, for a bundle that is not a Site. -async function runPagesDeployIntoLog( - onLog: (line: string) => void, - signal: AbortSignal, - opts: { - outDir: string; - project: string; - cwd: string; - // The binary (wranglerBin): the pinned devDependency, or WRANGLER_BIN. - wrangler: string; - previewBranch?: string; - }, -): Promise<number> { - let deploymentUrl: string | null = null; - const watch = (line: string) => { - if (deploymentUrl === null) deploymentUrl = deploymentUrlIn(line, opts.project); - onLog(line); - }; - const code = await runChildIntoLog(watch, signal, { - command: opts.wrangler, - args: pagesDeployArgs({ - outDir: opts.outDir, - project: opts.project, - previewBranch: opts.previewBranch, - }), - cwd: opts.cwd, - env: { ...process.env, NODE_ENV: "production" }, - }); - if (code === 0) { - if (opts.previewBranch) { - const alias = previewAliasUrl(opts.project, opts.previewBranch); - onLog( - `[preview] ${alias}` + - (deploymentUrl ? ` (this deployment: ${deploymentUrl})` : "") + - "\n", - ); - } else if (deploymentUrl) { - onLog(`[deployed] ${deploymentUrl}\n`); - } - } - return code; -} - -/** - * Deploy the hub built in export/out to homepage.json's Pages project (a - * PREVIEW with `previewBranch`). THROWS every refusal and failure; returns - * quietly on a cancel. No R2 step: the hub holds no archives. - */ -export async function deployHub( - opts: PublishOpts & { previewBranch?: string; outDir?: string } = {}, -): Promise<void> { - const { paths, onLog, signal } = resolved(opts); - if (opts.previewBranch !== undefined) { - const problem = previewBranchProblem(opts.previewBranch); - if (problem) throw new Error(problem); - } - const branch = opts.previewBranch?.trim() || undefined; - const project = getHomepageConfig(paths).cloudflareProject; - const projectProblem = hubProjectProblem(project); - if (projectProblem) throw new Error(projectProblem); - const outDir = opts.outDir ?? resolveOutDir("", paths); - const builtProblem = builtHubProblem(outDir); - if (builtProblem) throw new Error(builtProblem); - if (branch) onLog(`=== Deploy hub (preview "${branch}") ===\n`); - const code = await runPagesDeployIntoLog(onLog, signal, { - outDir, - project: project!.trim(), - cwd: paths.exportDir, - wrangler: wranglerBin(paths), - previewBranch: branch, - }); - if (signal.aborted) return; - if (code !== 0) throw new Error(`Hub deploy failed (exit ${code}).`); -} - function homepageDir(paths: Paths): string { return path.join(paths.monorepoRoot, "homepage"); } /** - * Where buildHomepage writes and deployHomepage ships: `homepage/out` of this - * checkout. Exported for the editor's homepage actions, which refuse a - * deploy-only before any job when nothing is built there - * (builtHomepageProblem) and show when it was built. + * Where buildHomepage writes and the deploy-homepage stage ships: `homepage/out` of this + * checkout. Exported for the build-homepage and deploy stages, which judge + * whether anything is built there (builtHomepageProblem). */ export function homepageOutDir(paths: Paths): string { return path.join(homepageDir(paths), "out"); @@ -1096,7 +652,7 @@ export async function composeHomepage(opts: PublishOpts = {}): Promise<number> { * source twice over: the step itself removes the last publish from * homepage/public, and this removes the last BUILD's copy from homepage/out * (`out/source`, the tarball, `snapshot.json`), so a deploy-only cannot ship - * a source today's rules were never applied to (deployHomepage checks too). + * a source today's rules were never applied to (the deploy stage checks too). * The audit report is in the log. * * `skipSource` (the CLI's `--no-source`) REMOVES the published source instead: @@ -1155,51 +711,6 @@ async function withdrawBuiltSource(paths: Paths, onLog: (line: string) => void): } } -/** - * The wrangler argv (after the binary, wranglerBin) for a homepage deploy of - * `outDir`: the production branch `main`, or, with `previewBranch`, that - * preview branch and no `main` — pagesDeployArgs, which names the branch for - * every project now. Pure, so the test pins exactly what deployHomepage runs. - */ -export function homepageDeployArgs(outDir: string, previewBranch?: string): string[] { - return pagesDeployArgs({ outDir, project: HOMEPAGE_PAGES_PROJECT, previewBranch }); -} - -/** - * Deploy homepage/out to the homepage's constant project: production (branch - * `main`), or a PREVIEW with `previewBranch` — the same refusals and alias line - * as deployHub. THROWS on failure or when there is no build. - */ -export async function deployHomepage( - opts: PublishOpts & { previewBranch?: string } = {}, -): Promise<void> { - const { paths, onLog, signal } = resolved(opts); - if (opts.previewBranch !== undefined) { - const problem = previewBranchProblem(opts.previewBranch); - if (problem) throw new Error(problem); - } - const branch = opts.previewBranch?.trim() || undefined; - // The directory the editor's actions judged "built" (builtHomepageProblem). - const outDir = homepageOutDir(paths); - if (!existsSync(path.join(outDir, "index.html"))) { - throw new Error("homepage/out holds no build — run archilyzer build homepage first"); - } - // The source in out/ ships only if it was audited under TODAY's rules, of - // today's main (source.ts). An out/ with no source deploys as before. - const sourceProblem = await (await import("./source")).publishedSourceProblem(paths, outDir); - if (sourceProblem) throw new Error(sourceProblem); - if (branch) onLog(`=== Deploy homepage (preview "${branch}") ===\n`); - const code = await runPagesDeployIntoLog(onLog, signal, { - outDir, - project: HOMEPAGE_PAGES_PROJECT, - cwd: homepageDir(paths), - wrangler: wranglerBin(paths), - previewBranch: branch, - }); - if (signal.aborted) return; - if (code !== 0) throw new Error(`Homepage deploy failed (exit ${code}).`); -} - // --------------------------------------------------------------------------- // Per-target bundles (release 18). `<exportBuildsDir>/<target>/out` is THE // bundle every deploy of `target` ships, whichever runner built it: a site's diff --git a/common/publish/source.ts b/common/publish/source.ts @@ -453,7 +453,7 @@ export function sourcePublicDir(paths: Paths, override?: string): string { } // The skip key, kept BESIDE the public dir, never in it: it holds the rules -// hash, and public/ is deployed. It is also what `deployHomepage` checks +// hash, and public/ is deployed. It is also what the homepage's deploy stage checks // homepage/out's source against (publishedSourceProblem). function statePath(publicDir: string): string { return path.join(path.dirname(publicDir), ".source-publish.json"); @@ -1276,7 +1276,7 @@ export async function clearPublishedSource( /** * Why homepage/out's source may not be deployed, as one sentence — or null. - * `deployHomepage` asks before every deploy, production and preview alike: a + * The homepage's deploy stage asks before every deploy, production and preview alike: a * deploy-only ships `out/` as the last build left it, and that build's source * was audited under the rules of its day. It deploys only when the last * publish (the skip key beside public/) was made under TODAY's rules and step