commit b6729c15dee46b093947d057ad56fcbf91f6beb4
parent a15670934874afb3c93201053f6e9ef01fcbac94
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 6 Oct 2026 08:34:03 -0400
publish: per-target bundles — build into export/out, install as <exportBuildsDir>/<id>/out, link export/out to it
installBundle moves the build to <target>/out.next and swaps it in (out.prev
removed, a leftover out.next deleted first; EXDEV copies). export/out becomes
a relative symlink to the bundle built last. stageSiteArchives moves the host
compose's R2 staging to dockerSiteStagingDir, so both runners stage alike.
buildSiteBundle / buildHubBundle are the stage bodies' builds; deploySite and
deployHub take the bundle to ship. ensureBuildImage, runDockerBuildOne,
runHostScript and runWithConcurrency are exported for the docker fan-out
stage; a build container gets its own EXPORT_BUILDS_DIR for its lock.
runChildIntoLog can kill a child's whole process group (off by default; a
stage child turns it on).
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
2 files changed, 280 insertions(+), 12 deletions(-)
diff --git a/common/jobs/runChild.ts b/common/jobs/runChild.ts
@@ -10,6 +10,24 @@ export type RunChildOpts = {
label?: string;
};
+// Release 18: a publish STAGE CHILD (and `archilyzer publish …`) is a process of
+// its own whose children run whole trees — `pnpm exec next build` and its
+// workers, wrangler, docker. Killing only the direct child leaves the rest
+// running after a Cancel. With tree-kill on, each child is spawned as the
+// leader of its own process group, and a cancel signals the whole GROUP. Off
+// by default, so the editor's own in-process jobs are unchanged; a stage child
+// turns it on once at start. Per process, on globalThis like the registry.
+const TREE_KILL_KEY = Symbol.for("archilyzer.runChild.treeKill");
+type TreeKillGlobal = { [TREE_KILL_KEY]?: boolean };
+
+export function setKillChildTrees(on: boolean): void {
+ (globalThis as TreeKillGlobal)[TREE_KILL_KEY] = on;
+}
+
+export function killsChildTrees(): boolean {
+ return (globalThis as TreeKillGlobal)[TREE_KILL_KEY] === true;
+}
+
// Spawn a child process and stream its combined stdout/stderr into `onLog`,
// resolving with the exit code. Used inside a managed function's `fn` to run a
// sub-command as part of a larger job (build-then-deploy, multi-phase builds)
@@ -25,13 +43,27 @@ export async function runChildIntoLog(
opts: RunChildOpts,
): Promise<number> {
const prefix = opts.label ?? "";
+ const tree = killsChildTrees();
const child: ResultPromise = execa(opts.command, opts.args, {
cwd: opts.cwd,
env: opts.env,
all: true,
buffer: false,
reject: false,
+ ...(tree ? { detached: true } : {}),
});
+ // Signal the child — or, with tree-kill, its whole process group.
+ const signalChild = (sig: NodeJS.Signals) => {
+ if (tree && child.pid) {
+ try {
+ process.kill(-child.pid, sig);
+ return;
+ } catch {
+ // The group is gone (or was never made): the child alone.
+ }
+ }
+ child.kill(sig);
+ };
// Line-buffer: execa chunks aren't line-aligned, and the managed-function
// onLog appends a newline per call, so emitting raw chunks would inject
@@ -55,9 +87,10 @@ export async function runChildIntoLog(
let killTimer: ReturnType<typeof setTimeout> | null = null;
const onAbort = () => {
- child.kill("SIGTERM");
+ signalChild("SIGTERM");
killTimer = setTimeout(() => {
- if (child.killed === false) child.kill("SIGKILL");
+ if (tree) signalChild("SIGKILL");
+ else if (child.killed === false) child.kill("SIGKILL");
}, 5_000);
killTimer.unref?.();
};
@@ -73,6 +106,8 @@ export async function runChildIntoLog(
buf = "";
}
if (killTimer) clearTimeout(killTimer);
+ // The leader is gone; whatever of its group a cancel left is not wanted.
+ if (tree && signal.aborted) signalChild("SIGKILL");
signal.removeEventListener("abort", onAbort);
}
}
diff --git a/common/publish/build.ts b/common/publish/build.ts
@@ -9,7 +9,7 @@
// forbids non-serializable args). Keep them as plain helpers.
import path from "node:path";
-import { mkdir, readdir, rm, stat } from "node:fs/promises";
+import { cp, lstat, mkdir, readdir, readFile, rename, rm, stat, symlink } from "node:fs/promises";
import { createReadStream, existsSync } from "node:fs";
import { S3Client, HeadObjectCommand } from "@aws-sdk/client-s3";
import { Upload } from "@aws-sdk/lib-storage";
@@ -482,7 +482,7 @@ export async function dockerAvailable(signal: AbortSignal): Promise<boolean> {
// These are pool-wide: no SITE_ID, and no EXPORT_PUBLIC_DIR override so the
// shared index/staging land at their canonical export/.export-index location
// (exactly what the fan-out containers mount read-only).
-async function runHostScript(
+export async function runHostScript(
onLog: (line: string) => void,
signal: AbortSignal,
paths: Paths,
@@ -507,7 +507,7 @@ async function runHostScript(
// change re-runs only `COPY . .` onward, but a Dockerfile or lockfile change
// re-installs every dependency first, inside this job. `archilyzer doctor`
// warns ahead of that when the image is absent or older than the Dockerfile.
-async function ensureBuildImage(
+export async function ensureBuildImage(
onLog: (line: string) => void,
signal: AbortSignal,
paths: Paths,
@@ -529,7 +529,7 @@ async function ensureBuildImage(
// next/font/google, which fetches the site's fonts from Google at build time;
// `--network=none` would fail the build. Isolation still comes from the per-site
// output dir, the read-only shared mounts, and the non-root `-u` user.
-async function runDockerBuildOne(
+export async function runDockerBuildOne(
onLog: (line: string) => void,
signal: AbortSignal,
siteId: string,
@@ -557,6 +557,10 @@ async function runDockerBuildOne(
"-v", `${paths.exportIndexDir}:/data/export/.export-index:ro`,
"-v", `${siteDir}:/site`,
"-e", `SITE_ID=${siteId}`,
+ // The container's `build site` is `publish build --force` (release 18):
+ // its publish lock lives inside the container, never on the host's
+ // mount — the host's lock is held by the fan-out stage itself.
+ "-e", `EXPORT_BUILDS_DIR=${CONTAINER_BUILDS_DIR}`,
);
// Mount the host settings.json fresh (build config: archive storage, size caps)
// rather than relying on a possibly-stale copy — it is NOT baked into the image.
@@ -718,7 +722,7 @@ export async function runDockerDeployAllPhase(
// Bounded-concurrency map over a fixed work set, preserving input order in the
// results. No external dep; a fresh worker pulls the next index until exhausted.
-async function runWithConcurrency<T, R>(
+export async function runWithConcurrency<T, R>(
items: T[],
limit: number,
worker: (item: T) => Promise<R>,
@@ -797,7 +801,14 @@ export async function buildSite(
*/
export async function deploySite(
siteId: string,
- opts: PublishOpts & { previewBranch?: string } = {},
+ opts: PublishOpts & {
+ previewBranch?: string;
+ // The bundle to ship and where its oversize archives were staged. Default:
+ // export/out and the host staging dir (the editor's deploy action); the
+ // deploy-site stage passes the site's own bundle under exportBuildsDir.
+ outDir?: string;
+ stagingDir?: string;
+ } = {},
): Promise<void> {
const { paths, onLog, signal } = resolved(opts);
if (opts.previewBranch !== undefined) {
@@ -814,7 +825,7 @@ export async function deploySite(
`Site "${site.siteId}" has no Cloudflare Pages project configured.`,
);
}
- const outDir = resolveOutDir(site.siteId, paths);
+ const outDir = opts.outDir ?? resolveOutDir(site.siteId, paths);
const builtProblem = builtSiteProblem(outDir, site.siteId);
if (builtProblem) throw new Error(builtProblem);
// Before the R2 upload below: a bundle built private is never deployed.
@@ -831,7 +842,7 @@ export async function deploySite(
}
// Push oversize archives to R2 first, so the manifest URLs the Pages deploy
// publishes resolve immediately. No-op when R2 isn't configured.
- const uploadCode = await runArchiveUploadIntoLog(onLog, signal, site, paths);
+ const uploadCode = await runArchiveUploadIntoLog(onLog, signal, site, paths, opts.stagingDir);
if (signal.aborted) return;
if (uploadCode !== 0) {
throw new Error(`Archive R2 upload failed (exit ${uploadCode}).`);
@@ -1018,7 +1029,7 @@ async function runPagesDeployIntoLog(
* quietly on a cancel. No R2 step: the hub holds no archives.
*/
export async function deployHub(
- opts: PublishOpts & { previewBranch?: string } = {},
+ opts: PublishOpts & { previewBranch?: string; outDir?: string } = {},
): Promise<void> {
const { paths, onLog, signal } = resolved(opts);
if (opts.previewBranch !== undefined) {
@@ -1029,7 +1040,7 @@ export async function deployHub(
const project = getHomepageConfig(paths).cloudflareProject;
const projectProblem = hubProjectProblem(project);
if (projectProblem) throw new Error(projectProblem);
- const outDir = resolveOutDir("", paths);
+ const outDir = opts.outDir ?? resolveOutDir("", paths);
const builtProblem = builtHubProblem(outDir);
if (builtProblem) throw new Error(builtProblem);
if (branch) onLog(`=== Deploy hub (preview "${branch}") ===\n`);
@@ -1197,3 +1208,225 @@ export async function deployHomepage(
if (signal.aborted) return;
if (code !== 0) throw new Error(`Homepage deploy failed (exit ${code}).`);
}
+
+// ---------------------------------------------------------------------------
+// Per-target bundles (release 18). `<exportBuildsDir>/<target>/out` is THE
+// bundle every deploy of `target` ships, whichever runner built it: a site's
+// id, `_hub`, or (stamps only) `_homepage` — the homepage's bundle stays
+// homepage/out, where the source gate withdraws from.
+//
+// The local runner builds into the shared export/out (Next's `distDir` may not
+// leave the project, and `public/` is copied into `out/` — the plan's
+// "Decided"), then INSTALLS it: moved to `<target>/out.next`, swapped in
+// (`out → out.prev`, `out.next → out`, `out.prev` removed). A leftover
+// `out.next` is deleted first. On one filesystem the move is a rename; across
+// two (the container: export/out is an image layer, the builds dir a volume)
+// rename fails EXDEV and the bundle is copied, then the source removed.
+// export/out is then a SYMLINK to the bundle built last, so older readers and
+// `serve out` keep working; `next build` removes the link itself (an `rm` of
+// the path, recursive — never its target) before it writes a real out/.
+// ---------------------------------------------------------------------------
+
+// Inside the docker per-site build container: where `publish build` keeps its
+// lock (runDockerBuildOne passes it as EXPORT_BUILDS_DIR).
+export const CONTAINER_BUILDS_DIR = "/tmp/archilyzer-builds";
+
+/** `<exportBuildsDir>/<target>/out` — the target's bundle (= dockerSiteOutDir for a site). */
+export function bundleDir(paths: Pick<Paths, "exportBuildsDir">, target: string): string {
+ return path.join(paths.exportBuildsDir, target, "out");
+}
+
+/** export/out: where `next build` writes, and afterwards a link to the last bundle. */
+export function exportOutPath(paths: Pick<Paths, "exportDir">): string {
+ return path.join(paths.exportDir, "out");
+}
+
+// The filesystem calls a bundle move makes, injectable so the EXDEV path is
+// tested without two filesystems.
+export type BundleFs = {
+ rename: typeof rename;
+ cp: typeof cp;
+ rm: typeof rm;
+ mkdir: typeof mkdir;
+};
+const realBundleFs: BundleFs = { rename, cp, rm, mkdir };
+
+async function lexists(p: string): Promise<boolean> {
+ return lstat(p).then(
+ () => true,
+ () => false,
+ );
+}
+
+// Move a directory: rename, or across filesystems a copy then a remove.
+async function moveDir(src: string, dest: string, fsOps: BundleFs): Promise<"rename" | "copy"> {
+ try {
+ await fsOps.rename(src, dest);
+ return "rename";
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code !== "EXDEV") throw err;
+ await fsOps.cp(src, dest, { recursive: true, verbatimSymlinks: true, preserveTimestamps: true });
+ await fsOps.rm(src, { recursive: true, force: true });
+ return "copy";
+ }
+}
+
+/**
+ * Install the build at `src` as the bundle `destOut` (see the section header):
+ * `src` is gone afterwards and `destOut` holds it. Answers how it moved.
+ */
+export async function installBundle(
+ src: string,
+ destOut: string,
+ fsOps: BundleFs = realBundleFs,
+): Promise<"rename" | "copy"> {
+ const next = `${destOut}.next`;
+ const prev = `${destOut}.prev`;
+ await fsOps.mkdir(path.dirname(destOut), { recursive: true });
+ await fsOps.rm(next, { recursive: true, force: true });
+ const how = await moveDir(src, next, fsOps);
+ await fsOps.rm(prev, { recursive: true, force: true });
+ // Siblings: these two renames never cross a filesystem.
+ if (await lexists(destOut)) await fsOps.rename(destOut, prev);
+ await fsOps.rename(next, destOut);
+ await fsOps.rm(prev, { recursive: true, force: true });
+ return how;
+}
+
+/**
+ * Before a build: export/out must not still point at an older bundle, or a
+ * build that failed before `next build` reached it would leave that bundle
+ * readable at export/out. A link is removed; a real directory is left for
+ * `next build` to replace.
+ */
+export async function unlinkExportOut(paths: Pick<Paths, "exportDir">): Promise<void> {
+ const link = exportOutPath(paths);
+ const st = await lstat(link).catch(() => null);
+ if (st?.isSymbolicLink()) await rm(link, { force: true });
+}
+
+/** After a build: export/out becomes a (relative) link to `bundle`, replaced atomically. */
+export async function pointExportOutAt(paths: Pick<Paths, "exportDir">, bundle: string): Promise<void> {
+ const link = exportOutPath(paths);
+ const st = await lstat(link).catch(() => null);
+ if (st && !st.isSymbolicLink()) await rm(link, { recursive: true, force: true });
+ const tmp = `${link}.link-${process.pid}`;
+ await rm(tmp, { force: true });
+ await symlink(path.relative(path.dirname(link), bundle), tmp, "dir");
+ await rename(tmp, link);
+}
+
+/**
+ * The host compose stages a site's oversize archives beside export/public
+ * (`.r2-staging/<id>/archives`); the bundle's are `dockerSiteStagingDir` for
+ * both runners. Moves this build's (replacing the last build's) and answers
+ * how many were staged. A no-op where the two are one place.
+ */
+export async function stageSiteArchives(
+ paths: Pick<Paths, "exportPublicDir" | "exportBuildsDir">,
+ siteId: string,
+ fsOps: BundleFs = realBundleFs,
+): Promise<number> {
+ const src = path.join(path.dirname(paths.exportPublicDir), ".r2-staging", siteId);
+ const dest = path.join(paths.exportBuildsDir, siteId, ".r2-staging", siteId);
+ if (path.resolve(src) !== path.resolve(dest)) {
+ await fsOps.rm(dest, { recursive: true, force: true });
+ if (await lexists(src)) {
+ await fsOps.mkdir(path.dirname(dest), { recursive: true });
+ await moveDir(src, dest, fsOps);
+ }
+ }
+ const staged = await readdir(dockerSiteStagingDir(paths as Paths, siteId)).catch(() => [] as string[]);
+ return staged.filter((f) => !f.startsWith(".")).length;
+}
+
+/** Files and bytes under `dir` (links not followed). */
+export async function bundleCounts(dir: string): Promise<{ files: number; bytes: number }> {
+ let files = 0;
+ let bytes = 0;
+ const walk = async (d: string): Promise<void> => {
+ const ents = await readdir(d, { withFileTypes: true }).catch(() => []);
+ for (const e of ents) {
+ const p = path.join(d, e.name);
+ if (e.isDirectory()) await walk(p);
+ else if (e.isFile()) {
+ files++;
+ bytes += (await stat(p)).size;
+ }
+ }
+ };
+ await walk(dir);
+ return { files, bytes };
+}
+
+/** The bundle's corpus.json `generatedAt`, or null. */
+export async function corpusGeneratedAtIn(outDir: string): Promise<string | null> {
+ try {
+ const v = JSON.parse(await readFile(path.join(outDir, "corpus.json"), "utf8")) as { generatedAt?: unknown };
+ return typeof v.generatedAt === "string" ? v.generatedAt : null;
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * Build one site and install it as its bundle (the build-site stage's local
+ * body): export/out unlinked, buildSiteSteps with `skipData` (the index stage
+ * ran the data phase), the cited/scope checks of runBuildPhase,
+ * builtBundleProblem, then the install, the archive staging and the link.
+ * Returns the exit code; on 0 the bundle is `bundleDir(paths, siteId)`.
+ *
+ * `inPlace` (the docker per-site container) stops after the checks: the
+ * container hands export/out back itself, and the host stamps it.
+ */
+export async function buildSiteBundle(
+ siteId: string,
+ opts: PublishOpts & { skipArchives?: boolean; allowMissingMedia?: boolean; inPlace?: boolean } = {},
+): Promise<{ code: number; archivesStaged: number }> {
+ const { paths, onLog, signal } = resolved(opts);
+ if (!opts.inPlace) await unlinkExportOut(paths);
+ const code = await runBuildPhase(onLog, signal, siteId, paths, {
+ skipData: true,
+ skipArchives: opts.skipArchives,
+ allowMissingMedia: opts.allowMissingMedia,
+ });
+ if (code !== 0 || signal.aborted) return { code: code || 1, archivesStaged: 0 };
+ const out = exportOutPath(paths);
+ const problem = builtBundleProblem(out, siteId);
+ if (problem) {
+ onLog(`[build] REFUSED — ${problem}.\n`);
+ return { code: 1, archivesStaged: 0 };
+ }
+ if (opts.inPlace) return { code: 0, archivesStaged: 0 };
+ const how = await installBundle(out, bundleDir(paths, siteId));
+ const archivesStaged = await stageSiteArchives(paths, siteId);
+ await pointExportOutAt(paths, bundleDir(paths, siteId));
+ onLog(
+ `[build] ${siteId}: bundle installed at ${bundleDir(paths, siteId)} (${how === "rename" ? "moved" : "copied across filesystems"})` +
+ (archivesStaged ? `, ${archivesStaged} archive(s) staged for R2` : "") +
+ "\n",
+ );
+ return { code: 0, archivesStaged };
+}
+
+/**
+ * Build the hub and install it as `_hub/out` (the build-hub stage's body).
+ * Returns the exit code.
+ */
+export async function buildHubBundle(opts: PublishOpts = {}): Promise<number> {
+ const { paths, onLog, signal } = resolved(opts);
+ await unlinkExportOut(paths);
+ const code = await buildHub({ paths, onLog, signal });
+ if (code !== 0 || signal.aborted) return code || 1;
+ const out = exportOutPath(paths);
+ const problem = builtHubProblem(out);
+ if (problem) {
+ onLog(`[build] REFUSED — ${problem}.\n`);
+ return 1;
+ }
+ const dest = bundleDir(paths, "_hub");
+ const how = await installBundle(out, dest);
+ await pointExportOutAt(paths, dest);
+ onLog(`[build] hub: bundle installed at ${dest} (${how === "rename" ? "moved" : "copied across filesystems"})\n`);
+ return 0;
+}
