commit 8d46c5578545b91b181d8d919a3e0b99cfc419a2
parent 938b4241bf3319976bf2fa43adfb936c161ed5f5
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 6 Oct 2026 09:51:20 -0400
Merge r18/stage-core (slice S1: the stage contract — stamps, the publish lock, per-target bundles, update-index/build/deploy stage bodies, the publish CLI rows and aliases)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
# Conflicts:
# editor/CHANGELOG.md
# plans/release-18.md
Diffstat:
24 files changed, 4803 insertions(+), 103 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -57,6 +57,9 @@ yarn-error.log*
# them, leaving generated data showing up as untracked in every worktree.
/export/public/subs
/export/public/posts
+# export/out is a SYMLINK to the bundle built last (release 18,
+# publish/build.ts pointExportOutAt) — `**/out/` matches only a directory.
+/export/out
/export/public/digests
/export/public/summaries
/export/public/transcripts
diff --git a/common/bin/_cli.test.ts b/common/bin/_cli.test.ts
@@ -404,3 +404,83 @@ test("every bin in common/bin is reachable as a subcommand", () => {
const paths = COMMANDS.map((c) => c.path.join(" "));
assert.equal(new Set(paths).size, paths.length);
});
+
+// ── publishing as stages (release 18 slice S1) ──────────────────────────────
+
+test("the stage row: kind and target, the flags the child's argv carries, nothing else", async () => {
+ const { STAGE_FLAGS, stageArgv } = await import("../publish/stages");
+ const hit = resolveCommand(COMMANDS, ["stage", "deploy-site", "jer"]);
+ assert.deepEqual(hit?.command.path, ["stage"]);
+ assert.deepEqual(hit?.rest, ["deploy-site", "jer"]);
+ assert.deepEqual(hit!.command.flags, { ...STAGE_FLAGS });
+ // The argv the editor spawns parses through runCli's own parser, cleanly.
+ const argv = stageArgv({
+ kind: "build-site",
+ target: "_all",
+ runId: "run-1",
+ runner: "docker",
+ force: true,
+ skipArchives: true,
+ indexAfter: 17,
+ });
+ assert.deepEqual(argv, [
+ "stage", "build-site", "_all", "--run-id", "run-1", "--runner", "docker", "--force", "--skip-archives", "--index-after", "17",
+ ]);
+ const parsed = parseArgv(argv, booleanFlags(COMMANDS));
+ assert.deepEqual(parsed.positionals, ["stage", "build-site", "_all"]);
+ assert.equal(argumentProblem(hit!.command, parsed.flags, ["build-site", "_all"]), null);
+ assert.match(argumentProblem(hit!.command, {}, ["a", "b", "c"])!, /unexpected argument "c"/);
+ assert.match(argumentProblem(hit!.command, { nodata: true }, [])!, /unknown flag --nodata/);
+});
+
+test("the publish rows and their flags (status and now are S3's)", () => {
+ const row = (...p: string[]) => resolveCommand(COMMANDS, p)?.command;
+ assert.deepEqual(row("publish", "index")?.flags ?? {}, {});
+ assert.deepEqual(row("publish", "build", "jer")?.flags, { runner: "string", force: "boolean", "skip-archives": "boolean" });
+ assert.equal(row("publish", "build", "all")?.maxPositionals, 1);
+ assert.deepEqual(row("publish", "deploy", "all")?.flags, { preview: "string", to: "string", force: "boolean" });
+ assert.deepEqual(row("publish", "hub")?.flags, { deploy: "boolean", preview: "string", force: "boolean" });
+ assert.deepEqual(row("publish", "homepage")?.flags, {
+ deploy: "boolean",
+ preview: "string",
+ to: "string",
+ force: "boolean",
+ });
+ const parsed = parseArgv(["publish", "hub", "--deploy", "--preview", "r18"], booleanFlags(COMMANDS));
+ assert.deepEqual(parsed, { positionals: ["publish", "hub"], flags: { deploy: true, preview: "r18" } });
+ const deploy = row("publish", "deploy")!;
+ assert.match(argumentProblem(deploy, { preveiw: "x" }, ["jer"])!, /unknown flag --preveiw \(accepts --preview, --to, --force\)/);
+ assert.match(argumentProblem(deploy, { to: true }, ["jer"])!, /--to needs a value/);
+ const u = usage(COMMANDS);
+ assert.match(u, /archilyzer publish index\s+update the index/);
+ assert.match(u, /archilyzer publish build\s+<id\|all> \[--runner local\|docker\|auto\] \[--force\] \[--skip-archives\]/);
+ assert.match(u, /archilyzer publish deploy\s+<id\|all> \[--preview <branch>\] \[--to local\] \[--force\]/);
+ assert.match(u, /archilyzer stage\s+<kind> <target> --run-id <id>/);
+});
+
+test("build site, build all and deploy site are printed aliases of the publish rows", async () => {
+ const { ALIASES } = await import("./publish");
+ assert.equal(ALIASES.buildSite("jer", false), "archilyzer publish index && archilyzer publish build jer --force");
+ assert.equal(ALIASES.buildSite("jer", true), "archilyzer publish build jer --force", "--nodata skips the index");
+ assert.equal(ALIASES.buildAll, "archilyzer publish index && archilyzer publish build all --runner auto");
+ assert.equal(ALIASES.deploySite("jer"), "archilyzer publish deploy jer");
+ assert.equal(ALIASES.deploySite("jer", "r18"), "archilyzer publish deploy jer --preview r18");
+ const row = (...p: string[]) => resolveCommand(COMMANDS, p)!.command;
+ // Same flags as ever: a script that called them still parses.
+ assert.deepEqual(row("build", "site").flags, { nodata: "boolean", "skip-archives": "boolean", "allow-missing-media": "boolean" });
+ assert.deepEqual(row("build", "all").flags, { "skip-archives": "boolean" });
+ assert.deepEqual(row("deploy", "site").flags, { preview: "string" });
+ for (const r of [row("build", "site"), row("build", "all"), row("deploy", "site")]) {
+ assert.match(r.usage, /alias: /);
+ }
+});
+
+test("publish hub / homepage refuse a deploy's flag without --deploy (usage, nothing run)", async () => {
+ const { publishHub, publishHomepage } = await import("./publish");
+ const errors: string[] = [];
+ const out = { log: () => {}, error: (s: string) => errors.push(s) };
+ assert.equal(await publishHub({ preview: "r18" }, out), 2);
+ assert.equal(await publishHomepage({ to: "local" }, out), 2);
+ assert.match(errors.join("\n"), /--preview is a deploy — add --deploy/);
+ assert.match(errors.join("\n"), /--preview and --to are a deploy's — add --deploy/);
+});
diff --git a/common/bin/archilyzer.ts b/common/bin/archilyzer.ts
@@ -76,60 +76,133 @@ export const COMMANDS: Command[] = [
return 0;
},
},
+ // --- publishing as stages (release 18; bin/publish.ts) ----------------------
+ {
+ path: ["publish", "index"],
+ usage:
+ "update the index: the LMDB index, the stats datasets and the chart templates in one child (8 GB heap), then the index stamp every build reads",
+ run: async () => (await import("./publish")).publishIndex(),
+ },
+ {
+ path: ["publish", "build"],
+ usage:
+ "<id|all> [--runner local|docker|auto] [--force] [--skip-archives] build a site (or every stale one) into its bundle <exportBuildsDir>/<id>/out from the current index; --runner docker builds every site in containers (host only); a fresh site is a no-op without --force",
+ flags: { runner: "string", force: "boolean", "skip-archives": "boolean" },
+ maxPositionals: 1,
+ run: async ({ positionals, flags }) => {
+ const [target] = positionals;
+ const runner = flags.runner;
+ if (!target || (runner !== undefined && runner !== "local" && runner !== "docker" && runner !== "auto")) {
+ console.error("publish build: give <id|all> [--runner local|docker|auto]");
+ return 2;
+ }
+ return (await import("./publish")).publishBuild({
+ target,
+ runner: runner as "local" | "docker" | "auto" | undefined,
+ force: flags.force === true,
+ skipArchives: flags["skip-archives"] === true,
+ });
+ },
+ },
+ {
+ path: ["publish", "deploy"],
+ usage:
+ "<id|all> [--preview <branch>] [--to local] [--force] ship a site's bundle to its Pages project (a preview with --preview), or with --to local into ARCHILYZER_SITE_OUT; a bundle already deployed there is a no-op without --force",
+ flags: { preview: "string", to: "string", force: "boolean" },
+ maxPositionals: 1,
+ run: async ({ positionals, flags }) => {
+ const [target] = positionals;
+ const to = flags.to;
+ if (!target || (to !== undefined && to !== "pages" && to !== "local")) {
+ console.error("publish deploy: give <id|all> [--preview <branch>] [--to local]");
+ return 2;
+ }
+ return (await import("./publish")).publishDeploy({
+ target,
+ preview: typeof flags.preview === "string" ? flags.preview : undefined,
+ to: to as "pages" | "local" | undefined,
+ force: flags.force === true,
+ });
+ },
+ },
+ {
+ path: ["publish", "hub"],
+ usage:
+ "[--deploy] [--preview <branch>] [--force] build the hub into its bundle <exportBuildsDir>/_hub/out, then (--deploy) ship it",
+ flags: { deploy: "boolean", preview: "string", force: "boolean" },
+ run: async ({ flags }) =>
+ (await import("./publish")).publishHub({
+ deploy: flags.deploy === true,
+ preview: typeof flags.preview === "string" ? flags.preview : undefined,
+ force: flags.force === true,
+ }),
+ },
+ {
+ path: ["publish", "homepage"],
+ usage:
+ "[--deploy] [--preview <branch>] [--to local] [--force] build homepage/out (source mirror included), then (--deploy) ship it",
+ flags: { deploy: "boolean", preview: "string", to: "string", force: "boolean" },
+ run: async ({ flags }) => {
+ const to = flags.to;
+ if (to !== undefined && to !== "pages" && to !== "local") {
+ console.error("publish homepage: --to is pages or local");
+ return 2;
+ }
+ return (await import("./publish")).publishHomepage({
+ deploy: flags.deploy === true,
+ preview: typeof flags.preview === "string" ? flags.preview : undefined,
+ to: to as "pages" | "local" | undefined,
+ force: flags.force === true,
+ });
+ },
+ },
+ // `publish status` and `publish now` read the publish state view — release 18
+ // slice S3 adds their rows here.
+ {
+ path: ["stage"],
+ usage:
+ "<kind> <target> --run-id <id> [--preview <b>] [--to local] [--runner docker] [--force] [--skip-archives] [--allow-missing-media] [--index-after <ms>] [--built-after <ms>] INTERNAL: one publish stage, as the editor's job runs it (exit 0 ran/no-op, 1 failed, 2 usage, 3 precondition not met, 130 cancelled)",
+ // publish/stages.ts STAGE_FLAGS, spelled out so this table stays free of
+ // imports (_cli.test.ts holds the two equal).
+ flags: {
+ "run-id": "string",
+ preview: "string",
+ to: "string",
+ runner: "string",
+ force: "boolean",
+ "skip-archives": "boolean",
+ "allow-missing-media": "boolean",
+ "index-after": "string",
+ "built-after": "string",
+ },
+ maxPositionals: 2,
+ run: async (ctx) => (await import("./publish")).stageRow(ctx),
+ },
+ // The rows the stages replaced, kept as printed aliases.
{
path: ["build", "site"],
usage:
- "<id> [--nodata] [--skip-archives] [--allow-missing-media] data phase + compose + next build into export/out (default id: SITE_ID)",
+ "<id> [--nodata] [--skip-archives] [--allow-missing-media] alias: publish index (not with --nodata) + publish build <id> --force (default id: SITE_ID)",
flags: { nodata: "boolean", "skip-archives": "boolean", "allow-missing-media": "boolean" },
maxPositionals: 1,
run: async ({ positionals, flags, env }) => {
const siteId = siteIdFrom(positionals, env, "build site");
if (!siteId) return 2;
- // A missing site.json reads as a site of defaults, so a typo would build
- // the whole data phase before compose noticed. Refuse it up front.
- const { listSiteIds } = await import("../lib/site");
- const known = listSiteIds();
- if (!known.includes(siteId)) {
- console.error(
- `build site: no site "${siteId}" (configured: ${known.join(", ") || "none"})`,
- );
- return 2;
- }
- const { buildSite } = await import("../publish/build");
- const code = await buildSite(siteId, {
- signal: interrupted(),
- skipData: flags.nodata === true,
+ return (await import("./publish")).buildSiteAlias({
+ siteId,
+ nodata: flags.nodata === true,
skipArchives: flags["skip-archives"] === true,
allowMissingMedia: flags["allow-missing-media"] === true,
});
- if (code !== 0) console.error(`build site ${siteId}: failed (exit ${code})`);
- return code;
},
},
{
path: ["build", "all"],
usage:
- "[--skip-archives] build every site: docker fan-out when an engine answers, else serially on the host",
+ "[--skip-archives] alias: publish index + publish build all --runner auto (containers when an engine answers, else serially on the host)",
flags: { "skip-archives": "boolean" },
- run: async ({ flags }) => {
- const { buildAll, dockerAvailable } = await import("../publish/build");
- const signal = interrupted();
- const useDocker = await dockerAvailable(signal);
- if (!useDocker) {
- console.log(
- "[notice] No container engine available — building sites serially on the host.",
- );
- }
- const outcomes = await buildAll({
- signal,
- mode: useDocker ? "docker" : "basic",
- skipArchives: flags["skip-archives"] === true,
- });
- const failed = outcomes.filter((o) => o.code !== 0);
- console.log(`\n=== Summary: ${outcomes.length - failed.length}/${outcomes.length} built ===`);
- for (const f of failed) console.error(` ${f.siteId}: exit ${f.code}`);
- return failed.length || signal.aborted ? 1 : 0;
- },
+ run: async ({ flags }) =>
+ (await import("./publish")).buildAllAlias({ skipArchives: flags["skip-archives"] === true }),
},
{
path: ["build", "hub"],
@@ -263,19 +336,16 @@ export const COMMANDS: Command[] = [
{
path: ["deploy", "site"],
usage:
- "<id> [--preview <branch>] ship the site built in export/out to its Pages project (default id: SITE_ID)",
+ "<id> [--preview <branch>] alias: publish deploy <id> [--preview <branch>] — ship the site's bundle to its Pages project (default id: SITE_ID)",
flags: { preview: "string" },
maxPositionals: 1,
run: async ({ positionals, flags, env }) => {
const siteId = siteIdFrom(positionals, env, "deploy site");
if (!siteId) return 2;
- const { deploySite } = await import("../publish/build");
- return refusalsExit(() =>
- deploySite(siteId, {
- signal: interrupted(),
- previewBranch: typeof flags.preview === "string" ? flags.preview : undefined,
- }),
- );
+ return (await import("./publish")).deploySiteAlias({
+ siteId,
+ preview: typeof flags.preview === "string" ? flags.preview : undefined,
+ });
},
},
{
diff --git a/common/bin/compose-site.ts b/common/bin/compose-site.ts
@@ -23,9 +23,9 @@
import path from "node:path";
import { cp, link, mkdir, rm, readdir, access, readFile, writeFile, stat, rename } from "node:fs/promises";
-import { createHash } from "node:crypto";
import type { Dirent } from "node:fs";
import { getPaths, type Paths } from "../lib/paths";
+import { dirSignature } from "../lib/dirSignature";
import { getSite, resolveSocialLinks, resolveHubUrl, type Site } from "../lib/site";
import { getSettings } from "../lib/settings";
import {
@@ -688,51 +688,6 @@ async function writeComposeCache(p: string, cache: ComposeCache): Promise<void>
await writeFile(p, JSON.stringify(cache));
}
-// A cheap content signature for a directory: relative path + size + mtime of
-// every file within, hashed. It reflects exactly what a recursive copy would
-// move, so an unchanged source (build:index skipped it incrementally) yields the
-// same signature and the copy is skipped. The shared transcript/subs trees hold
-// only a bounded handful of paginated page files per channel, so this is far
-// cheaper than the copy it guards. Returns "" when the dir is absent/empty.
-//
-// `ignoreBasename` excludes files by name from the signature. The shared
-// per-channel trees carry a manifest.json that build:index rewrites with a fresh
-// `generatedAt` for EVERY channel whenever ANY channel mutates (the shared
-// page-writer loop runs over all channels). Its pageCount/slugToPage only change
-// when the channel's pages change — which the page files already capture — so
-// excluding it lets an unchanged channel stay skipped on a partial-change build
-// instead of re-copying all 50-odd channels. The tiny stale manifest left behind
-// on a skip is structurally identical (only its timestamp differs).
-async function dirSignature(
- dir: string,
- ignoreBasename?: string,
-): Promise<string> {
- const h = createHash("sha1");
- let any = false;
- const walk = async (rel: string): Promise<void> => {
- let ents: Dirent[];
- try {
- ents = await readdir(path.join(dir, rel), { withFileTypes: true });
- } catch {
- return;
- }
- ents.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
- for (const e of ents) {
- if (!e.isDirectory() && e.name === ignoreBasename) continue;
- const childRel = rel ? `${rel}/${e.name}` : e.name;
- if (e.isDirectory()) {
- await walk(childRel);
- } else {
- any = true;
- const s = await stat(path.join(dir, childRel));
- h.update(`${childRel}\t${s.size}\t${s.mtimeMs}\n`);
- }
- }
- };
- await walk("");
- return any ? h.digest("hex") : "";
-}
-
// Materialize the member subset of a shared per-channel tree into public/,
// skipping channels whose source is unchanged since the last compose. Replaces
// the previous "rm -rf the whole tree then cp every member" with an in-place
diff --git a/common/bin/publish.ts b/common/bin/publish.ts
@@ -0,0 +1,299 @@
+// `archilyzer publish …` and `archilyzer stage …` (release 18): the publish
+// stages from the command line. The table rows are in archilyzer.ts; this is
+// what they run.
+//
+// publish index update-index (as a child: its heap cap)
+// publish build <id|all> [--runner local|docker|auto] [--force] [--skip-archives]
+// publish deploy <id|all> [--preview <b>] [--to local] [--force]
+// publish hub [--deploy] [--preview <b>] [--force]
+// publish homepage [--deploy] [--preview <b>] [--to local] [--force]
+// stage <kind> <target> [flags] the child the editor spawns
+//
+// Every stage runs under the publish lock (publish/stageLock.ts), so a CLI run
+// beside the editor waits for the editor's stage — and the editor's for it.
+// `publish index` runs the SAME child the editor spawns (stageCommand: the
+// index and stats builds want its 8 GB heap); the rest run in this process.
+// `publish status` and `publish now` are release 18 S3's (publishState.ts).
+
+import { killChildTreesNow, runChildIntoLog, setKillChildTrees } from "../jobs/runChild";
+import { getPaths, type Paths } from "../lib/paths";
+import { siteDeployProblem } from "../lib/builtExport";
+import { listSites, listSiteIds } from "../lib/site";
+import { newStampId } from "../publish/stamps";
+import { STAGE_EXIT, runStage, stageCommand, stageMain } from "../publish/stageRun";
+import { parseStageArgs, type StageRequest } from "../publish/stages";
+import type { CommandContext } from "./_cli";
+
+type Out = { log: (s: string) => void; error: (s: string) => void };
+
+function terminalLog(line: string): void {
+ process.stdout.write(line.endsWith("\n") ? line : `${line}\n`);
+}
+
+/** A run id for the stages one CLI command runs (the editor's are its own). */
+export function cliRunId(): string {
+ return `cli-${newStampId()}`;
+}
+
+// Ctrl-C / SIGTERM cancel the stage in flight, as the editor's Cancel does. A
+// SECOND one does not wait for the unwind: it kills the detached process
+// groups this process started (`next build`'s, wrangler's) and exits 130 —
+// never leaving an orphan builder writing export/out. Idempotent: one handler.
+function interrupted(): AbortSignal {
+ const ac = new AbortController();
+ const onSignal = () => {
+ if (!ac.signal.aborted) {
+ ac.abort();
+ return;
+ }
+ killChildTreesNow("SIGKILL");
+ process.exit(STAGE_EXIT.cancelled);
+ };
+ process.on("SIGINT", onSignal);
+ process.on("SIGTERM", onSignal);
+ return ac.signal;
+}
+
+// --- stage (the child) --------------------------------------------------------
+
+export async function stageRow(ctx: CommandContext, out: Out = console): Promise<number> {
+ const req = parseStageArgs(ctx.positionals, ctx.flags);
+ if ("error" in req) {
+ out.error(req.error);
+ return STAGE_EXIT.usage;
+ }
+ return stageMain(req);
+}
+
+// --- publish index ------------------------------------------------------------
+
+/**
+ * Run update-index as the editor does: a child with the index heap cap. A
+ * Ctrl-C reaches the child from the terminal (it unwinds and exits 130); a
+ * SIGTERM to this process is passed on to it.
+ */
+export async function publishIndex(opts: { paths?: Paths; runId?: string } = {}): Promise<number> {
+ const paths = opts.paths ?? getPaths();
+ const req: StageRequest = { kind: "update-index", target: "_index", runId: opts.runId ?? cliRunId() };
+ const cmd = stageCommand(paths, req);
+ const ac = new AbortController();
+ const onInt = () => {};
+ const onTerm = () => ac.abort();
+ process.on("SIGINT", onInt);
+ process.on("SIGTERM", onTerm);
+ try {
+ return await runChildIntoLog(terminalLog, ac.signal, cmd);
+ } finally {
+ process.off("SIGINT", onInt);
+ process.off("SIGTERM", onTerm);
+ }
+}
+
+// --- publish build / deploy / hub / homepage ----------------------------------
+
+function knownSite(id: string, name: string, out: Out, paths?: Paths): boolean {
+ const known = listSiteIds(paths);
+ if (known.includes(id)) return true;
+ out.error(`${name}: no site "${id}" (configured: ${known.join(", ") || "none"})`);
+ return false;
+}
+
+async function run(req: StageRequest, signal: AbortSignal, paths?: Paths): Promise<number> {
+ setKillChildTrees(true);
+ return (await runStage(req, { paths, signal })).code;
+}
+
+export type BuildArgs = {
+ target: string; // a site id or "all"
+ runner?: "local" | "docker" | "auto";
+ force?: boolean;
+ skipArchives?: boolean;
+ allowMissingMedia?: boolean;
+ runId?: string;
+ paths?: Paths;
+ signal?: AbortSignal;
+};
+
+export async function publishBuild(a: BuildArgs, out: Out = console): Promise<number> {
+ const all = a.target === "all";
+ if (!all && !knownSite(a.target, "publish build", out, a.paths)) return STAGE_EXIT.usage;
+ const signal = a.signal ?? interrupted();
+ let runner: "local" | "docker" = "local";
+ if (a.runner === "docker" || a.runner === "auto") {
+ if (!all) {
+ out.error("publish build: --runner docker builds every site in containers — publish build all --runner docker");
+ return STAGE_EXIT.usage;
+ }
+ if (a.runner === "docker") runner = "docker";
+ else {
+ const { dockerAvailable } = await import("../publish/build");
+ runner = (await dockerAvailable(signal)) ? "docker" : "local";
+ if (runner === "local") out.log("[notice] No container engine available — building sites serially on the host.");
+ }
+ }
+ return run(
+ {
+ kind: "build-site",
+ target: all ? "_all" : a.target,
+ runId: a.runId ?? cliRunId(),
+ ...(all ? { runner } : {}),
+ ...(a.force ? { force: true } : {}),
+ ...(a.skipArchives ? { skipArchives: true } : {}),
+ ...(a.allowMissingMedia ? { allowMissingMedia: true } : {}),
+ },
+ signal,
+ a.paths,
+ );
+}
+
+export type DeployArgs = {
+ target: string; // a site id or "all"
+ preview?: string;
+ to?: "pages" | "local";
+ force?: boolean;
+ runId?: string;
+ paths?: Paths;
+ signal?: AbortSignal;
+};
+
+export async function publishDeploy(a: DeployArgs, out: Out = console): Promise<number> {
+ const all = a.target === "all";
+ if (!all && !knownSite(a.target, "publish deploy", out, a.paths)) return STAGE_EXIT.usage;
+ const signal = a.signal ?? interrupted();
+ const runId = a.runId ?? cliRunId();
+ // `all` passes over — quietly, one line — only the sites that are never
+ // deployable by their configuration: a private site, and (to Pages) a site
+ // with no Pages project. Every other refusal (never built, a bundle
+ // problem, a build not of main, a build this run has not made) is a
+ // FAILURE: the rest are still tried, and the run exits 1.
+ const ids: string[] = [];
+ for (const site of all ? listSites(a.paths) : []) {
+ const never = siteDeployProblem(site) ?? (a.to === "local" || site.cloudflareProject?.trim() ? null : "no Cloudflare Pages project");
+ if (never) out.log(`[publish] ${site.siteId}: skipped — ${never}`);
+ else ids.push(site.siteId);
+ }
+ if (!all) ids.push(a.target);
+ let worst = 0;
+ for (const id of ids) {
+ if (signal.aborted) return STAGE_EXIT.cancelled;
+ const code = await run(
+ {
+ kind: "deploy-site",
+ target: id,
+ runId,
+ ...(a.preview ? { preview: a.preview } : {}),
+ ...(a.to ? { to: a.to } : {}),
+ ...(a.force ? { force: true } : {}),
+ },
+ signal,
+ a.paths,
+ );
+ if (code === STAGE_EXIT.cancelled) return code;
+ if (code !== 0) worst = all ? STAGE_EXIT.failed : worst || code;
+ }
+ return worst;
+}
+
+export async function publishHub(
+ a: { deploy?: boolean; preview?: string; force?: boolean; runId?: string; paths?: Paths; signal?: AbortSignal },
+ out: Out = console,
+): Promise<number> {
+ if (a.preview && !a.deploy) {
+ out.error("publish hub: --preview is a deploy — add --deploy");
+ return STAGE_EXIT.usage;
+ }
+ const signal = a.signal ?? interrupted();
+ const runId = a.runId ?? cliRunId();
+ const built = await run({ kind: "build-hub", target: "_hub", runId, ...(a.force ? { force: true } : {}) }, signal, a.paths);
+ if (built !== 0 || !a.deploy) return built;
+ return run({ kind: "deploy-hub", target: "_hub", runId, ...(a.preview ? { preview: a.preview } : {}) }, signal, a.paths);
+}
+
+export async function publishHomepage(
+ a: {
+ deploy?: boolean;
+ preview?: string;
+ to?: "pages" | "local";
+ force?: boolean;
+ runId?: string;
+ paths?: Paths;
+ signal?: AbortSignal;
+ },
+ out: Out = console,
+): Promise<number> {
+ if ((a.preview || a.to) && !a.deploy) {
+ out.error("publish homepage: --preview and --to are a deploy's — add --deploy");
+ return STAGE_EXIT.usage;
+ }
+ const signal = a.signal ?? interrupted();
+ const runId = a.runId ?? cliRunId();
+ const built = await run(
+ { kind: "build-homepage", target: "_homepage", runId, ...(a.force ? { force: true } : {}) },
+ signal,
+ a.paths,
+ );
+ if (built !== 0 || !a.deploy) return built;
+ return run(
+ {
+ kind: "deploy-homepage",
+ target: "_homepage",
+ runId,
+ ...(a.preview ? { preview: a.preview } : {}),
+ ...(a.to ? { to: a.to } : {}),
+ },
+ signal,
+ a.paths,
+ );
+}
+
+// --- the old rows, as printed aliases -------------------------------------------
+
+export const ALIASES = {
+ buildSite: (id: string, nodata: boolean) =>
+ `${nodata ? "" : "archilyzer publish index && "}archilyzer publish build ${id} --force`,
+ buildAll: "archilyzer publish index && archilyzer publish build all --runner auto",
+ deploySite: (id: string, preview?: string) =>
+ `archilyzer publish deploy ${id}${preview ? ` --preview ${preview}` : ""}`,
+} as const;
+
+/** `build site <id> [--nodata]` = `publish index` (skipped by --nodata) + `publish build <id> --force`. */
+export async function buildSiteAlias(
+ a: { siteId: string; nodata: boolean; skipArchives: boolean; allowMissingMedia: boolean },
+ out: Out = console,
+): Promise<number> {
+ if (!knownSite(a.siteId, "build site", out)) return STAGE_EXIT.usage;
+ out.log(`[alias] build site is now: ${ALIASES.buildSite(a.siteId, a.nodata)}`);
+ const runId = cliRunId();
+ if (!a.nodata) {
+ const code = await publishIndex({ runId });
+ if (code !== 0) return code;
+ }
+ return publishBuild(
+ {
+ target: a.siteId,
+ force: true,
+ skipArchives: a.skipArchives,
+ allowMissingMedia: a.allowMissingMedia,
+ runId,
+ },
+ out,
+ );
+}
+
+/** `build all` = `publish index` + `publish build all --runner auto`. */
+export async function buildAllAlias(a: { skipArchives: boolean }, out: Out = console): Promise<number> {
+ out.log(`[alias] build all is now: ${ALIASES.buildAll}`);
+ const runId = cliRunId();
+ const code = await publishIndex({ runId });
+ if (code !== 0) return code;
+ return publishBuild({ target: "all", runner: "auto", skipArchives: a.skipArchives, runId }, out);
+}
+
+/** `deploy site <id>` = `publish deploy <id>`. */
+export async function deploySiteAlias(
+ a: { siteId: string; preview?: string },
+ out: Out = console,
+): Promise<number> {
+ out.log(`[alias] deploy site is now: ${ALIASES.deploySite(a.siteId, a.preview)}`);
+ return publishDeploy({ target: a.siteId, preview: a.preview }, out);
+}
diff --git a/common/jobs/runChild.test.ts b/common/jobs/runChild.test.ts
@@ -0,0 +1,98 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { killChildTreesNow, killsChildTrees, runChildIntoLog, setKillChildTrees } from "./runChild";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common test
+//
+// Release 18: a publish stage child turns on tree-kill, so a cancel takes the
+// child's whole process group — `next build`'s workers, wrangler — not just
+// the direct child. A shell that starts a background grandchild stands in.
+
+function alive(pid: number): boolean {
+ try {
+ process.kill(pid, 0);
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+async function waitFor(cond: () => boolean, ms = 5_000): Promise<boolean> {
+ const end = Date.now() + ms;
+ while (Date.now() < end) {
+ if (cond()) return true;
+ await new Promise((r) => setTimeout(r, 25));
+ }
+ return cond();
+}
+
+async function runWithGrandchild(tree: boolean): Promise<{ grandchild: number; code: number }> {
+ const dir = mkdtempSync(path.join(os.tmpdir(), "run-child-"));
+ const pidFile = path.join(dir, "gc.pid");
+ setKillChildTrees(tree);
+ try {
+ const ac = new AbortController();
+ const running = runChildIntoLog(() => {}, ac.signal, {
+ command: "sh",
+ args: ["-c", `sleep 30 & echo $! > '${pidFile}'; wait`],
+ cwd: dir,
+ });
+ let grandchild = 0;
+ await waitFor(() => {
+ try {
+ grandchild = Number(readFileSync(pidFile, "utf8").trim());
+ return grandchild > 0;
+ } catch {
+ return false;
+ }
+ });
+ ac.abort();
+ const code = await running;
+ return { grandchild, code };
+ } finally {
+ setKillChildTrees(false);
+ rmSync(dir, { recursive: true, force: true });
+ }
+}
+
+test("tree-kill is off by default", () => {
+ assert.equal(killsChildTrees(), false);
+});
+
+test("with tree-kill on, a cancel takes the grandchildren too", { skip: process.platform === "win32" }, async () => {
+ const { grandchild, code } = await runWithGrandchild(true);
+ assert.notEqual(code, 0);
+ assert.ok(await waitFor(() => !alive(grandchild)), `grandchild ${grandchild} survived`);
+});
+
+test("killChildTreesNow (a second Ctrl-C) takes every live group down at once, no cancel needed", { skip: process.platform === "win32" }, async () => {
+ const dir = mkdtempSync(path.join(os.tmpdir(), "run-child-now-"));
+ const pidFile = path.join(dir, "gc.pid");
+ setKillChildTrees(true);
+ try {
+ const running = runChildIntoLog(() => {}, new AbortController().signal, {
+ command: "sh",
+ args: ["-c", `sleep 30 & echo $! > '${pidFile}'; wait`],
+ cwd: dir,
+ });
+ let grandchild = 0;
+ await waitFor(() => {
+ try {
+ grandchild = Number(readFileSync(pidFile, "utf8").trim());
+ return grandchild > 0;
+ } catch {
+ return false;
+ }
+ });
+ killChildTreesNow("SIGKILL");
+ assert.notEqual(await running, 0);
+ assert.ok(await waitFor(() => !alive(grandchild)), `grandchild ${grandchild} survived`);
+ } finally {
+ setKillChildTrees(false);
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
diff --git a/common/jobs/runChild.ts b/common/jobs/runChild.ts
@@ -10,6 +10,44 @@ export type RunChildOpts = {
label?: string;
};
+// Release 18: a publish STAGE CHILD (and `archilyzer publish …`) is a process of
+// its own whose children run whole trees — `pnpm exec next build` and its
+// workers, wrangler, docker. Killing only the direct child leaves the rest
+// running after a Cancel. With tree-kill on, each child is spawned as the
+// leader of its own process group, and a cancel signals the whole GROUP. Off
+// by default, so the editor's own in-process jobs are unchanged; a stage child
+// turns it on once at start. Per process, on globalThis like the registry.
+const TREE_KILL_KEY = Symbol.for("archilyzer.runChild.treeKill");
+type TreeKillGlobal = { [TREE_KILL_KEY]?: boolean };
+
+export function setKillChildTrees(on: boolean): void {
+ (globalThis as TreeKillGlobal)[TREE_KILL_KEY] = on;
+}
+
+export function killsChildTrees(): boolean {
+ return (globalThis as TreeKillGlobal)[TREE_KILL_KEY] === true;
+}
+
+// The process groups this process leads right now (tree-kill mode), so a
+// second Ctrl-C / SIGTERM — which exits at once, without waiting for the
+// cancel to unwind — can still take them down first (`killChildTreesNow`).
+const LIVE_GROUPS_KEY = Symbol.for("archilyzer.runChild.liveGroups");
+function liveGroups(): Set<number> {
+ const g = globalThis as { [LIVE_GROUPS_KEY]?: Set<number> };
+ return (g[LIVE_GROUPS_KEY] ??= new Set());
+}
+
+/** Signal every live child process group this process started (tree-kill mode). */
+export function killChildTreesNow(sig: NodeJS.Signals = "SIGKILL"): void {
+ for (const pgid of liveGroups()) {
+ try {
+ process.kill(-pgid, sig);
+ } catch {
+ // Already gone.
+ }
+ }
+}
+
// Spawn a child process and stream its combined stdout/stderr into `onLog`,
// resolving with the exit code. Used inside a managed function's `fn` to run a
// sub-command as part of a larger job (build-then-deploy, multi-phase builds)
@@ -25,13 +63,28 @@ export async function runChildIntoLog(
opts: RunChildOpts,
): Promise<number> {
const prefix = opts.label ?? "";
+ const tree = killsChildTrees();
const child: ResultPromise = execa(opts.command, opts.args, {
cwd: opts.cwd,
env: opts.env,
all: true,
buffer: false,
reject: false,
+ ...(tree ? { detached: true } : {}),
});
+ if (tree && child.pid) liveGroups().add(child.pid);
+ // Signal the child — or, with tree-kill, its whole process group.
+ const signalChild = (sig: NodeJS.Signals) => {
+ if (tree && child.pid) {
+ try {
+ process.kill(-child.pid, sig);
+ return;
+ } catch {
+ // The group is gone (or was never made): the child alone.
+ }
+ }
+ child.kill(sig);
+ };
// Line-buffer: execa chunks aren't line-aligned, and the managed-function
// onLog appends a newline per call, so emitting raw chunks would inject
@@ -55,9 +108,10 @@ export async function runChildIntoLog(
let killTimer: ReturnType<typeof setTimeout> | null = null;
const onAbort = () => {
- child.kill("SIGTERM");
+ signalChild("SIGTERM");
killTimer = setTimeout(() => {
- if (child.killed === false) child.kill("SIGKILL");
+ if (tree) signalChild("SIGKILL");
+ else if (child.killed === false) child.kill("SIGKILL");
}, 5_000);
killTimer.unref?.();
};
@@ -73,6 +127,9 @@ export async function runChildIntoLog(
buf = "";
}
if (killTimer) clearTimeout(killTimer);
+ // The leader is gone; whatever of its group a cancel left is not wanted.
+ if (tree && signal.aborted) signalChild("SIGKILL");
+ if (tree && child.pid) liveGroups().delete(child.pid);
signal.removeEventListener("abort", onAbort);
}
}
diff --git a/common/lib/dirSignature.ts b/common/lib/dirSignature.ts
@@ -0,0 +1,54 @@
+// A directory's cheap content signature — moved here, unchanged, from
+// compose-site.ts (release 18): the publish stages' `inputSig` is computed
+// with the SAME function compose uses to decide what to skip, so "a site is
+// fresh" and "compose would skip everything" can never disagree.
+
+import path from "node:path";
+import { readdir, stat } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import type { Dirent } from "node:fs";
+
+// A cheap content signature for a directory: relative path + size + mtime of
+// every file within, hashed. It reflects exactly what a recursive copy would
+// move, so an unchanged source (build:index skipped it incrementally) yields the
+// same signature and the copy is skipped. The shared transcript/subs trees hold
+// only a bounded handful of paginated page files per channel, so this is far
+// cheaper than the copy it guards. Returns "" when the dir is absent/empty.
+//
+// `ignoreBasename` excludes files by name from the signature. The shared
+// per-channel trees carry a manifest.json that build:index rewrites with a fresh
+// `generatedAt` for EVERY channel whenever ANY channel mutates (the shared
+// page-writer loop runs over all channels). Its pageCount/slugToPage only change
+// when the channel's pages change — which the page files already capture — so
+// excluding it lets an unchanged channel stay skipped on a partial-change build
+// instead of re-copying all 50-odd channels. The tiny stale manifest left behind
+// on a skip is structurally identical (only its timestamp differs).
+export async function dirSignature(
+ dir: string,
+ ignoreBasename?: string,
+): Promise<string> {
+ const h = createHash("sha1");
+ let any = false;
+ const walk = async (rel: string): Promise<void> => {
+ let ents: Dirent[];
+ try {
+ ents = await readdir(path.join(dir, rel), { withFileTypes: true });
+ } catch {
+ return;
+ }
+ ents.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+ for (const e of ents) {
+ if (!e.isDirectory() && e.name === ignoreBasename) continue;
+ const childRel = rel ? `${rel}/${e.name}` : e.name;
+ if (e.isDirectory()) {
+ await walk(childRel);
+ } else {
+ any = true;
+ const s = await stat(path.join(dir, childRel));
+ h.update(`${childRel}\t${s.size}\t${s.mtimeMs}\n`);
+ }
+ }
+ };
+ await walk("");
+ return any ? h.digest("hex") : "";
+}
diff --git a/common/publish/__fixtures__/stamps.ts b/common/publish/__fixtures__/stamps.ts
@@ -0,0 +1,40 @@
+// Stamp builders shared by the publish tests (stamps, stages).
+import type { BuiltStamp, IndexStamp } from "../stamps";
+
+export function indexStamp(over: Partial<IndexStamp> = {}): IndexStamp {
+ return {
+ v: 1,
+ stampId: "s1",
+ generation: 7,
+ scannedAt: 1_000,
+ builtAt: 2_000,
+ templatesAt: 1_900,
+ commit: "abc",
+ index: { shortCircuited: false, added: 1, changed: 2, removed: 0, heldChannels: [] },
+ stats: { shortCircuited: true, notIndexedYet: 0, notIndexable: 3 },
+ sites: { jer: { siteFp: "f1", statsFp: null, inputSig: "sig-jer" } },
+ hubSig: "hub-1",
+ ...over,
+ };
+}
+
+export function builtStamp(over: Partial<BuiltStamp> = {}): BuiltStamp {
+ return {
+ v: 1,
+ stampId: "b1",
+ target: "jer",
+ kind: "site",
+ indexStampId: "s1",
+ inputSig: "sig-jer",
+ builtAt: 3_000,
+ commit: "abc",
+ branch: "main",
+ runner: "local",
+ audience: "public",
+ corpusGeneratedAt: "2026-10-06T00:00:00.000Z",
+ files: 10,
+ bytes: 1234,
+ archivesStaged: 0,
+ ...over,
+ };
+}
diff --git a/common/publish/build.ts b/common/publish/build.ts
@@ -9,7 +9,7 @@
// forbids non-serializable args). Keep them as plain helpers.
import path from "node:path";
-import { mkdir, readdir, rm, stat } from "node:fs/promises";
+import { cp, lstat, mkdir, readdir, readFile, rename, rm, stat, symlink } from "node:fs/promises";
import { createReadStream, existsSync } from "node:fs";
import { S3Client, HeadObjectCommand } from "@aws-sdk/client-s3";
import { Upload } from "@aws-sdk/lib-storage";
@@ -34,6 +34,7 @@ import {
import { getPaths, type Paths } from "../lib/paths";
import { getSettings } from "../lib/settings";
import { getSite, listSites, type Site } from "../lib/site";
+import { builtStampPath } from "./stamps";
// Where the basic (host) build writes the static bundle to deploy: the fixed
// export/out, composed one site at a time. The docker fan-out writes per-site
@@ -482,7 +483,7 @@ export async function dockerAvailable(signal: AbortSignal): Promise<boolean> {
// These are pool-wide: no SITE_ID, and no EXPORT_PUBLIC_DIR override so the
// shared index/staging land at their canonical export/.export-index location
// (exactly what the fan-out containers mount read-only).
-async function runHostScript(
+export async function runHostScript(
onLog: (line: string) => void,
signal: AbortSignal,
paths: Paths,
@@ -507,7 +508,7 @@ async function runHostScript(
// change re-runs only `COPY . .` onward, but a Dockerfile or lockfile change
// re-installs every dependency first, inside this job. `archilyzer doctor`
// warns ahead of that when the image is absent or older than the Dockerfile.
-async function ensureBuildImage(
+export async function ensureBuildImage(
onLog: (line: string) => void,
signal: AbortSignal,
paths: Paths,
@@ -529,7 +530,7 @@ async function ensureBuildImage(
// next/font/google, which fetches the site's fonts from Google at build time;
// `--network=none` would fail the build. Isolation still comes from the per-site
// output dir, the read-only shared mounts, and the non-root `-u` user.
-async function runDockerBuildOne(
+export async function runDockerBuildOne(
onLog: (line: string) => void,
signal: AbortSignal,
siteId: string,
@@ -557,6 +558,10 @@ async function runDockerBuildOne(
"-v", `${paths.exportIndexDir}:/data/export/.export-index:ro`,
"-v", `${siteDir}:/site`,
"-e", `SITE_ID=${siteId}`,
+ // The container's `build site` is `publish build --force` (release 18):
+ // its publish lock lives inside the container, never on the host's
+ // mount — the host's lock is held by the fan-out stage itself.
+ "-e", `EXPORT_BUILDS_DIR=${CONTAINER_BUILDS_DIR}`,
);
// Mount the host settings.json fresh (build config: archive storage, size caps)
// rather than relying on a possibly-stale copy — it is NOT baked into the image.
@@ -718,7 +723,7 @@ export async function runDockerDeployAllPhase(
// Bounded-concurrency map over a fixed work set, preserving input order in the
// results. No external dep; a fresh worker pulls the next index until exhausted.
-async function runWithConcurrency<T, R>(
+export async function runWithConcurrency<T, R>(
items: T[],
limit: number,
worker: (item: T) => Promise<R>,
@@ -797,7 +802,14 @@ export async function buildSite(
*/
export async function deploySite(
siteId: string,
- opts: PublishOpts & { previewBranch?: string } = {},
+ opts: PublishOpts & {
+ previewBranch?: string;
+ // The bundle to ship and where its oversize archives were staged. Default:
+ // export/out and the host staging dir (the editor's deploy action); the
+ // deploy-site stage passes the site's own bundle under exportBuildsDir.
+ outDir?: string;
+ stagingDir?: string;
+ } = {},
): Promise<void> {
const { paths, onLog, signal } = resolved(opts);
if (opts.previewBranch !== undefined) {
@@ -814,7 +826,7 @@ export async function deploySite(
`Site "${site.siteId}" has no Cloudflare Pages project configured.`,
);
}
- const outDir = resolveOutDir(site.siteId, paths);
+ const outDir = opts.outDir ?? resolveOutDir(site.siteId, paths);
const builtProblem = builtSiteProblem(outDir, site.siteId);
if (builtProblem) throw new Error(builtProblem);
// Before the R2 upload below: a bundle built private is never deployed.
@@ -831,7 +843,7 @@ export async function deploySite(
}
// Push oversize archives to R2 first, so the manifest URLs the Pages deploy
// publishes resolve immediately. No-op when R2 isn't configured.
- const uploadCode = await runArchiveUploadIntoLog(onLog, signal, site, paths);
+ const uploadCode = await runArchiveUploadIntoLog(onLog, signal, site, paths, opts.stagingDir);
if (signal.aborted) return;
if (uploadCode !== 0) {
throw new Error(`Archive R2 upload failed (exit ${uploadCode}).`);
@@ -1018,7 +1030,7 @@ async function runPagesDeployIntoLog(
* quietly on a cancel. No R2 step: the hub holds no archives.
*/
export async function deployHub(
- opts: PublishOpts & { previewBranch?: string } = {},
+ opts: PublishOpts & { previewBranch?: string; outDir?: string } = {},
): Promise<void> {
const { paths, onLog, signal } = resolved(opts);
if (opts.previewBranch !== undefined) {
@@ -1029,7 +1041,7 @@ export async function deployHub(
const project = getHomepageConfig(paths).cloudflareProject;
const projectProblem = hubProjectProblem(project);
if (projectProblem) throw new Error(projectProblem);
- const outDir = resolveOutDir("", paths);
+ const outDir = opts.outDir ?? resolveOutDir("", paths);
const builtProblem = builtHubProblem(outDir);
if (builtProblem) throw new Error(builtProblem);
if (branch) onLog(`=== Deploy hub (preview "${branch}") ===\n`);
@@ -1197,3 +1209,254 @@ export async function deployHomepage(
if (signal.aborted) return;
if (code !== 0) throw new Error(`Homepage deploy failed (exit ${code}).`);
}
+
+// ---------------------------------------------------------------------------
+// Per-target bundles (release 18). `<exportBuildsDir>/<target>/out` is THE
+// bundle every deploy of `target` ships, whichever runner built it: a site's
+// id, `_hub`, or (stamps only) `_homepage` — the homepage's bundle stays
+// homepage/out, where the source gate withdraws from.
+//
+// The local runner builds into the shared export/out (Next's `distDir` may not
+// leave the project, and `public/` is copied into `out/` — the plan's
+// "Decided"), then INSTALLS it: moved to `<target>/out.next`, swapped in
+// (`out → out.prev`, `out.next → out`, `out.prev` removed). A leftover
+// `out.next` is deleted first. On one filesystem the move is a rename; across
+// two (the container: export/out is an image layer, the builds dir a volume)
+// rename fails EXDEV and the bundle is copied, then the source removed.
+// export/out is then a SYMLINK to the bundle built last, so older readers and
+// `serve out` keep working; `next build` removes the link itself (an `rm` of
+// the path, recursive — never its target) before it writes a real out/.
+// ---------------------------------------------------------------------------
+
+// Inside the docker per-site build container: where `publish build` keeps its
+// lock (runDockerBuildOne passes it as EXPORT_BUILDS_DIR).
+export const CONTAINER_BUILDS_DIR = "/tmp/archilyzer-builds";
+
+/** `<exportBuildsDir>/<target>/out` — the target's bundle (= dockerSiteOutDir for a site). */
+export function bundleDir(paths: Pick<Paths, "exportBuildsDir">, target: string): string {
+ return path.join(paths.exportBuildsDir, target, "out");
+}
+
+/** export/out: where `next build` writes, and afterwards a link to the last bundle. */
+export function exportOutPath(paths: Pick<Paths, "exportDir">): string {
+ return path.join(paths.exportDir, "out");
+}
+
+// The filesystem calls a bundle move makes, injectable so the EXDEV path is
+// tested without two filesystems.
+export type BundleFs = {
+ rename: typeof rename;
+ cp: typeof cp;
+ rm: typeof rm;
+ mkdir: typeof mkdir;
+};
+const realBundleFs: BundleFs = { rename, cp, rm, mkdir };
+
+async function lexists(p: string): Promise<boolean> {
+ return lstat(p).then(
+ () => true,
+ () => false,
+ );
+}
+
+// Move a directory: rename, or across filesystems a copy then a remove.
+async function moveDir(src: string, dest: string, fsOps: BundleFs): Promise<"rename" | "copy"> {
+ try {
+ await fsOps.rename(src, dest);
+ return "rename";
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code !== "EXDEV") throw err;
+ await fsOps.cp(src, dest, { recursive: true, verbatimSymlinks: true, preserveTimestamps: true });
+ await fsOps.rm(src, { recursive: true, force: true });
+ return "copy";
+ }
+}
+
+/**
+ * A crash between installBundle's two renames leaves `out.prev` and no `out`:
+ * the last bundle is whole, under the wrong name. Put it back. Asked first by
+ * every build and every install, before anything else touches the target.
+ * Answers whether it restored one.
+ */
+export async function recoverInterruptedInstall(
+ destOut: string,
+ fsOps: BundleFs = realBundleFs,
+): Promise<boolean> {
+ const prev = `${destOut}.prev`;
+ if ((await lexists(destOut)) || !(await lexists(prev))) return false;
+ await fsOps.rename(prev, destOut);
+ return true;
+}
+
+/**
+ * Install the build at `src` as the bundle `destOut` (see the section header):
+ * `src` is gone afterwards and `destOut` holds it. Answers how it moved.
+ */
+export async function installBundle(
+ src: string,
+ destOut: string,
+ fsOps: BundleFs = realBundleFs,
+): Promise<"rename" | "copy"> {
+ const next = `${destOut}.next`;
+ const prev = `${destOut}.prev`;
+ await recoverInterruptedInstall(destOut, fsOps);
+ await fsOps.mkdir(path.dirname(destOut), { recursive: true });
+ await fsOps.rm(next, { recursive: true, force: true });
+ const how = await moveDir(src, next, fsOps);
+ await fsOps.rm(prev, { recursive: true, force: true });
+ // Siblings: these two renames never cross a filesystem.
+ if (await lexists(destOut)) await fsOps.rename(destOut, prev);
+ await fsOps.rename(next, destOut);
+ await fsOps.rm(prev, { recursive: true, force: true });
+ return how;
+}
+
+/**
+ * Before a build: export/out must not still point at an older bundle, or a
+ * build that failed before `next build` reached it would leave that bundle
+ * readable at export/out. A link is removed; a real directory is left for
+ * `next build` to replace.
+ */
+export async function unlinkExportOut(paths: Pick<Paths, "exportDir">): Promise<void> {
+ const link = exportOutPath(paths);
+ const st = await lstat(link).catch(() => null);
+ if (st?.isSymbolicLink()) await rm(link, { force: true });
+}
+
+/** After a build: export/out becomes a (relative) link to `bundle`, replaced atomically. */
+export async function pointExportOutAt(paths: Pick<Paths, "exportDir">, bundle: string): Promise<void> {
+ const link = exportOutPath(paths);
+ const st = await lstat(link).catch(() => null);
+ if (st && !st.isSymbolicLink()) await rm(link, { recursive: true, force: true });
+ const tmp = `${link}.link-${process.pid}`;
+ await rm(tmp, { force: true });
+ await symlink(path.relative(path.dirname(link), bundle), tmp, "dir");
+ await rename(tmp, link);
+}
+
+/**
+ * The host compose stages a site's oversize archives beside export/public
+ * (`.r2-staging/<id>/archives`); the bundle's are `dockerSiteStagingDir` for
+ * both runners. Moves this build's (replacing the last build's) and answers
+ * how many were staged. A no-op where the two are one place.
+ */
+export async function stageSiteArchives(
+ paths: Pick<Paths, "exportPublicDir" | "exportBuildsDir">,
+ siteId: string,
+ fsOps: BundleFs = realBundleFs,
+): Promise<number> {
+ const src = path.join(path.dirname(paths.exportPublicDir), ".r2-staging", siteId);
+ const dest = path.join(paths.exportBuildsDir, siteId, ".r2-staging", siteId);
+ if (path.resolve(src) !== path.resolve(dest)) {
+ await fsOps.rm(dest, { recursive: true, force: true });
+ if (await lexists(src)) {
+ await fsOps.mkdir(path.dirname(dest), { recursive: true });
+ await moveDir(src, dest, fsOps);
+ }
+ }
+ const staged = await readdir(dockerSiteStagingDir(paths as Paths, siteId)).catch(() => [] as string[]);
+ return staged.filter((f) => !f.startsWith(".")).length;
+}
+
+/** Files and bytes under `dir` (links not followed). */
+export async function bundleCounts(dir: string): Promise<{ files: number; bytes: number }> {
+ let files = 0;
+ let bytes = 0;
+ const walk = async (d: string): Promise<void> => {
+ const ents = await readdir(d, { withFileTypes: true }).catch(() => []);
+ for (const e of ents) {
+ const p = path.join(d, e.name);
+ if (e.isDirectory()) await walk(p);
+ else if (e.isFile()) {
+ files++;
+ bytes += (await stat(p)).size;
+ }
+ }
+ };
+ await walk(dir);
+ return { files, bytes };
+}
+
+/** The bundle's corpus.json `generatedAt`, or null. */
+export async function corpusGeneratedAtIn(outDir: string): Promise<string | null> {
+ try {
+ const v = JSON.parse(await readFile(path.join(outDir, "corpus.json"), "utf8")) as { generatedAt?: unknown };
+ return typeof v.generatedAt === "string" ? v.generatedAt : null;
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * Build one site and install it as its bundle (the build-site stage's local
+ * body): export/out unlinked, buildSiteSteps with `skipData` (the index stage
+ * ran the data phase), the cited/scope checks of runBuildPhase,
+ * builtBundleProblem, then the install, the archive staging and the link.
+ * Returns the exit code; on 0 the bundle is `bundleDir(paths, siteId)`.
+ *
+ * `inPlace` (the docker per-site container) stops after the checks: the
+ * container hands export/out back itself, and the host stamps it.
+ */
+export async function buildSiteBundle(
+ siteId: string,
+ opts: PublishOpts & { skipArchives?: boolean; allowMissingMedia?: boolean; inPlace?: boolean } = {},
+): Promise<{ code: number; archivesStaged: number }> {
+ const { paths, onLog, signal } = resolved(opts);
+ if (!opts.inPlace) {
+ if (await recoverInterruptedInstall(bundleDir(paths, siteId))) {
+ onLog(`[build] ${siteId}: restored the bundle an interrupted install left as out.prev\n`);
+ }
+ await unlinkExportOut(paths);
+ }
+ const code = await runBuildPhase(onLog, signal, siteId, paths, {
+ skipData: true,
+ skipArchives: opts.skipArchives,
+ allowMissingMedia: opts.allowMissingMedia,
+ });
+ if (code !== 0 || signal.aborted) return { code: code || 1, archivesStaged: 0 };
+ const out = exportOutPath(paths);
+ const problem = builtBundleProblem(out, siteId);
+ if (problem) {
+ onLog(`[build] REFUSED — ${problem}.\n`);
+ return { code: 1, archivesStaged: 0 };
+ }
+ if (opts.inPlace) return { code: 0, archivesStaged: 0 };
+ // The old stamp must never describe the new bundle: it goes first, and the
+ // stage writes the new one as its LAST step, once the bundle is in place.
+ await rm(builtStampPath(paths, siteId), { force: true });
+ const how = await installBundle(out, bundleDir(paths, siteId));
+ const archivesStaged = await stageSiteArchives(paths, siteId);
+ await pointExportOutAt(paths, bundleDir(paths, siteId));
+ onLog(
+ `[build] ${siteId}: bundle installed at ${bundleDir(paths, siteId)} (${how === "rename" ? "moved" : "copied across filesystems"})` +
+ (archivesStaged ? `, ${archivesStaged} archive(s) staged for R2` : "") +
+ "\n",
+ );
+ return { code: 0, archivesStaged };
+}
+
+/**
+ * Build the hub and install it as `_hub/out` (the build-hub stage's body).
+ * Returns the exit code.
+ */
+export async function buildHubBundle(opts: PublishOpts = {}): Promise<number> {
+ const { paths, onLog, signal } = resolved(opts);
+ if (await recoverInterruptedInstall(bundleDir(paths, "_hub"))) {
+ onLog("[build] hub: restored the bundle an interrupted install left as out.prev\n");
+ }
+ await unlinkExportOut(paths);
+ const code = await buildHub({ paths, onLog, signal });
+ if (code !== 0 || signal.aborted) return code || 1;
+ const out = exportOutPath(paths);
+ const problem = builtHubProblem(out);
+ if (problem) {
+ onLog(`[build] REFUSED — ${problem}.\n`);
+ return 1;
+ }
+ const dest = bundleDir(paths, "_hub");
+ await rm(builtStampPath(paths, "_hub"), { force: true });
+ const how = await installBundle(out, dest);
+ await pointExportOutAt(paths, dest);
+ onLog(`[build] hub: bundle installed at ${dest} (${how === "rename" ? "moved" : "copied across filesystems"})\n`);
+ return 0;
+}
diff --git a/common/publish/bundle.test.ts b/common/publish/bundle.test.ts
@@ -0,0 +1,257 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ existsSync,
+ lstatSync,
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ readlinkSync,
+ rmSync,
+ symlinkSync,
+ writeFileSync,
+} from "node:fs";
+import { cp, mkdir, rename, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import {
+ bundleCounts,
+ bundleDir,
+ corpusGeneratedAtIn,
+ dockerSiteOutDir,
+ exportOutPath,
+ installBundle,
+ pointExportOutAt,
+ recoverInterruptedInstall,
+ stageSiteArchives,
+ unlinkExportOut,
+ type BundleFs,
+} from "./build";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common test
+//
+// The per-target bundle layout (release 18): a build in export/out is
+// installed as <exportBuildsDir>/<target>/out by a swap through out.next, the
+// move falls back to a copy across filesystems (an injected EXDEV here), and
+// export/out becomes a link to the bundle built last.
+
+function tmp(): { root: string; paths: Paths } {
+ const root = mkdtempSync(path.join(os.tmpdir(), "bundle-"));
+ const exportDir = path.join(root, "export");
+ mkdirSync(exportDir, { recursive: true });
+ return {
+ root,
+ paths: {
+ exportDir,
+ exportPublicDir: path.join(exportDir, "public"),
+ exportBuildsDir: path.join(exportDir, ".export-builds"),
+ } as Paths,
+ };
+}
+
+function writeBuild(dir: string, marker: string): void {
+ mkdirSync(path.join(dir, "_next"), { recursive: true });
+ writeFileSync(path.join(dir, "index.html"), marker);
+ writeFileSync(path.join(dir, "_next", "app.js"), "x".repeat(10));
+ writeFileSync(path.join(dir, "corpus.json"), JSON.stringify({ generatedAt: `at-${marker}` }));
+}
+
+// A filesystem whose renames out of `crossFrom` fail EXDEV, as export/out's do
+// in the container (an image layer) when the builds dir is a volume.
+function exdevFs(crossFrom: string, calls: string[]): BundleFs {
+ return {
+ rename: (async (a: string, b: string) => {
+ calls.push(`rename ${path.basename(String(a))} -> ${path.basename(String(b))}`);
+ if (String(a).startsWith(crossFrom)) {
+ throw Object.assign(new Error("EXDEV: cross-device link not permitted"), { code: "EXDEV" });
+ }
+ return rename(a, b);
+ }) as typeof rename,
+ cp: (async (a: string, b: string, o?: object) => {
+ calls.push(`cp ${path.basename(String(a))} -> ${path.basename(String(b))}`);
+ return cp(a, b, o);
+ }) as typeof cp,
+ rm,
+ mkdir,
+ };
+}
+
+test("bundleDir is <exportBuildsDir>/<target>/out — a site's is dockerSiteOutDir", () => {
+ const p = { exportBuildsDir: "/e/.export-builds", exportDir: "/e" } as Paths;
+ assert.equal(bundleDir(p, "jer"), "/e/.export-builds/jer/out");
+ assert.equal(bundleDir(p, "jer"), dockerSiteOutDir(p, "jer"));
+ assert.equal(bundleDir(p, "_hub"), "/e/.export-builds/_hub/out");
+ assert.equal(exportOutPath(p), "/e/out");
+});
+
+test("installBundle moves the build in through out.next, replacing the last bundle; a leftover out.next goes first", async () => {
+ const { root, paths } = tmp();
+ try {
+ const dest = bundleDir(paths, "jer");
+ writeBuild(dest, "old");
+ writeBuild(`${dest}.next`, "leftover");
+ const out = exportOutPath(paths);
+ writeBuild(out, "new");
+ assert.equal(await installBundle(out, dest), "rename");
+ assert.equal(readFileSync(path.join(dest, "index.html"), "utf8"), "new");
+ assert.ok(!existsSync(out), "export/out moved away");
+ assert.ok(!existsSync(`${dest}.next`));
+ assert.ok(!existsSync(`${dest}.prev`));
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("installBundle copies across filesystems (EXDEV), then removes the source", async () => {
+ const { root, paths } = tmp();
+ try {
+ const dest = bundleDir(paths, "jer");
+ writeBuild(dest, "old");
+ const out = exportOutPath(paths);
+ writeBuild(out, "new");
+ const calls: string[] = [];
+ assert.equal(await installBundle(out, dest, exdevFs(out, calls)), "copy");
+ assert.equal(readFileSync(path.join(dest, "index.html"), "utf8"), "new");
+ assert.equal(readFileSync(path.join(dest, "_next", "app.js"), "utf8"), "x".repeat(10));
+ assert.ok(!existsSync(out), "the source is removed after the copy");
+ assert.deepEqual(calls, [
+ "rename out -> out.next", // EXDEV
+ "cp out -> out.next",
+ "rename out -> out.prev", // siblings: same filesystem
+ "rename out.next -> out",
+ ]);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("export/out becomes a relative link to the bundle; unlinkExportOut removes a link, never a real build", async () => {
+ const { root, paths } = tmp();
+ try {
+ const out = exportOutPath(paths);
+ writeBuild(out, "real");
+ await unlinkExportOut(paths);
+ assert.ok(lstatSync(out).isDirectory(), "a real out/ is left for next build");
+ const dest = bundleDir(paths, "jer");
+ await installBundle(out, dest);
+ await pointExportOutAt(paths, dest);
+ assert.ok(lstatSync(out).isSymbolicLink());
+ assert.equal(readlinkSync(out), path.join(".export-builds", "jer", "out"));
+ assert.equal(readFileSync(path.join(out, "index.html"), "utf8"), "real", "reads through the link");
+ // Pointing it at another bundle replaces the link, not the bundle.
+ const hub = bundleDir(paths, "_hub");
+ writeBuild(hub, "hub");
+ await pointExportOutAt(paths, hub);
+ assert.equal(readFileSync(path.join(out, "index.html"), "utf8"), "hub");
+ assert.ok(existsSync(path.join(dest, "index.html")), "the other bundle is untouched");
+ await unlinkExportOut(paths);
+ assert.ok(!existsSync(out));
+ assert.ok(existsSync(path.join(hub, "index.html")), "the link's target survives");
+ // A real out/ left behind is replaced by the link.
+ writeBuild(out, "stray");
+ await pointExportOutAt(paths, dest);
+ assert.ok(lstatSync(out).isSymbolicLink());
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("stageSiteArchives moves the host staging into the bundle's (dockerSiteStagingDir) and counts it", async () => {
+ const { root, paths } = tmp();
+ try {
+ const host = path.join(paths.exportDir, ".r2-staging", "jer", "archives");
+ mkdirSync(host, { recursive: true });
+ writeFileSync(path.join(host, "a.zip"), "a");
+ writeFileSync(path.join(host, "b.zip"), "b");
+ writeFileSync(path.join(host, ".hidden"), "");
+ const bundleStaging = path.join(paths.exportBuildsDir, "jer", ".r2-staging", "jer", "archives");
+ mkdirSync(bundleStaging, { recursive: true });
+ writeFileSync(path.join(bundleStaging, "last-build.zip"), "old");
+ assert.equal(await stageSiteArchives(paths, "jer"), 2);
+ assert.ok(existsSync(path.join(bundleStaging, "a.zip")));
+ assert.ok(!existsSync(path.join(bundleStaging, "last-build.zip")), "the last build's staging is replaced");
+ assert.ok(!existsSync(host));
+ // Nothing staged this build: the bundle has none either.
+ assert.equal(await stageSiteArchives(paths, "jer"), 0);
+ assert.ok(!existsSync(bundleStaging));
+ // Across filesystems.
+ mkdirSync(host, { recursive: true });
+ writeFileSync(path.join(host, "c.zip"), "c");
+ const calls: string[] = [];
+ assert.equal(await stageSiteArchives(paths, "jer", exdevFs(path.join(paths.exportDir, ".r2-staging"), calls)), 1);
+ assert.ok(calls.some((c) => c.startsWith("cp ")));
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("stageSiteArchives is a no-op where the two are one place (the build container's compose)", async () => {
+ const root = mkdtempSync(path.join(os.tmpdir(), "bundle-same-"));
+ try {
+ // EXPORT_PUBLIC_DIR=/site/public and a builds dir whose <id> IS /site.
+ const site = path.join(root, "jer");
+ const paths = { exportPublicDir: path.join(site, "public"), exportBuildsDir: root } as Paths;
+ const staged = path.join(site, ".r2-staging", "jer", "archives");
+ mkdirSync(staged, { recursive: true });
+ writeFileSync(path.join(staged, "a.zip"), "a");
+ assert.equal(await stageSiteArchives(paths, "jer"), 1);
+ assert.ok(existsSync(path.join(staged, "a.zip")));
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("bundleCounts and corpusGeneratedAtIn read the bundle; links are not followed", async () => {
+ const { root } = tmp();
+ try {
+ const dir = path.join(root, "b");
+ writeBuild(dir, "m");
+ symlinkSync("/etc", path.join(dir, "link"));
+ const c = await bundleCounts(dir);
+ assert.equal(c.files, 3);
+ assert.equal(c.bytes, 1 + 10 + JSON.stringify({ generatedAt: "at-m" }).length);
+ assert.equal(await corpusGeneratedAtIn(dir), "at-m");
+ assert.equal(await corpusGeneratedAtIn(path.join(root, "none")), null);
+ assert.deepEqual(await bundleCounts(path.join(root, "none")), { files: 0, bytes: 0 });
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("a crash between the two renames leaves out.prev and no out; the next build or install restores it first", async () => {
+ const { root, paths } = tmp();
+ try {
+ const dest = bundleDir(paths, "jer");
+ writeBuild(dest, "old");
+ const out = exportOutPath(paths);
+ writeBuild(out, "new");
+ // The process dies on the second rename (out.next -> out).
+ const dying: BundleFs = {
+ rename: (async (a: string, b: string) => {
+ if (String(a).endsWith("out.next") && String(b).endsWith(path.join("jer", "out"))) throw new Error("killed");
+ return rename(a, b);
+ }) as typeof rename,
+ cp,
+ rm,
+ mkdir,
+ };
+ await assert.rejects(installBundle(out, dest, dying), /killed/);
+ assert.ok(!existsSync(dest), "no out");
+ assert.ok(existsSync(`${dest}.prev`), "the last bundle, under the wrong name");
+ // Nothing to do when out is there; the old bundle back when it is not.
+ assert.equal(await recoverInterruptedInstall(dest), true);
+ assert.equal(readFileSync(path.join(dest, "index.html"), "utf8"), "old");
+ assert.equal(await recoverInterruptedInstall(dest), false);
+ // And an install over the same crash recovers before it swaps.
+ rmSync(`${dest}.next`, { recursive: true, force: true });
+ await rename(dest, `${dest}.prev`);
+ writeBuild(out, "newer");
+ await installBundle(out, dest);
+ assert.equal(readFileSync(path.join(dest, "index.html"), "utf8"), "newer");
+ assert.ok(!existsSync(`${dest}.prev`));
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
diff --git a/common/publish/inputSig.test.ts b/common/publish/inputSig.test.ts
@@ -0,0 +1,215 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import type { ChannelConfig } from "../lib/channelConfig";
+import type { Paths } from "../lib/paths";
+import type { Site } from "../lib/site";
+import { hubInputSig, siteInputSig } from "./inputSig";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common test
+//
+// A site's inputSig moves exactly with what compose reads (release 18): its
+// staged index tree, its PUBLISHED members' shared trees (manifest.json aside,
+// a manifest-only tree signed as one), site.json, the corpus-wide files and the
+// two settings — and not with another site's channel, a churned manifest, or
+// a rewritten-but-identical chart-templates.json.
+
+function setup() {
+ const root = mkdtempSync(path.join(os.tmpdir(), "input-sig-"));
+ const idx = path.join(root, ".export-index");
+ const paths = {
+ transcriptsDir: path.join(root, "transcripts"),
+ sitesDir: path.join(root, "transcripts", "sites"),
+ exportSitesIndexDir: path.join(idx, "sites"),
+ exportSharedTranscriptsDir: path.join(idx, "shared", "transcripts"),
+ exportSharedSubsDir: path.join(idx, "shared", "subs"),
+ exportSharedPostsDir: path.join(idx, "shared", "posts"),
+ exportSharedDigestsDir: path.join(idx, "shared", "digests"),
+ globalAliasesFile: path.join(root, "transcripts", "search-aliases.json"),
+ globalTagsFile: path.join(root, "transcripts", "tags.json"),
+ homepageConfigFile: path.join(root, "transcripts", "sites", "_homepage", "homepage.json"),
+ } as Paths;
+ const site = {
+ siteId: "jer",
+ siteTitle: "Jer",
+ channels: [{ slug: "a" }, { slug: "x" }],
+ } as unknown as Site;
+ const write = (file: string, body: string) => {
+ mkdirSync(path.dirname(file), { recursive: true });
+ writeFileSync(file, body);
+ };
+ write(path.join(paths.sitesDir, "jer", "site.json"), JSON.stringify({ siteId: "jer" }));
+ write(path.join(paths.exportSitesIndexDir, "jer", "summaries", "page-0000.json"), "[]");
+ write(path.join(paths.exportSitesIndexDir, "jer", "chart-templates.json"), '{"v":1}');
+ write(path.join(paths.exportSharedTranscriptsDir, "a", "page-0000.json"), "[1]");
+ write(path.join(paths.exportSharedTranscriptsDir, "a", "manifest.json"), '{"generatedAt":"1"}');
+ write(path.join(paths.exportSharedPostsDir, "x", "page-0000.json"), "[2]");
+ write(path.join(paths.exportSharedTranscriptsDir, "other", "page-0000.json"), "[3]");
+ const configs: Record<string, Partial<ChannelConfig>> = {
+ a: {},
+ x: { sourceKind: "social", platform: "twitter" } as Partial<ChannelConfig>,
+ };
+ let settings: Record<string, unknown> = { socialLinks: [], buildArchives: true };
+ let sites: Site[] = [
+ site,
+ { siteId: "ani", siteTitle: "Ani", siteUrl: "https://ani.pages.dev", channels: [] } as unknown as Site,
+ ];
+ const sig = () =>
+ siteInputSig({
+ paths,
+ site,
+ settings: settings as never,
+ sites,
+ configOf: (slug) => configs[slug] as ChannelConfig,
+ });
+ const bump = (file: string) => {
+ const t = new Date(Date.now() + 5_000);
+ utimesSync(file, t, t);
+ };
+ return {
+ root,
+ paths,
+ write,
+ sig,
+ bump,
+ setXPrivate: () => {
+ settings = { ...settings, social: { x: { visibility: "private" } } };
+ },
+ setSettings: (over: Record<string, unknown>) => {
+ settings = { ...settings, ...over };
+ },
+ setSites: (next: Site[]) => {
+ sites = next;
+ },
+ site,
+ };
+}
+
+test("inputSig is stable, and ignores what compose ignores", async () => {
+ const s = setup();
+ try {
+ const base = await s.sig();
+ assert.match(base, /^[0-9a-f]{40}$/);
+ assert.equal(await s.sig(), base, "nothing changed");
+ // A churned shared manifest (generatedAt) does not count…
+ s.write(path.join(s.paths.exportSharedTranscriptsDir, "a", "manifest.json"), '{"generatedAt":"2"}');
+ assert.equal(await s.sig(), base);
+ // …nor another site's channel…
+ s.write(path.join(s.paths.exportSharedTranscriptsDir, "other", "page-0001.json"), "[4]");
+ assert.equal(await s.sig(), base);
+ // …nor chart-templates.json rewritten with the same bytes.
+ const tpl = path.join(s.paths.exportSitesIndexDir, "jer", "chart-templates.json");
+ s.bump(tpl);
+ assert.equal(await s.sig(), base);
+ } finally {
+ rmSync(s.root, { recursive: true, force: true });
+ }
+});
+
+test("inputSig moves with each input compose reads", async () => {
+ const s = setup();
+ try {
+ let last = await s.sig();
+ const moved = async (what: string) => {
+ const now = await s.sig();
+ assert.notEqual(now, last, what);
+ last = now;
+ };
+ s.write(path.join(s.paths.exportSharedTranscriptsDir, "a", "page-0001.json"), "[5]");
+ await moved("a member's shared transcripts");
+ s.write(path.join(s.paths.exportSharedPostsDir, "x", "page-0001.json"), "[6]");
+ await moved("a member's shared posts");
+ s.write(path.join(s.paths.exportSitesIndexDir, "jer", "summaries", "page-0001.json"), "[]");
+ await moved("the site's staged summaries");
+ s.write(path.join(s.paths.exportSitesIndexDir, "jer", "chart-templates.json"), '{"v":2}');
+ await moved("the chart templates' bytes");
+ s.write(path.join(s.paths.sitesDir, "jer", "site.json"), JSON.stringify({ siteId: "jer", t: 1 }));
+ await moved("site.json");
+ s.write(path.join(s.paths.sitesDir, "jer", "tags.json"), "{}");
+ await moved("the site's config dir");
+ s.write(s.paths.globalAliasesFile, "{}");
+ await moved("the global search aliases");
+ s.write(s.paths.globalTagsFile, "{}");
+ await moved("the curated tags");
+ s.write(path.join(s.paths.transcriptsDir, "duplicates.json"), "{}");
+ await moved("the duplicates report");
+ // X posts private: the X member is withheld from a public site.
+ s.setXPrivate();
+ await moved("social.x.visibility");
+ s.write(path.join(s.paths.exportSharedPostsDir, "x", "page-0002.json"), "[7]");
+ assert.equal(await s.sig(), last, "a withheld member's tree is not this site's input");
+ } finally {
+ rmSync(s.root, { recursive: true, force: true });
+ }
+});
+
+test("inputSig moves with what the export BUILD renders: social links, the hub url, buildArchives, the sibling sites", async () => {
+ const s = setup();
+ try {
+ let last = await s.sig();
+ const moved = async (what: string) => {
+ const now = await s.sig();
+ assert.notEqual(now, last, what);
+ last = now;
+ };
+ s.setSettings({ socialLinks: [{ label: "X", url: "https://x.com/a" }] });
+ await moved("settings.socialLinks (the header/footer links)");
+ s.setSettings({ homepageUrl: "https://hub.pages.dev" });
+ await moved("settings.homepageUrl (the site's hubUrl)");
+ s.setSettings({ buildArchives: false });
+ await moved("settings.buildArchives");
+ const ani = { siteId: "ani", siteTitle: "Ani", siteUrl: "https://ani.pages.dev", channels: [] };
+ s.setSites([s.site, { ...ani, siteTitle: "Ani 2" } as unknown as Site]);
+ await moved("a sibling's title");
+ s.setSites([s.site, { ...ani, siteTitle: "Ani 2", siteUrl: "https://ani2.pages.dev" } as unknown as Site]);
+ await moved("a sibling's url");
+ s.setSites([s.site, { ...ani, siteTitle: "Ani 2", siteUrl: "https://ani2.pages.dev" } as unknown as Site, {
+ siteId: "bon", siteTitle: "Bon", siteUrl: "https://bon.pages.dev", channels: [],
+ } as unknown as Site]);
+ await moved("a sibling launched");
+ // A sibling the footer does not list (no url) changes nothing.
+ s.setSites([s.site, { ...ani, siteTitle: "Ani 2", siteUrl: "https://ani2.pages.dev" } as unknown as Site, {
+ siteId: "bon", siteTitle: "Bon", siteUrl: "https://bon.pages.dev", channels: [],
+ } as unknown as Site, { siteId: "dark", siteTitle: "Dark", channels: [] } as unknown as Site]);
+ assert.equal(await s.sig(), last, "an unlinkable sibling is not in the footer");
+ } finally {
+ rmSync(s.root, { recursive: true, force: true });
+ }
+});
+
+test("a manifest-only member tree is signed as one, not as absent", async () => {
+ const s = setup();
+ try {
+ const before = await s.sig();
+ s.write(path.join(s.paths.exportSharedSubsDir, "a", "manifest.json"), "{}");
+ assert.notEqual(await s.sig(), before);
+ } finally {
+ rmSync(s.root, { recursive: true, force: true });
+ }
+});
+
+test("hubSig: the stamp, homepage.json, and each listed site's id + url + title", async () => {
+ const s = setup();
+ try {
+ const sites = [
+ { siteId: "jer", siteTitle: "Jer", siteUrl: "https://jer.pages.dev", channels: [] },
+ { siteId: "mine", siteTitle: "Mine", siteUrl: "https://m.pages.dev", audience: "private", channels: [] },
+ { siteId: "nourl", siteTitle: "No", channels: [] },
+ ] as unknown as Site[];
+ const a = await hubInputSig(s.paths, "s1", sites);
+ assert.equal(await hubInputSig(s.paths, "s1", sites), a);
+ assert.notEqual(await hubInputSig(s.paths, "s2", sites), a, "a new index stamp");
+ const retitled = sites.map((x) => (x.siteId === "jer" ? { ...x, siteTitle: "Jer 2" } : x));
+ assert.notEqual(await hubInputSig(s.paths, "s1", retitled), a);
+ // A private site or one with no url is not listed: changing it moves nothing.
+ const privRetitled = sites.map((x) => (x.siteId === "mine" ? { ...x, siteTitle: "M2" } : x));
+ assert.equal(await hubInputSig(s.paths, "s1", privRetitled), a);
+ s.write(s.paths.homepageConfigFile, '{"title":"hub"}');
+ assert.notEqual(await hubInputSig(s.paths, "s1", sites), a);
+ } finally {
+ rmSync(s.root, { recursive: true, force: true });
+ }
+});
diff --git a/common/publish/inputSig.ts b/common/publish/inputSig.ts
@@ -0,0 +1,222 @@
+// What a site's (and the hub's) build reads, as one sha1 — computed by the
+// update-index stage and stored in the IndexStamp (release 18).
+//
+// THE RULE: a site is fresh exactly when compose AND the export build would
+// produce the same bundle. So the signature is made of what they read, signed
+// with compose's own `dirSignature` (lib/dirSignature.ts) and compose's own
+// manifest-only rule:
+//
+// - the site's whole `.export-index/sites/<id>/` tree (summaries, stats,
+// chart templates, tag counts, the subs/posts/digests manifests, the
+// report media and exports);
+// - each PUBLISHED member channel's shared transcripts / subs / posts /
+// digests tree, `manifest.json` ignored (its `generatedAt` churns);
+// - the bytes of `site.json`, and the signature of the site's config dir
+// (`sites/<id>/`: tags, aliases, reports);
+// - the corpus-wide files compose reads at compose time (search aliases,
+// curated tags, the duplicates report and its overrides), by size + mtime;
+// - the `archiveStorage`, `social.x.visibility` and `buildArchives` settings;
+// - what `next build` renders beyond compose (release 18 S1 review): the
+// resolved social links, the hub url, and the footer's sibling sites.
+//
+// A superset of those inputs is CONSERVATIVE: a needless rebuild, never a
+// wrong skip. `hubSig` = sha1(stampId, homepage.json, each listed
+// site's id + siteUrl + title).
+
+import { createHash } from "node:crypto";
+import { existsSync } from "node:fs";
+import { readFile, stat } from "node:fs/promises";
+import path from "node:path";
+import { open } from "lmdb";
+import { dirSignature } from "../lib/dirSignature";
+import { DUPLICATES_FILENAME, DUPLICATE_OVERRIDES_FILENAME } from "../lib/duplicates";
+import type { Paths } from "../lib/paths";
+import { publishedMemberSlugs } from "../lib/postsVisibility";
+import type { SiteSettings } from "../lib/settings";
+import {
+ resolveHubUrl,
+ resolveRelatedSites,
+ resolveSocialLinks,
+ siteConfigFile,
+ siteDir,
+ siteIndexDir,
+ type Site,
+} from "../lib/site";
+import { isListedSite } from "../lib/siteSchema";
+import { INDEX_SCANNED_AT_KEY } from "../lib/stats";
+import { readChannelConfig } from "../controller/channels";
+import type { ChannelConfig } from "../lib/channelConfig";
+
+// compose-site.ts MANIFEST_ONLY_SIGNATURE — a tree holding only its manifest is
+// a channel with nothing in it, signed by a constant (compose's rule).
+const MANIFEST_ONLY = "manifest-only";
+// lib/chartsStore.ts siteTemplatesStagingPath's basename.
+const CHART_TEMPLATES = "chart-templates.json";
+
+const sha1 = (s: string | Buffer) => createHash("sha1").update(s).digest("hex");
+
+async function fileBytesSig(file: string): Promise<string> {
+ try {
+ return sha1(await readFile(file));
+ } catch {
+ return "";
+ }
+}
+
+async function fileStatSig(file: string): Promise<string> {
+ try {
+ const s = await stat(file);
+ return `${s.size}\t${s.mtimeMs}`;
+ } catch {
+ return "";
+ }
+}
+
+/** A memo over the shared per-channel trees: each is walked once per stamp. */
+export type TreeSigCache = Map<string, string>;
+
+async function channelTreeSig(root: string, slug: string, cache: TreeSigCache): Promise<string> {
+ const dir = path.join(root, slug);
+ const hit = cache.get(dir);
+ if (hit !== undefined) return hit;
+ let sig = await dirSignature(dir, "manifest.json");
+ if (sig === "" && existsSync(path.join(dir, "manifest.json"))) sig = MANIFEST_ONLY;
+ cache.set(dir, sig);
+ return sig;
+}
+
+export type SiteSigInputs = {
+ paths: Paths;
+ site: Site;
+ settings: Pick<SiteSettings, "archiveStorage" | "social" | "socialLinks" | "homepageUrl" | "buildArchives">;
+ // Every configured site: the footer's sibling list is part of the bundle.
+ sites: Site[];
+ // The site's members' configs (the visibility rule reads their platform).
+ configOf: (slug: string) => ChannelConfig | null | undefined;
+ cache?: TreeSigCache;
+};
+
+/** The site's inputSig (see the header). */
+export async function siteInputSig(i: SiteSigInputs): Promise<string> {
+ const { paths, site, settings } = i;
+ const cache = i.cache ?? new Map();
+ const members = [...publishedMemberSlugs(site, i.configOf, settings)].sort();
+ const lines: string[] = ["inputSig v1"];
+ lines.push(`members\t${members.join(",")}`);
+ // `build templates` rewrites chart-templates.json on every run (and compose
+ // copies it on every run): signed by its bytes, not its mtime.
+ const indexDir = siteIndexDir(paths, site.siteId);
+ lines.push(`site-index\t${await dirSignature(indexDir, CHART_TEMPLATES)}`);
+ lines.push(`${CHART_TEMPLATES}\t${await fileBytesSig(path.join(indexDir, CHART_TEMPLATES))}`);
+ for (const slug of members) {
+ for (const [tree, root] of [
+ ["transcripts", paths.exportSharedTranscriptsDir],
+ ["subs", paths.exportSharedSubsDir],
+ ["posts", paths.exportSharedPostsDir],
+ ["digests", paths.exportSharedDigestsDir],
+ ] as const) {
+ lines.push(`${tree}/${slug}\t${await channelTreeSig(root, slug, cache)}`);
+ }
+ }
+ lines.push(`site.json\t${await fileBytesSig(siteConfigFile(paths, site.siteId))}`);
+ lines.push(`site-dir\t${await dirSignature(siteDir(paths, site.siteId))}`);
+ for (const file of [
+ paths.globalAliasesFile,
+ paths.globalTagsFile,
+ path.join(paths.transcriptsDir, DUPLICATES_FILENAME),
+ path.join(paths.transcriptsDir, DUPLICATE_OVERRIDES_FILENAME),
+ ]) {
+ lines.push(`${path.basename(file)}\t${await fileStatSig(file)}`);
+ }
+ lines.push(
+ `settings\t${JSON.stringify({
+ archiveStorage: settings.archiveStorage ?? null,
+ xVisibility: settings.social?.x?.visibility ?? null,
+ buildArchives: settings.buildArchives ?? null,
+ })}`,
+ );
+ // What the export BUILD renders from beyond compose's inputs, resolved as it
+ // resolves them: the header/footer social links, the hub link the descriptor
+ // carries, and the footer's sibling sites (their urls, titles, listing).
+ const full = settings as SiteSettings;
+ lines.push(`social-links\t${JSON.stringify(resolveSocialLinks(site, full) ?? null)}`);
+ lines.push(`hub-url\t${resolveHubUrl(site, full) ?? ""}`);
+ lines.push(`related-sites\t${JSON.stringify(resolveRelatedSites(site, i.sites))}`);
+ return sha1(lines.join("\n"));
+}
+
+/** Every member's channel config, read once (unreadable = null). */
+export async function readMemberConfigs(
+ paths: Paths,
+ sites: Site[],
+): Promise<Map<string, ChannelConfig | null>> {
+ const slugs = new Set(sites.flatMap((s) => s.channels.map((c) => c.slug)));
+ const out = new Map<string, ChannelConfig | null>();
+ await Promise.all(
+ [...slugs].map(async (slug) => {
+ out.set(slug, await readChannelConfig(paths, slug).catch(() => null));
+ }),
+ );
+ return out;
+}
+
+/** The hub's signature: the index it was built from, its config, the pool it lists. */
+export async function hubInputSig(
+ paths: Paths,
+ stampId: string,
+ sites: Site[],
+): Promise<string> {
+ const lines = [`hubSig v1`, `stamp\t${stampId}`];
+ lines.push(`homepage.json\t${await fileBytesSig(paths.homepageConfigFile)}`);
+ for (const s of [...sites].sort((a, b) => a.siteId.localeCompare(b.siteId))) {
+ if (!isListedSite(s) || !s.siteUrl) continue;
+ lines.push(`site\t${s.siteId}\t${s.siteUrl}\t${s.siteTitle}`);
+ }
+ return sha1(lines.join("\n"));
+}
+
+export type IndexMeta = {
+ generation: number;
+ scannedAt: number | null;
+ // sha1 of each site's stored fingerprints, null when absent.
+ siteFp: Record<string, string | null>;
+ statsFp: Record<string, string | null>;
+};
+
+/**
+ * The LMDB index's own bookkeeping, read-only, after the index and stats
+ * builds have closed it: `generation`, INDEX_SCANNED_AT_KEY and each site's
+ * `siteFp:<id>` / `statsFp:<id>`. An absent index reads as zeros.
+ */
+export function readIndexMeta(paths: Paths, siteIds: string[]): IndexMeta {
+ const out: IndexMeta = { generation: 0, scannedAt: null, siteFp: {}, statsFp: {} };
+ for (const id of siteIds) {
+ out.siteFp[id] = null;
+ out.statsFp[id] = null;
+ }
+ if (!existsSync(paths.lmdbPath)) return out;
+ const root = open({ path: paths.lmdbPath, readOnly: true, maxDbs: 18, compression: true });
+ try {
+ const meta = root.openDB<unknown, string>({ name: "meta", encoding: "msgpack" });
+ // An index that stats never ran over has no statsMeta sub-DB.
+ let statsMeta: typeof meta | null = null;
+ try {
+ statsMeta = root.openDB<unknown, string>({ name: "statsMeta", encoding: "msgpack" });
+ } catch {
+ statsMeta = null;
+ }
+ const gen = meta.get("generation");
+ out.generation = typeof gen === "number" ? gen : 0;
+ const scanned = meta.get(INDEX_SCANNED_AT_KEY);
+ out.scannedAt = typeof scanned === "number" ? scanned : null;
+ for (const id of siteIds) {
+ const fp = meta.get(`siteFp:${id}`);
+ const sfp = statsMeta?.get(`statsFp:${id}`);
+ out.siteFp[id] = typeof fp === "string" ? sha1(fp) : null;
+ out.statsFp[id] = typeof sfp === "string" ? sha1(sfp) : null;
+ }
+ } finally {
+ root.close();
+ }
+ return out;
+}
diff --git a/common/publish/stageBodies.ts b/common/publish/stageBodies.ts
@@ -0,0 +1,736 @@
+// The publish stages' bodies (release 18): what `run()` does, in the stage
+// child the editor spawns (`archilyzer stage <kind> <target> …`) or in the
+// CLI's own process (`archilyzer publish …`), always under the publish lock
+// (stageRun.ts takes it).
+//
+// Every body but update-index first asks its stage's `needs()` over the state
+// ON DISK (`readNeedsInput`): blocked → StageFailure exit 3 (the precondition
+// is not met — "update the index first", "no build of X"), fresh and not
+// forced → a no-op. Ordering between the stages of one run is enforced here,
+// not in anybody's memory.
+//
+// The three deploy bodies call today's deploy functions in build.ts with the
+// target's own bundle; release 18 S2 rewires them to its deploy stage
+// (pinned wrangler, credential preflight, the live check).
+
+import { execFile } from "node:child_process";
+import { cp, mkdir, readdir, rm } from "node:fs/promises";
+import path from "node:path";
+import { promisify } from "node:util";
+import {
+ builtAudienceProblem,
+ builtBundleProblem,
+ builtHomepageProblem,
+ builtHubProblem,
+ siteDeployProblem,
+} from "../lib/builtExport";
+import { getHomepageConfig } from "../lib/homepage";
+import { previewAliasUrl, previewBranchProblem } from "../lib/pagesDeploy";
+import type { Paths } from "../lib/paths";
+import { getSettings } from "../lib/settings";
+import { getSite, listSites, type Site } from "../lib/site";
+import {
+ ALL_TARGET,
+ HOMEPAGE_TARGET,
+ HUB_TARGET,
+ imageBuildFacts,
+ newStampId,
+ readBuiltStamp,
+ readDeployedFile,
+ readIndexStamp,
+ recordDeploy,
+ writeBuiltStamp,
+ type BuiltKind,
+ type BuiltStamp,
+ type DeployRecord,
+ type IndexStamp,
+ type Runner,
+} from "./stamps";
+import {
+ STAGES,
+ deployKindOf,
+ type NeedsInput,
+ type StageContext,
+ type StageOutcome,
+ type StageRequest,
+ type TargetState,
+} from "./stages";
+
+/** A stage that did not run to the end: its exit code says why (stageRun.ts). */
+export class StageFailure extends Error {
+ constructor(
+ message: string,
+ readonly exitCode: 1 | 2 | 3,
+ ) {
+ super(message);
+ this.name = "StageFailure";
+ }
+}
+
+export class StageCancelled extends Error {
+ constructor() {
+ super("cancelled");
+ this.name = "StageCancelled";
+ }
+}
+
+function checkCancel(signal: AbortSignal): void {
+ if (signal.aborted) throw new StageCancelled();
+}
+
+// ---------------------------------------------------------------------------
+// git facts for the stamps
+// ---------------------------------------------------------------------------
+
+const exec = promisify(execFile);
+
+async function git(cwd: string, args: string[]): Promise<string | null> {
+ try {
+ const { stdout } = await exec("git", args, { cwd, timeout: 10_000 });
+ const out = stdout.trim();
+ return out || null;
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * The commit and branch a stamp records. `ARCHILYZER_COMMIT` /
+ * `ARCHILYZER_BRANCH`, when set, WIN over git, each on its own: the runtime
+ * image bakes them (it has no .git), and a test server sets
+ * `ARCHILYZER_BRANCH=main` because a worktree's branch is never `main`. Else
+ * the checkout's HEAD and branch; a detached HEAD records `branch: null`,
+ * which a production deploy refuses like any branch but `main`.
+ */
+export async function checkoutInfo(
+ paths: Pick<Paths, "monorepoRoot">,
+ env: NodeJS.ProcessEnv = process.env,
+): Promise<{ commit: string | null; branch: string | null }> {
+ const facts = imageBuildFacts(env);
+ const commit = facts.commit ?? (await git(paths.monorepoRoot, ["rev-parse", "HEAD"]));
+ if (facts.branch !== null) return { commit, branch: facts.branch };
+ const branch = await git(paths.monorepoRoot, ["rev-parse", "--abbrev-ref", "HEAD"]);
+ return { commit, branch: branch === null || branch === "HEAD" ? null : branch };
+}
+
+/** `main`'s HEAD where a repository is reachable, else null. */
+export async function mainHeadOf(paths: Pick<Paths, "monorepoRoot">): Promise<string | null> {
+ return git(paths.monorepoRoot, ["rev-parse", "--verify", "--quiet", "refs/heads/main^{commit}"]);
+}
+
+// ---------------------------------------------------------------------------
+// The state needs() reads, from disk alone
+// ---------------------------------------------------------------------------
+
+function sitePagesProblem(site: Site): string | null {
+ return site.cloudflareProject?.trim()
+ ? null
+ : `Site "${site.siteId}" has no Cloudflare Pages project configured`;
+}
+
+/**
+ * The NeedsInput a stage child can build from disk alone: the stamps and the
+ * bundles. It does not read job metas or config mtimes (`changedChannels` is
+ * empty, `configChangedAt` null): a child judges by signatures, and the
+ * signature in the index stamp is the authority once the index has run.
+ */
+export async function readNeedsInput(
+ paths: Paths,
+ opts: { mainHead?: boolean } = {},
+): Promise<NeedsInput> {
+ const { bundleDir, homepageOutDir, hubProjectProblem } = await import("./build");
+ const stamp = await readIndexStamp(paths);
+ const sites: Record<string, TargetState> = {};
+ for (const site of listSites(paths)) {
+ const built = await readBuiltStamp(paths, site.siteId);
+ sites[site.siteId] = {
+ built,
+ deployed: await readDeployedFile(paths, site.siteId),
+ changedChannels: [],
+ configChangedAt: null,
+ bundleProblem: built ? builtBundleProblem(bundleDir(paths, site.siteId), site.siteId) : null,
+ deployProblem: siteDeployProblem(site),
+ pagesProblem: sitePagesProblem(site),
+ };
+ }
+ const hubBuilt = await readBuiltStamp(paths, HUB_TARGET);
+ const homeBuilt = await readBuiltStamp(paths, HOMEPAGE_TARGET);
+ return {
+ index: { stamp, lastIngestDoneAt: null, configChangedAt: null },
+ sites,
+ hub: {
+ built: hubBuilt,
+ deployed: await readDeployedFile(paths, HUB_TARGET),
+ changedChannels: [],
+ configChangedAt: null,
+ bundleProblem: hubBuilt ? builtHubProblem(bundleDir(paths, HUB_TARGET)) : null,
+ pagesProblem: hubProjectProblem(getHomepageConfig(paths).cloudflareProject),
+ },
+ homepage: {
+ built: homeBuilt,
+ deployed: await readDeployedFile(paths, HOMEPAGE_TARGET),
+ changedChannels: [],
+ configChangedAt: null,
+ bundleProblem: homeBuilt ? builtHomepageProblem(homepageOutDir(paths)) : null,
+ mainHead: opts.mainHead ? await mainHeadOf(paths) : null,
+ },
+ };
+}
+
+// ---------------------------------------------------------------------------
+// Stamping a build
+// ---------------------------------------------------------------------------
+
+async function stampBuilt(
+ paths: Paths,
+ outDir: string,
+ s: {
+ target: string;
+ kind: BuiltKind;
+ indexStampId: string | null;
+ inputSig: string;
+ runner: Runner;
+ archivesStaged: number;
+ sourceCommit?: string | null;
+ },
+): Promise<BuiltStamp> {
+ const { bundleCounts, corpusGeneratedAtIn } = await import("./build");
+ const { commit, branch } = await checkoutInfo(paths);
+ const counts = await bundleCounts(outDir);
+ const built: BuiltStamp = {
+ v: 1,
+ stampId: newStampId(),
+ target: s.target,
+ kind: s.kind,
+ indexStampId: s.indexStampId,
+ inputSig: s.inputSig,
+ builtAt: Date.now(),
+ commit,
+ branch,
+ runner: s.runner,
+ audience: builtAudienceProblem(outDir) ? "private" : "public",
+ corpusGeneratedAt: await corpusGeneratedAtIn(outDir),
+ files: counts.files,
+ bytes: counts.bytes,
+ archivesStaged: s.archivesStaged,
+ ...(s.sourceCommit !== undefined ? { sourceCommit: s.sourceCommit } : {}),
+ };
+ await writeBuiltStamp(paths, built);
+ return built;
+}
+
+function plural(n: number, word: string): string {
+ return `${n} ${word}${n === 1 ? "" : "s"}`;
+}
+
+// ---------------------------------------------------------------------------
+// update-index
+// ---------------------------------------------------------------------------
+
+async function runUpdateIndex(ctx: StageContext): Promise<StageOutcome> {
+ const { paths, onLog, signal } = ctx;
+ const { settingsFromFile } = await import("../lib/settings");
+ const { applyHealthTimings } = await import("../lib/storageHealth");
+ const { buildIndex } = await import("../controller/buildIndex");
+ const { buildStats } = await import("../controller/buildStats");
+ const { syncTemplatesToExport } = await import("../lib/chartsStore");
+ const { hubInputSig, readIndexMeta, readMemberConfigs, siteInputSig } = await import("./inputSig");
+
+ // A CLI process has no health pass: the drive-health timings the builds'
+ // watchdog runs on are applied here, once (bin/build-index.ts does the same).
+ applyHealthTimings(settingsFromFile(paths.settingsFile).storage.health);
+ const log = (m: string) => onLog(m.endsWith("\n") ? m : `${m}\n`);
+
+ onLog("=== update-index: the LMDB index ===\n");
+ const idx = await buildIndex({ paths, onLog: log });
+ checkCancel(signal);
+ onLog("=== update-index: the stats datasets ===\n");
+ const st = await buildStats({ paths, onLog: log, signal });
+ checkCancel(signal);
+ onLog("=== update-index: chart templates ===\n");
+ const sites = listSites(paths);
+ for (const site of sites) syncTemplatesToExport(paths, site.siteId);
+ const templatesAt = Date.now();
+ checkCancel(signal);
+
+ onLog("=== update-index: signatures ===\n");
+ const meta = readIndexMeta(
+ paths,
+ sites.map((s) => s.siteId),
+ );
+ const settings = getSettings();
+ const configs = await readMemberConfigs(paths, sites);
+ const cache = new Map<string, string>();
+ const siteEntries: IndexStamp["sites"] = {};
+ for (const site of sites) {
+ siteEntries[site.siteId] = {
+ siteFp: meta.siteFp[site.siteId] ?? null,
+ statsFp: meta.statsFp[site.siteId] ?? null,
+ inputSig: await siteInputSig({
+ paths,
+ site,
+ settings,
+ sites,
+ configOf: (slug) => configs.get(slug),
+ cache,
+ }),
+ };
+ }
+
+ // The same index as the last stamp (nothing rebuilt, every signature the
+ // same) keeps its stamp id, so the builds made from it stay current.
+ const prev = await readIndexStamp(paths);
+ let stampId = newStampId();
+ let reused = false;
+ if (
+ prev &&
+ idx.shortCircuited &&
+ st.shortCircuited &&
+ prev.generation === meta.generation &&
+ sameSites(prev.sites, siteEntries) &&
+ (await hubInputSig(paths, prev.stampId, sites)) === prev.hubSig
+ ) {
+ stampId = prev.stampId;
+ reused = true;
+ }
+ const { commit } = await checkoutInfo(paths);
+ const stamp: IndexStamp = {
+ v: 1,
+ stampId,
+ generation: meta.generation,
+ scannedAt: meta.scannedAt ?? Date.now(),
+ builtAt: Date.now(),
+ templatesAt,
+ commit,
+ index: {
+ shortCircuited: idx.shortCircuited,
+ added: idx.added,
+ changed: idx.changed,
+ removed: idx.removed,
+ heldChannels: idx.heldChannels,
+ },
+ stats: {
+ shortCircuited: st.shortCircuited,
+ notIndexedYet: st.notIndexedYet,
+ notIndexable: st.notIndexable,
+ },
+ sites: siteEntries,
+ hubSig: await hubInputSig(paths, stampId, sites),
+ };
+ const { writeIndexStamp } = await import("./stamps");
+ await writeIndexStamp(paths, stamp);
+ const summary =
+ `index +${idx.added} ~${idx.changed} -${idx.removed}` +
+ (idx.heldChannels.length ? ` (held: ${idx.heldChannels.join(", ")})` : "") +
+ `; ${plural(sites.length, "site")} signed` +
+ (reused ? "; nothing changed — the stamp stands" : "");
+ return { status: reused ? "noop" : "ran", stamp: stampId, summary };
+}
+
+function sameSites(a: IndexStamp["sites"], b: IndexStamp["sites"]): boolean {
+ const ka = Object.keys(a).sort();
+ const kb = Object.keys(b).sort();
+ if (ka.join("\n") !== kb.join("\n")) return false;
+ return ka.every((k) => a[k].inputSig === b[k].inputSig);
+}
+
+// ---------------------------------------------------------------------------
+// build-site (one site, every site locally, every site in containers)
+// ---------------------------------------------------------------------------
+
+// Inside the docker per-site build container (docker/build-site.sh sets
+// ARCHIVES_READONLY=1): the build stays in export/out for the container to hand
+// back, and the HOST stamps it — nothing is installed or stamped here.
+function inBuildContainer(): boolean {
+ return process.env.ARCHIVES_READONLY === "1";
+}
+
+// What a container build reads where the host wrote no stamp (the editor's
+// Build all before release 18's surfaces): nothing is stamped in a container.
+const CONTAINER_NO_STAMP: IndexStamp = {
+ v: 1,
+ stampId: "",
+ generation: 0,
+ scannedAt: 0,
+ builtAt: 0,
+ templatesAt: 0,
+ commit: null,
+ index: { shortCircuited: true, added: 0, changed: 0, removed: 0, heldChannels: [] },
+ stats: { shortCircuited: true, notIndexedYet: 0, notIndexable: 0 },
+ sites: {},
+ hubSig: "",
+};
+
+async function buildOneSite(ctx: StageContext, r: StageRequest, stamp: IndexStamp): Promise<StageOutcome> {
+ const { paths, onLog, signal } = ctx;
+ const { buildSiteBundle, bundleDir } = await import("./build");
+ const siteId = r.target;
+ const inPlace = inBuildContainer();
+ const res = await buildSiteBundle(siteId, {
+ paths,
+ onLog,
+ signal,
+ skipArchives: r.skipArchives,
+ allowMissingMedia: r.allowMissingMedia,
+ inPlace,
+ });
+ checkCancel(signal);
+ if (res.code !== 0) throw new StageFailure(`build of ${siteId} failed (exit ${res.code})`, 1);
+ if (inPlace) return { status: "ran", stamp: "", summary: `${siteId} built in place (build container)` };
+ const built = await stampBuilt(paths, bundleDir(paths, siteId), {
+ target: siteId,
+ kind: "site",
+ indexStampId: stamp.stampId,
+ inputSig: stamp.sites[siteId].inputSig,
+ runner: "local",
+ archivesStaged: res.archivesStaged,
+ });
+ return {
+ status: "ran",
+ stamp: built.stampId,
+ summary: `${siteId} built: ${plural(built.files, "file")}, ${(built.bytes / 1e6).toFixed(1)} MB`,
+ };
+}
+
+// The sites `_all` builds: each stale one (every one with --force).
+// (A fresh one is a no-op build: its `checkedAt` is moved on.)
+async function sitesToBuild(
+ paths: Paths,
+ input: NeedsInput,
+ r: StageRequest,
+ onLog: (l: string) => void,
+): Promise<string[]> {
+ const ids: string[] = [];
+ for (const id of Object.keys(input.sites)) {
+ const f = STAGES["build-site"].needs(input, { ...r, target: id });
+ if (f.state === "fresh") {
+ onLog(`[publish] ${id}: fresh — skipped\n`);
+ await markChecked(paths, input.sites[id].built);
+ } else if (f.state === "blocked") onLog(`[publish] ${id}: blocked — ${f.reason}\n`);
+ else ids.push(id);
+ }
+ return ids;
+}
+
+/**
+ * A no-op build: the bundle still matches its inputs, as of now. Recorded as
+ * `checkedAt`, so the "channels changed" signal (measured against
+ * max(builtAt, checkedAt)) clears, without pretending a build happened.
+ */
+async function markChecked(paths: Paths, built: BuiltStamp | null | undefined): Promise<void> {
+ if (built) await writeBuiltStamp(paths, { ...built, checkedAt: Date.now() });
+}
+
+async function buildAllLocal(ctx: StageContext, r: StageRequest, input: NeedsInput): Promise<StageOutcome> {
+ const stamp = input.index.stamp!;
+ const ids = await sitesToBuild(ctx.paths, input, r, ctx.onLog);
+ const failed: string[] = [];
+ let built = 0;
+ for (const [i, id] of ids.entries()) {
+ checkCancel(ctx.signal);
+ ctx.onLog(`\n=== Build ${id} (${i + 1}/${ids.length}) ===\n`);
+ try {
+ await buildOneSite(ctx, { ...r, target: id }, stamp);
+ built++;
+ } catch (err) {
+ if (err instanceof StageCancelled) throw err;
+ failed.push(id);
+ ctx.onLog(`[publish] ${id}: ${(err as Error).message}\n`);
+ }
+ }
+ const summary = `${built}/${ids.length} built` + (failed.length ? `; failed: ${failed.join(", ")}` : "");
+ if (failed.length) throw new StageFailure(summary, 1);
+ return { status: built ? "ran" : "noop", stamp: stamp.stampId, summary };
+}
+
+export const NO_ENGINE = "the docker runner needs an engine on this host";
+
+async function buildAllDocker(ctx: StageContext, r: StageRequest, input: NeedsInput): Promise<StageOutcome> {
+ const { paths, onLog, signal } = ctx;
+ const b = await import("./build");
+ if (!(await b.dockerAvailable(signal))) throw new StageFailure(NO_ENGINE, 3);
+ const stamp = input.index.stamp!;
+ const ids = await sitesToBuild(paths, input, r, onLog);
+ if (ids.length === 0) return { status: "noop", stamp: stamp.stampId, summary: "every site is fresh" };
+ if (!r.skipArchives) {
+ onLog("=== the archive cache (host) ===\n");
+ const code = await b.runHostScript(onLog, signal, paths, "build:archives");
+ checkCancel(signal);
+ if (code !== 0) throw new StageFailure(`warming the archive cache failed (exit ${code})`, 1);
+ }
+ const img = await b.ensureBuildImage(onLog, signal, paths);
+ checkCancel(signal);
+ if (img !== 0) throw new StageFailure(`the build image failed (exit ${img})`, 1);
+ const { maxParallelBuilds } = getSettings().buildPipeline;
+ onLog(`=== building ${plural(ids.length, "site")} in containers, up to ${maxParallelBuilds} at once ===\n`);
+ const outcomes = await b.runWithConcurrency(ids, maxParallelBuilds, async (id) => {
+ if (signal.aborted) return { id, ok: false };
+ const code = await b.runDockerBuildOne(onLog, signal, id, paths, { skipArchives: r.skipArchives });
+ const out = b.bundleDir(paths, id);
+ const problem = code === 0 ? builtBundleProblem(out, id) : null;
+ if (code !== 0 || problem) {
+ onLog(`[${id}] build FAILED — ${problem ?? `exit ${code}`}\n`);
+ return { id, ok: false };
+ }
+ const staged = await readdir(b.dockerSiteStagingDir(paths, id)).catch(() => [] as string[]);
+ await stampBuilt(paths, out, {
+ target: id,
+ kind: "site",
+ indexStampId: stamp.stampId,
+ inputSig: stamp.sites[id].inputSig,
+ runner: "docker",
+ archivesStaged: staged.filter((f) => !f.startsWith(".")).length,
+ });
+ onLog(`[${id}] build ok\n`);
+ return { id, ok: true };
+ });
+ checkCancel(signal);
+ const failed = outcomes.filter((o) => !o.ok).map((o) => o.id);
+ const summary = `${ids.length - failed.length}/${ids.length} built in containers` + (failed.length ? `; failed: ${failed.join(", ")}` : "");
+ if (failed.length) throw new StageFailure(summary, 1);
+ return { status: "ran", stamp: stamp.stampId, summary };
+}
+
+// ---------------------------------------------------------------------------
+// hub and homepage builds
+// ---------------------------------------------------------------------------
+
+async function buildHubStage(ctx: StageContext, stamp: IndexStamp): Promise<StageOutcome> {
+ const { buildHubBundle, bundleDir } = await import("./build");
+ const code = await buildHubBundle(ctx);
+ checkCancel(ctx.signal);
+ if (code !== 0) throw new StageFailure(`the hub build failed (exit ${code})`, 1);
+ const built = await stampBuilt(ctx.paths, bundleDir(ctx.paths, HUB_TARGET), {
+ target: HUB_TARGET,
+ kind: "hub",
+ indexStampId: stamp.stampId,
+ inputSig: stamp.hubSig,
+ runner: "local",
+ archivesStaged: 0,
+ });
+ return { status: "ran", stamp: built.stampId, summary: `hub built: ${plural(built.files, "file")}` };
+}
+
+async function buildHomepageStage(ctx: StageContext, stamp: IndexStamp): Promise<StageOutcome> {
+ const { buildHomepage, homepageOutDir } = await import("./build");
+ const { readPublishedManifest } = await import("./source");
+ const code = await buildHomepage(ctx);
+ checkCancel(ctx.signal);
+ if (code !== 0) throw new StageFailure(`the homepage build failed (exit ${code})`, 1);
+ const out = homepageOutDir(ctx.paths);
+ const manifest = await readPublishedManifest(out);
+ const built = await stampBuilt(ctx.paths, out, {
+ target: HOMEPAGE_TARGET,
+ kind: "homepage",
+ indexStampId: stamp.stampId,
+ inputSig: stamp.stampId,
+ runner: "local",
+ archivesStaged: 0,
+ sourceCommit: manifest?.sourceCommit ?? null,
+ });
+ return { status: "ran", stamp: built.stampId, summary: `homepage built: ${plural(built.files, "file")}` };
+}
+
+// ---------------------------------------------------------------------------
+// deploys
+// ---------------------------------------------------------------------------
+
+/** Where `--to local` publishes a site: the `site` service's volume. */
+export function localSiteOut(env: NodeJS.ProcessEnv = process.env): string | null {
+ return env.ARCHILYZER_SITE_OUT?.trim() || null;
+}
+
+// …and the homepage: what the `homepage` service serves (release 18 S5
+// declares the name; read by name here until both slices are merged).
+const HOMEPAGE_OUT_NAME = "ARCHILYZER_HOMEPAGE_OUT";
+
+export function localHomepageOut(env: NodeJS.ProcessEnv = process.env): string | null {
+ return env[HOMEPAGE_OUT_NAME]?.trim() || null;
+}
+
+// Replace the CONTENTS of `dest` (a volume mount: never the directory itself).
+async function publishLocal(src: string, dest: string): Promise<void> {
+ await mkdir(dest, { recursive: true });
+ for (const name of await readdir(dest)) await rm(path.join(dest, name), { recursive: true, force: true });
+ await cp(src, dest, { recursive: true });
+}
+
+// Spot the deployment URL build.ts logs on success.
+function urlWatcher(onLog: (l: string) => void): { onLog: (l: string) => void; url: () => string | null } {
+ let url: string | null = null;
+ return {
+ onLog: (line) => {
+ const m = /^\[deployed\] (\S+)/.exec(line) ?? /\(this deployment: (\S+)\)/.exec(line);
+ if (m) url = m[1];
+ onLog(line);
+ },
+ url: () => url,
+ };
+}
+
+async function deployStage(
+ ctx: StageContext,
+ r: StageRequest,
+ target: string,
+ ship: (o: { onLog: (l: string) => void; previewBranch?: string }) => Promise<void>,
+ local: { src: string; dest: string | null; needs: string } | null,
+ project: string | null,
+): Promise<StageOutcome> {
+ const { paths, onLog, signal } = ctx;
+ const built = (await readBuiltStamp(paths, target))!;
+ const kind = deployKindOf(r);
+ let url: string | null = null;
+ let alias: string | undefined;
+ if (kind === "local") {
+ if (!local) throw new StageFailure(`${target} has no local target`, 2);
+ if (!local.dest) {
+ throw new StageFailure(`--to local needs ${local.needs}`, 3);
+ }
+ // A bundle built private is never published, here either.
+ const priv = builtAudienceProblem(local.src);
+ if (priv) throw new StageFailure(`${priv}. Build ${target} again, then deploy.`, 3);
+ onLog(`[publish] copying ${local.src} -> ${local.dest}\n`);
+ await publishLocal(local.src, local.dest);
+ } else {
+ if (r.preview !== undefined) {
+ const problem = previewBranchProblem(r.preview);
+ if (problem) throw new StageFailure(problem, 2);
+ }
+ const w = urlWatcher(onLog);
+ try {
+ await ship({ onLog: w.onLog, previewBranch: r.preview });
+ } catch (err) {
+ checkCancel(signal);
+ throw new StageFailure((err as Error).message, 1);
+ }
+ checkCancel(signal);
+ url = w.url();
+ if (r.preview && project) alias = previewAliasUrl(project, r.preview);
+ }
+ const record: DeployRecord = {
+ builtStampId: built.stampId,
+ builtAt: built.builtAt,
+ kind,
+ ...(r.preview ? { branch: r.preview } : {}),
+ url,
+ ...(alias ? { alias } : {}),
+ at: Date.now(),
+ liveCheck: null,
+ };
+ await recordDeploy(paths, target, record);
+ const where = kind === "local" ? "local" : kind === "preview" ? `preview "${r.preview}"` : "production";
+ return { status: "ran", stamp: built.stampId, summary: `${target} deployed (${where})${url ? ` ${url}` : ""}` };
+}
+
+// ---------------------------------------------------------------------------
+// The dispatcher
+// ---------------------------------------------------------------------------
+
+/** Run the stage `r` names (the body of every Stage's `run`). */
+export async function runStageBody(ctx: StageContext, r: StageRequest): Promise<StageOutcome> {
+ const { paths } = ctx;
+ if (r.kind === "update-index") return runUpdateIndex(ctx);
+ // Usage before state: a bad preview name is said as such, whatever is built.
+ if (r.kind.startsWith("deploy-")) {
+ if (r.preview !== undefined) {
+ const problem = previewBranchProblem(r.preview);
+ if (problem) throw new StageFailure(problem, 2);
+ }
+ if (r.preview && r.to === "local") {
+ throw new StageFailure("--preview and --to local are two different deploys", 2);
+ }
+ }
+ // The docker per-site container builds what the host's fan-out decided to
+ // build: no stamps of its own to judge by (its builds dir is scratch).
+ if (r.kind === "build-site" && inBuildContainer()) {
+ return buildOneSite(ctx, r, (await readIndexStamp(paths)) ?? CONTAINER_NO_STAMP);
+ }
+
+ const input = await readNeedsInput(paths, { mainHead: r.kind === "build-homepage" });
+ const f = STAGES[r.kind].needs(input, r);
+ if (f.state === "blocked") throw new StageFailure(f.reason, 3);
+ if (f.state === "fresh") {
+ const stampOf =
+ r.kind.startsWith("deploy-") || r.target === ALL_TARGET
+ ? (input.index.stamp?.stampId ?? "")
+ : "";
+ const built =
+ r.kind === "build-site" || r.kind === "deploy-site"
+ ? input.sites[r.target]?.built
+ : r.kind.endsWith("-hub")
+ ? input.hub.built
+ : input.homepage.built;
+ if (r.kind.startsWith("build-")) {
+ if (r.target === ALL_TARGET) {
+ for (const t of Object.values(input.sites)) await markChecked(paths, t.built);
+ } else {
+ await markChecked(paths, built);
+ }
+ }
+ return {
+ status: "noop",
+ stamp: built?.stampId ?? stampOf,
+ summary: `${r.target}: fresh — nothing to do (--force runs it anyway)`,
+ };
+ }
+ ctx.onLog(`[publish] ${STAGES[r.kind].label} ${r.target}: ${f.reason}\n`);
+ const stamp = input.index.stamp;
+
+ const b = await import("./build");
+ switch (r.kind) {
+ case "build-site":
+ if (r.target === ALL_TARGET) {
+ return r.runner === "docker" ? buildAllDocker(ctx, r, input) : buildAllLocal(ctx, r, input);
+ }
+ return buildOneSite(ctx, r, stamp!);
+ case "build-hub":
+ return buildHubStage(ctx, stamp!);
+ case "build-homepage":
+ return buildHomepageStage(ctx, stamp!);
+ case "deploy-site": {
+ const site = getSite(r.target, paths);
+ const out = b.bundleDir(paths, site.siteId);
+ return deployStage(
+ ctx,
+ r,
+ site.siteId,
+ (o) =>
+ b.deploySite(site.siteId, {
+ paths,
+ signal: ctx.signal,
+ onLog: o.onLog,
+ previewBranch: o.previewBranch,
+ outDir: out,
+ stagingDir: b.dockerSiteStagingDir(paths, site.siteId),
+ }),
+ { src: out, dest: localSiteOut(), needs: "ARCHILYZER_SITE_OUT (the directory the docker `site` service serves)" },
+ site.cloudflareProject?.trim() || null,
+ );
+ }
+ case "deploy-hub":
+ return deployStage(
+ ctx,
+ r,
+ HUB_TARGET,
+ (o) =>
+ b.deployHub({
+ paths,
+ signal: ctx.signal,
+ onLog: o.onLog,
+ previewBranch: o.previewBranch,
+ outDir: b.bundleDir(paths, HUB_TARGET),
+ }),
+ null,
+ getHomepageConfig(paths).cloudflareProject?.trim() || null,
+ );
+ case "deploy-homepage":
+ return deployStage(
+ ctx,
+ r,
+ HOMEPAGE_TARGET,
+ (o) => b.deployHomepage({ paths, signal: ctx.signal, onLog: o.onLog, previewBranch: o.previewBranch }),
+ { src: b.homepageOutDir(paths), dest: localHomepageOut(), needs: "ARCHILYZER_HOMEPAGE_OUT (the directory the docker `homepage` service serves)" },
+ b.HOMEPAGE_PAGES_PROJECT,
+ );
+ }
+}
diff --git a/common/publish/stageLock.test.ts b/common/publish/stageLock.test.ts
@@ -0,0 +1,205 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import {
+ LockWaitCancelled,
+ acquirePublishLock,
+ START_SLACK_MS,
+ holderIsGone,
+ lockHostId,
+ pidStartOf,
+ processStartedAtMs,
+ publishLockPath,
+ readLockHolder,
+ withPublishLock,
+ type LockHolder,
+} from "./stageLock";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common test
+//
+// The publish lock (release 18): a stale holder on this host is taken over, a
+// live one is waited for (cancellably), and a holder on another host is never
+// stolen. The process probes are injected; nothing here signals a real pid.
+
+function tmp(): { root: string; paths: Paths } {
+ const root = mkdtempSync(path.join(os.tmpdir(), "stage-lock-"));
+ return { root, paths: { exportBuildsDir: path.join(root, ".export-builds") } as Paths };
+}
+
+function plant(paths: Paths, holder: Partial<LockHolder>): void {
+ mkdirSync(paths.exportBuildsDir, { recursive: true });
+ writeFileSync(
+ publishLockPath(paths),
+ JSON.stringify({ pid: 4242, host: "here", kind: "build-site", target: "jer", since: 1, ...holder }),
+ );
+}
+
+const here = { host: "here", pid: 100, startOf: () => null, startedAtMs: () => null };
+
+test("holderIsGone: same host and a dead pid, or a pid reused by a later process", () => {
+ const h: LockHolder = { pid: 7, host: "here", kind: "k", target: "t", since: 1, pidStart: "500" };
+ assert.equal(holderIsGone(h, { host: "here", isAlive: () => false }), true);
+ assert.equal(holderIsGone(h, { host: "here", isAlive: () => true, startOf: () => "500", startedAtMs: () => null }), false);
+ assert.equal(holderIsGone(h, { host: "here", isAlive: () => true, startOf: () => "900", startedAtMs: () => null }), true, "pid reused");
+ assert.equal(holderIsGone(h, { host: "here", isAlive: () => true, startOf: () => null, startedAtMs: () => null }), false);
+ assert.equal(holderIsGone(h, { host: "there", isAlive: () => false }), false, "another host: never");
+ assert.equal(holderIsGone({ ...h, pidStart: null }, { host: "here", isAlive: () => true, startOf: () => "1", startedAtMs: () => null }), false);
+});
+
+test("holderIsGone: a live pid whose process started AFTER the lock was taken is not the holder (a recreated container's pid 1)", () => {
+ const since = 1_000_000;
+ const h: LockHolder = { pid: 1, host: "archilyzer-editor", kind: "k", target: "t", since };
+ const probe = (startedAt: number | null) =>
+ holderIsGone(h, { host: "archilyzer-editor", isAlive: () => true, startOf: () => null, startedAtMs: () => startedAt });
+ assert.equal(probe(since + 60_000), true, "started a minute after the lock: stale");
+ assert.equal(probe(since - 60_000), false, "started before the lock: may be the holder");
+ assert.equal(probe(since + 1_000), false, "within btime's slack: not judged");
+ assert.equal(probe(null), false, "no /proc: pid-alive alone decides");
+});
+
+test("the host identity is ARCHILYZER_HOST_ID when set, else the hostname", () => {
+ assert.equal(lockHostId({ ARCHILYZER_HOST_ID: " archilyzer-editor " }), "archilyzer-editor");
+ assert.equal(lockHostId({ ARCHILYZER_HOST_ID: "" }), os.hostname());
+ assert.equal(lockHostId({}), os.hostname());
+});
+
+test("processStartedAtMs: this process started before now, and no such pid is null", () => {
+ if (process.platform === "linux") {
+ const t = processStartedAtMs(process.pid);
+ assert.ok(t !== null && t <= Date.now() + START_SLACK_MS && t > Date.now() - 86_400_000 * 365, String(t));
+ }
+ assert.equal(processStartedAtMs(2 ** 30), null);
+});
+
+test("pidStartOf reads this process's start time on Linux and is null for no such pid", () => {
+ if (process.platform === "linux") {
+ assert.match(pidStartOf(process.pid) ?? "", /^\d+$/);
+ assert.equal(pidStartOf(process.pid), pidStartOf(process.pid));
+ }
+ assert.equal(pidStartOf(2 ** 30), null);
+});
+
+test("the lock is taken, names its holder, and is released", async () => {
+ const { root, paths } = tmp();
+ try {
+ const lock = await acquirePublishLock(paths, { kind: "update-index", target: "_index" }, here);
+ assert.deepEqual(await readLockHolder(paths), lock.holder);
+ assert.equal(lock.holder.pid, 100);
+ assert.equal(lock.holder.kind, "update-index");
+ await lock.release();
+ assert.equal(existsSync(publishLockPath(paths)), false);
+ // withPublishLock releases on a throw too.
+ await assert.rejects(
+ withPublishLock(paths, { kind: "k", target: "t" }, async () => {
+ throw new Error("boom");
+ }, here),
+ /boom/,
+ );
+ assert.equal(existsSync(publishLockPath(paths)), false);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("a stale holder on this host is taken over, and said so", async () => {
+ const { root, paths } = tmp();
+ try {
+ plant(paths, { pid: 4242, host: "here" });
+ const logs: string[] = [];
+ const lock = await acquirePublishLock(paths, { kind: "build-site", target: "ani" }, {
+ ...here,
+ isAlive: (pid) => pid !== 4242,
+ onLog: (l) => logs.push(l),
+ });
+ assert.equal(lock.holder.target, "ani");
+ assert.match(logs.join(""), /taking over a stale publish lock: build-site jer \(pid 4242 on here/);
+ await lock.release();
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("a live holder is waited for, with one log line, until it releases", async () => {
+ const { root, paths } = tmp();
+ try {
+ plant(paths, { pid: 4242, host: "here" });
+ const logs: string[] = [];
+ let polls = 0;
+ const taking = acquirePublishLock(paths, { kind: "build-site", target: "ani" }, {
+ ...here,
+ isAlive: () => true,
+ pollMs: 5,
+ onLog: (l) => {
+ logs.push(l);
+ },
+ now: () => {
+ polls++;
+ // The holder releases after a few polls.
+ if (polls === 6) rmSync(publishLockPath(paths));
+ return polls;
+ },
+ });
+ const lock = await taking;
+ assert.equal(lock.holder.target, "ani");
+ assert.equal(logs.length, 1, logs.join(""));
+ assert.match(logs[0], /waiting for the publish lock — held by build-site jer \(pid 4242 on here/);
+ await lock.release();
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("a holder on another host is never stolen; the wait is cancellable", async () => {
+ const { root, paths } = tmp();
+ try {
+ plant(paths, { pid: 4242, host: "elsewhere" });
+ const ac = new AbortController();
+ const logs: string[] = [];
+ const taking = acquirePublishLock(paths, { kind: "k", target: "t" }, {
+ ...here,
+ isAlive: () => false, // dead HERE means nothing for a pid over there
+ pollMs: 5,
+ signal: ac.signal,
+ onLog: (l) => logs.push(l),
+ });
+ setTimeout(() => ac.abort(), 40);
+ await assert.rejects(taking, LockWaitCancelled);
+ assert.equal(logs.length, 1);
+ assert.match(logs[0], /on ANOTHER host \("elsewhere"; this one is "here"\)/);
+ assert.match(logs[0], /If no publish stage is running there, remove .*\.publish\.lock/);
+ assert.equal(JSON.parse(readFileSync(publishLockPath(paths), "utf8")).host, "elsewhere", "untouched");
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("a torn lock file is waited for, then taken over after the grace", async () => {
+ const { root, paths } = tmp();
+ try {
+ mkdirSync(paths.exportBuildsDir, { recursive: true });
+ writeFileSync(publishLockPath(paths), "");
+ const old = new Date(Date.now() - 120_000);
+ utimesSync(publishLockPath(paths), old, old);
+ const lock = await acquirePublishLock(paths, { kind: "k", target: "t" }, { ...here, pollMs: 5 });
+ assert.equal(lock.holder.kind, "k");
+ await lock.release();
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("release never removes a lock somebody else holds now", async () => {
+ const { root, paths } = tmp();
+ try {
+ const lock = await acquirePublishLock(paths, { kind: "k", target: "t" }, here);
+ plant(paths, { pid: 9, host: "here", since: 99 });
+ await lock.release();
+ assert.equal(JSON.parse(readFileSync(publishLockPath(paths), "utf8")).pid, 9);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
diff --git a/common/publish/stageLock.ts b/common/publish/stageLock.ts
@@ -0,0 +1,291 @@
+// The publish lock (release 18): ONE publish stage at a time on this machine,
+// whoever started it — a stage child the editor spawned, or `archilyzer
+// publish …` run by hand (`docker compose exec editor …` included). The
+// editor's `publish` queue already runs its stages one by one; this file is
+// what keeps a CLI started beside it from building into the same shared
+// export/public at once.
+//
+// <exportBuildsDir>/.publish.lock {pid, host, kind, target, since, pidStart}
+//
+// Taken with O_EXCL (`open(…, "wx")`). A holder is STALE — and its lock taken
+// over — only when it is on THIS host and its process is gone: the pid does
+// not answer (`processIsAlive`, jobs/bootQueuedJobs.ts), or it answers with a
+// different start time (`/proc/<pid>/stat`, where there is one: a container's
+// editor comes back with the same small pids on every restart), or the
+// process that pid names now STARTED AFTER the lock was taken (`since`) — so
+// it cannot be the holder, whatever `pidStart` the lock carries. A lock naming
+// another host is never stolen — a pid means nothing across a namespace. A
+// live holder makes the taker WAIT: a poll every 5 s, one log line, and a
+// cancel (the AbortSignal) gives up the wait.
+//
+// THE HOST is `ARCHILYZER_HOST_ID` when set, else `os.hostname()`. In the
+// container the hostname is the container id, new on every recreate — which
+// would make the last container's lock "another host's" for ever — so the
+// compose file sets a fixed ARCHILYZER_HOST_ID (release 18 S5). With a fixed
+// id a recreated container's editor is pid 1 again, alive: the start-time
+// rules above are what tell it from the holder. Where there is no /proc (not
+// Linux) neither start-time rule can be asked, and staleness is pid-alive alone.
+
+import { readFileSync } from "node:fs";
+import { mkdir, open, readFile, rm, stat } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { processIsAlive } from "../jobs/bootQueuedJobs";
+import type { Paths } from "../lib/paths";
+
+export type LockHolder = {
+ pid: number;
+ host: string;
+ kind: string;
+ target: string;
+ since: number;
+ // The holder process's start time (/proc/<pid>/stat field 22), when known.
+ pidStart?: string | null;
+};
+
+export const LOCK_POLL_MS = 5_000;
+// A lock file that stays unreadable this long was left by a taker that died
+// between creating it and writing it.
+export const LOCK_TORN_GRACE_MS = 60_000;
+
+export class LockWaitCancelled extends Error {
+ constructor() {
+ super("cancelled while waiting for the publish lock");
+ this.name = "LockWaitCancelled";
+ }
+}
+
+export function publishLockPath(paths: Pick<Paths, "exportBuildsDir">): string {
+ return path.join(paths.exportBuildsDir, ".publish.lock");
+}
+
+// /proc reports starttime in USER_HZ ticks, which Linux fixes at 100 for
+// every userspace interface whatever the kernel's HZ.
+const USER_HZ = 100;
+
+/**
+ * When `pid`'s process started, in ms since the epoch — /proc/<pid>/stat's
+ * starttime (ticks since boot) plus /proc/stat's `btime` — or null where
+ * there is no /proc. `btime` is whole seconds, so this is good to about 1 s.
+ */
+export function processStartedAtMs(pid: number): number | null {
+ const raw = pidStartOf(pid);
+ const ticks = raw === null ? NaN : Number(raw);
+ if (!Number.isFinite(ticks)) return null;
+ try {
+ const btime = /^btime (\d+)$/m.exec(readFileSync("/proc/stat", "utf8"));
+ if (!btime) return null;
+ return Number(btime[1]) * 1000 + (ticks * 1000) / USER_HZ;
+ } catch {
+ return null;
+ }
+}
+
+// The slack on "started after the lock was taken": btime's whole seconds.
+export const START_SLACK_MS = 2_000;
+
+// The host identity's override (release 18 S5 declares it in lib/envVars.ts;
+// read by name here until both slices are merged).
+const HOST_ID_NAME = "ARCHILYZER_HOST_ID";
+
+/** This machine's identity for the lock: ARCHILYZER_HOST_ID, else the hostname. */
+export function lockHostId(envVars: NodeJS.ProcessEnv = process.env): string {
+ return envVars[HOST_ID_NAME]?.trim() || os.hostname();
+}
+
+/** A process's start time from /proc (Linux), or null where there is none. */
+export function pidStartOf(pid: number): string | null {
+ try {
+ const raw = readFileSync(`/proc/${pid}/stat`, "utf8");
+ // `pid (comm) state ppid …` — comm may hold spaces and parens, so split
+ // after the LAST ')'. Field 22 (starttime) is index 19 of the rest.
+ const rest = raw.slice(raw.lastIndexOf(")") + 2).split(" ");
+ return rest[19] ?? null;
+ } catch {
+ return null;
+ }
+}
+
+export type LockEnv = {
+ host?: string;
+ pid?: number;
+ isAlive?: (pid: number) => boolean;
+ startOf?: (pid: number) => string | null;
+ // When the process `pid` names now started (ms), or null when unknown.
+ startedAtMs?: (pid: number) => number | null;
+ now?: () => number;
+};
+
+function env(e: LockEnv = {}) {
+ return {
+ host: e.host ?? lockHostId(),
+ pid: e.pid ?? process.pid,
+ isAlive: e.isAlive ?? processIsAlive,
+ startOf: e.startOf ?? pidStartOf,
+ startedAtMs: e.startedAtMs ?? processStartedAtMs,
+ now: e.now ?? Date.now,
+ };
+}
+
+/**
+ * True when `holder`'s process is certainly gone: same host, and its pid is
+ * dead or now belongs to a process that started at another time. Pure over
+ * the injected probes.
+ */
+export function holderIsGone(holder: LockHolder, e: LockEnv = {}): boolean {
+ const { host, isAlive, startOf, startedAtMs } = env(e);
+ if (holder.host !== host) return false;
+ if (!isAlive(holder.pid)) return true;
+ if (holder.pidStart) {
+ const now = startOf(holder.pid);
+ if (now !== null && now !== holder.pidStart) return true;
+ }
+ // The pid's process started after the lock was taken: not the holder.
+ const started = startedAtMs(holder.pid);
+ if (started !== null && started > holder.since + START_SLACK_MS) return true;
+ return false;
+}
+
+export function parseLockHolder(text: string): LockHolder | null {
+ try {
+ const v = JSON.parse(text) as Partial<LockHolder>;
+ if (
+ typeof v?.pid !== "number" ||
+ typeof v.host !== "string" ||
+ typeof v.kind !== "string" ||
+ typeof v.target !== "string" ||
+ typeof v.since !== "number"
+ ) {
+ return null;
+ }
+ return v as LockHolder;
+ } catch {
+ return null;
+ }
+}
+
+/** Who holds the publish lock now, or null (none, or unreadable). */
+export async function readLockHolder(paths: Pick<Paths, "exportBuildsDir">): Promise<LockHolder | null> {
+ try {
+ return parseLockHolder(await readFile(publishLockPath(paths), "utf8"));
+ } catch {
+ return null;
+ }
+}
+
+export function describeHolder(h: LockHolder): string {
+ return `${h.kind} ${h.target} (pid ${h.pid} on ${h.host}, since ${new Date(h.since).toISOString()})`;
+}
+
+export type PublishLock = { holder: LockHolder; release: () => Promise<void> };
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+ return new Promise((resolve) => {
+ const t = setTimeout(done, ms);
+ function done() {
+ clearTimeout(t);
+ signal?.removeEventListener("abort", done);
+ resolve();
+ }
+ signal?.addEventListener("abort", done, { once: true });
+ });
+}
+
+/**
+ * Take the publish lock for `who`, waiting while a live holder has it.
+ * Throws LockWaitCancelled when `signal` aborts first.
+ */
+export async function acquirePublishLock(
+ paths: Pick<Paths, "exportBuildsDir">,
+ who: { kind: string; target: string },
+ opts: LockEnv & { signal?: AbortSignal; onLog?: (line: string) => void; pollMs?: number } = {},
+): Promise<PublishLock> {
+ const e = env(opts);
+ const file = publishLockPath(paths);
+ await mkdir(path.dirname(file), { recursive: true });
+ let said = false;
+ for (;;) {
+ if (opts.signal?.aborted) throw new LockWaitCancelled();
+ const holder: LockHolder = {
+ pid: e.pid,
+ host: e.host,
+ kind: who.kind,
+ target: who.target,
+ since: e.now(),
+ pidStart: e.startOf(e.pid),
+ };
+ try {
+ const fh = await open(file, "wx");
+ try {
+ await fh.writeFile(JSON.stringify(holder) + "\n");
+ await fh.sync().catch(() => {});
+ } finally {
+ await fh.close();
+ }
+ return { holder, release: () => releaseLock(file, holder) };
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
+ }
+ // Someone holds it (or held it).
+ let text = "";
+ try {
+ text = await readFile(file, "utf8");
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code === "ENOENT") continue; // released meanwhile
+ throw err;
+ }
+ const current = parseLockHolder(text);
+ if (current === null) {
+ // Torn: a taker between its create and its write — or one that died
+ // there. Give it the grace, then take the lock over.
+ const age = await stat(file).then((s) => e.now() - s.mtimeMs, () => 0);
+ if (age > LOCK_TORN_GRACE_MS) {
+ await removeIfUnchanged(file, text);
+ continue;
+ }
+ } else if (holderIsGone(current, e)) {
+ opts.onLog?.(`[publish] taking over a stale publish lock: ${describeHolder(current)} is gone\n`);
+ await removeIfUnchanged(file, text);
+ continue;
+ } else if (!said) {
+ said = true;
+ opts.onLog?.(
+ current.host === e.host
+ ? `[publish] waiting for the publish lock — held by ${describeHolder(current)}\n`
+ : `[publish] waiting for the publish lock — held by ${describeHolder(current)}, on ANOTHER host ` +
+ `("${current.host}"; this one is "${e.host}"), which this one can never judge stale. If no ` +
+ `publish stage is running there, remove ${file} (or give both the same ARCHILYZER_HOST_ID ` +
+ `when they are one machine)\n`,
+ );
+ }
+ await sleep(opts.pollMs ?? LOCK_POLL_MS, opts.signal);
+ }
+}
+
+// Remove the lock only if it still holds exactly what was judged stale.
+async function removeIfUnchanged(file: string, judged: string): Promise<void> {
+ const now = await readFile(file, "utf8").catch(() => null);
+ if (now === judged) await rm(file, { force: true });
+}
+
+async function releaseLock(file: string, holder: LockHolder): Promise<void> {
+ const current = parseLockHolder(await readFile(file, "utf8").catch(() => ""));
+ if (current && current.pid === holder.pid && current.host === holder.host && current.since === holder.since) {
+ await rm(file, { force: true });
+ }
+}
+
+/** Run `fn` holding the publish lock; always released after. */
+export async function withPublishLock<T>(
+ paths: Pick<Paths, "exportBuildsDir">,
+ who: { kind: string; target: string },
+ fn: () => Promise<T>,
+ opts: LockEnv & { signal?: AbortSignal; onLog?: (line: string) => void; pollMs?: number } = {},
+): Promise<T> {
+ const lock = await acquirePublishLock(paths, who, opts);
+ try {
+ return await fn();
+ } finally {
+ await lock.release();
+ }
+}
diff --git a/common/publish/stageRun.test.ts b/common/publish/stageRun.test.ts
@@ -0,0 +1,306 @@
+import { after, test } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import os from "node:os";
+import path from "node:path";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common test
+//
+// The publish stages run for real over a scratch corpus (release 18): the
+// update-index body writes the stamp and keeps its id when nothing changed;
+// every other stage asks its precondition ON DISK first (exit 3), a fresh
+// target is a no-op, `--to local` copies the bundle and records it; the lock
+// is waited for and a wait is cancellable (130). No `next build` and no
+// wrangler run here: the build and Pages paths are the bundle and build tests'.
+//
+// Every path getPaths() can resolve to a place this file may write is pinned
+// under ROOT before anything calls it (the buildStats.test.ts pattern).
+const ROOT = mkdtempSync(path.join(tmpdir(), "stage-run-"));
+const PINNED: Record<string, string> = {
+ TRANSCRIPTS_DIR: path.join(ROOT, "transcripts"),
+ SAVED_VIDEOS_DIR: path.join(ROOT, "saved-videos"),
+ SITES_DIR: path.join(ROOT, "transcripts", "sites"),
+ SETTINGS_FILE: path.join(ROOT, "settings.json"),
+ EXPORT_PUBLIC_DIR: path.join(ROOT, "export", "public"),
+ EXPORT_INDEX_DIR: path.join(ROOT, "export", ".export-index"),
+ EXPORT_BUILDS_DIR: path.join(ROOT, "export", ".export-builds"),
+ EDITOR_CHANGELOG_FILE: path.join(ROOT, "editor-CHANGELOG.md"),
+ EXPORT_CHANGELOG_FILE: path.join(ROOT, "export-CHANGELOG.md"),
+ CHARTS_CONFIG_FILE: path.join(ROOT, "chart-templates.json"),
+ SEARCH_ALIASES_FILE: path.join(ROOT, "transcripts", "search-aliases.json"),
+ CURATED_TAGS_FILE: path.join(ROOT, "transcripts", "tags.json"),
+ ARCHILYZER_CONFIG_DIR: path.join(ROOT, "config"),
+ ARCHILYZER_SOURCE_SCRATCH: path.join(ROOT, "source-scratch"),
+};
+Object.assign(process.env, PINNED);
+delete process.env.ARCHILYZER_SITE_OUT;
+delete process.env.ARCHIVES_READONLY;
+after(() => rmSync(ROOT, { recursive: true, force: true }));
+
+mkdirSync(path.join(ROOT, "transcripts", "channels"), { recursive: true });
+writeFileSync(PINNED.SETTINGS_FILE, "{}");
+function writeSite(id: string, extra: Record<string, unknown> = {}): void {
+ mkdirSync(path.join(PINNED.SITES_DIR, id), { recursive: true });
+ writeFileSync(
+ path.join(PINNED.SITES_DIR, id, "site.json"),
+ JSON.stringify({ siteId: id, siteTitle: id, channels: [], ...extra }),
+ );
+}
+writeSite("jer", { cloudflareProject: "w3c-never-real" });
+
+const { getPaths } = await import("../lib/paths");
+const { runStage } = await import("./stageRun");
+const stamps = await import("./stamps");
+const { bundleDir } = await import("./build");
+const { publishLockPath } = await import("./stageLock");
+const paths = getPaths();
+
+const quiet = () => {};
+async function stage(kind: string, target: string, extra: Record<string, unknown> = {}, logs?: string[]) {
+ return runStage(
+ { kind: kind as never, target, runId: "t", ...extra },
+ { paths, onLog: (l) => logs?.push(l) ?? quiet() },
+ );
+}
+
+function plantBundle(siteId: string): void {
+ const out = bundleDir(paths, siteId);
+ mkdirSync(out, { recursive: true });
+ writeFileSync(path.join(out, "index.html"), "<!doctype html>");
+ writeFileSync(path.join(out, "site.json"), JSON.stringify({ siteId }));
+ writeFileSync(path.join(out, "corpus.json"), JSON.stringify({ site: { id: siteId }, generatedAt: "g1" }));
+}
+
+test("before any index: a build is refused 'update the index first' (exit 3), and the lock is released", async () => {
+ const logs: string[] = [];
+ const r = await stage("build-site", "jer", {}, logs);
+ assert.equal(r.code, 3);
+ assert.equal(r.message, "update the index first");
+ assert.match(logs.join(""), /\[stage\] build-site jer: REFUSED — update the index first/);
+ assert.ok(!existsSync(publishLockPath(paths)));
+ assert.equal((await stage("build-hub", "_hub")).code, 3);
+ assert.equal((await stage("build-homepage", "_homepage")).code, 3);
+});
+
+test("update-index writes the stamp; a rerun with nothing changed keeps its id; a site.json edit makes a new one", async () => {
+ const first = await stage("update-index", "_index");
+ assert.equal(first.code, 0, first.message ?? "");
+ assert.equal(first.outcome?.status, "ran");
+ const stamp = await stamps.readIndexStamp(paths);
+ assert.ok(stamp);
+ assert.equal(stamp.stampId, first.outcome?.stamp);
+ assert.deepEqual(Object.keys(stamp.sites), ["jer"]);
+ assert.match(stamp.sites.jer.inputSig, /^[0-9a-f]{40}$/);
+ assert.equal(typeof stamp.scannedAt, "number");
+
+ const again = await stage("update-index", "_index");
+ assert.equal(again.code, 0);
+ assert.equal(again.outcome?.status, "noop");
+ const same = await stamps.readIndexStamp(paths);
+ assert.equal(same?.stampId, stamp.stampId);
+ assert.equal(same?.sites.jer.inputSig, stamp.sites.jer.inputSig);
+ assert.ok(same!.builtAt >= stamp.builtAt);
+
+ writeSite("jer", { cloudflareProject: "w3c-never-real", siteDescription: "edited" });
+ const edited = await stage("update-index", "_index");
+ assert.equal(edited.outcome?.status, "ran");
+ const next = await stamps.readIndexStamp(paths);
+ assert.notEqual(next?.stampId, stamp.stampId);
+ assert.notEqual(next?.sites.jer.inputSig, stamp.sites.jer.inputSig, "site.json is in the signature");
+ assert.notEqual(next?.hubSig, stamp.hubSig, "the hub follows the stamp");
+});
+
+test("a site the index has not seen is blocked; a fresh site's build is a no-op; deploys judge the built stamp", async () => {
+ writeSite("ani");
+ assert.match((await stage("build-site", "ani")).message!, /the index has not seen site "ani"/);
+ rmSync(path.join(PINNED.SITES_DIR, "ani"), { recursive: true });
+
+ // Never built: a deploy is refused with the command that fixes it.
+ const none = await stage("deploy-site", "jer");
+ assert.equal(none.code, 3);
+ assert.equal(none.message, "no build of jer — archilyzer publish build jer");
+
+ // A bundle stamped from the current index: fresh, so building is a no-op.
+ const stamp = (await stamps.readIndexStamp(paths))!;
+ plantBundle("jer");
+ await stamps.writeBuiltStamp(paths, {
+ v: 1,
+ stampId: "b-jer",
+ target: "jer",
+ kind: "site",
+ indexStampId: stamp.stampId,
+ inputSig: stamp.sites.jer.inputSig,
+ builtAt: Date.now(),
+ commit: null,
+ branch: "main",
+ runner: "local",
+ audience: "public",
+ corpusGeneratedAt: "g1",
+ files: 3,
+ bytes: 1,
+ archivesStaged: 0,
+ });
+ const fresh = await stage("build-site", "jer");
+ assert.equal(fresh.code, 0);
+ assert.equal(fresh.outcome?.status, "noop");
+ assert.equal(fresh.outcome?.stamp, "b-jer");
+ // A no-op build records that it checked (checkedAt), and nothing else.
+ const checked = await stamps.readBuiltStamp(paths, "jer");
+ assert.equal(checked?.stampId, "b-jer");
+ assert.equal(typeof checked?.checkedAt, "number");
+ assert.equal(checked?.inputSig, stamp.sites.jer.inputSig);
+
+ // A bad preview name is usage (2), before anything else is asked.
+ assert.equal((await stage("deploy-site", "jer", { preview: "main" })).code, 2);
+ assert.equal((await stage("deploy-site", "jer", { preview: "x", to: "local" })).code, 2);
+
+ // --to local needs the site service's directory.
+ const noOut = await stage("deploy-site", "jer", { to: "local" });
+ assert.equal(noOut.code, 3);
+ assert.match(noOut.message!, /--to local needs ARCHILYZER_SITE_OUT/);
+
+ // …and with it, copies the bundle into it (its CONTENTS replaced) and records the deploy.
+ const siteOut = path.join(ROOT, "builds", "site");
+ mkdirSync(siteOut, { recursive: true });
+ writeFileSync(path.join(siteOut, "stale.html"), "old");
+ process.env.ARCHILYZER_SITE_OUT = siteOut;
+ try {
+ const local = await stage("deploy-site", "jer", { to: "local" });
+ assert.equal(local.code, 0, local.message ?? "");
+ assert.equal(local.outcome?.status, "ran");
+ assert.ok(existsSync(path.join(siteOut, "index.html")));
+ assert.ok(!existsSync(path.join(siteOut, "stale.html")));
+ const rec = stamps.deployRecordFor(await stamps.readDeployedFile(paths, "jer"), "local");
+ assert.equal(rec?.builtStampId, "b-jer");
+ assert.equal(rec?.liveCheck, null);
+ // Deployed already: a no-op, until --force.
+ assert.equal((await stage("deploy-site", "jer", { to: "local" })).outcome?.status, "noop");
+ assert.equal((await stage("deploy-site", "jer", { to: "local", force: true })).outcome?.status, "ran");
+ // builtAfter: the build this run waits on has not happened.
+ const waiting = await stage("deploy-site", "jer", { to: "local", builtAfter: Date.now() + 60_000 });
+ assert.equal(waiting.code, 3);
+ assert.match(waiting.message!, /waiting for the build of jer/);
+ } finally {
+ delete process.env.ARCHILYZER_SITE_OUT;
+ }
+
+ // Production refuses a build made on another branch.
+ const built = (await stamps.readBuiltStamp(paths, "jer"))!;
+ await stamps.writeBuiltStamp(paths, { ...built, branch: "r18/stage-core" });
+ const off = await stage("deploy-site", "jer");
+ assert.equal(off.code, 3);
+ assert.match(off.message!, /production ships only a build of main/);
+});
+
+test("a private site is never deployed, here either (exit 3)", async () => {
+ writeSite("mine", { audience: "private", cloudflareProject: "w3c-never-real" });
+ try {
+ const r = await stage("deploy-site", "mine", { to: "local" });
+ assert.equal(r.code, 3);
+ assert.match(r.message!, /Site "mine" is private/);
+ } finally {
+ rmSync(path.join(PINNED.SITES_DIR, "mine"), { recursive: true });
+ }
+});
+
+test("a live holder of the publish lock is waited for; a cancel during the wait exits 130", async () => {
+ mkdirSync(path.dirname(publishLockPath(paths)), { recursive: true });
+ writeFileSync(
+ publishLockPath(paths),
+ JSON.stringify({ pid: process.pid, host: os.hostname(), kind: "build-site", target: "x", since: Date.now() }),
+ );
+ try {
+ const ac = new AbortController();
+ const logs: string[] = [];
+ const running = runStage(
+ { kind: "build-site", target: "jer", runId: "t" },
+ { paths, signal: ac.signal, onLog: (l) => logs.push(l), lockEnv: { pollMs: 5 } },
+ );
+ setTimeout(() => ac.abort(), 50);
+ const r = await running;
+ assert.equal(r.code, 130);
+ assert.match(logs.join(""), /waiting for the publish lock — held by build-site x/);
+ assert.match(logs.join(""), /cancelled while waiting/);
+ } finally {
+ rmSync(publishLockPath(paths), { force: true });
+ }
+});
+
+test("a lock left by a dead process on this host is taken over", async () => {
+ writeFileSync(
+ publishLockPath(paths),
+ JSON.stringify({ pid: 2 ** 30, host: os.hostname(), kind: "update-index", target: "_index", since: 1 }),
+ );
+ const old = new Date(Date.now() - 1000);
+ utimesSync(publishLockPath(paths), old, old);
+ const logs: string[] = [];
+ const r = await stage("build-site", "jer", {}, logs);
+ assert.equal(r.code, 0, r.message ?? "");
+ assert.match(logs.join(""), /taking over a stale publish lock/);
+ assert.ok(!existsSync(publishLockPath(paths)));
+ assert.equal(readFileSync(path.join(bundleDir(paths, "jer"), "index.html"), "utf8"), "<!doctype html>");
+});
+
+test("stamps' commit/branch: ARCHILYZER_COMMIT / ARCHILYZER_BRANCH win over git, each on its own; no repository is null", async () => {
+ const { checkoutInfo } = await import("./stageBodies");
+ const repo = { monorepoRoot: path.resolve(path.dirname(new URL(import.meta.url).pathname), "..", "..") };
+ const fromGit = await checkoutInfo(repo, {});
+ assert.match(fromGit.commit ?? "", /^[0-9a-f]{40}$/);
+ assert.deepEqual(await checkoutInfo(repo, { ARCHILYZER_BRANCH: "main", ARCHILYZER_COMMIT: "abc" }), {
+ commit: "abc",
+ branch: "main",
+ });
+ const branchOnly = await checkoutInfo(repo, { ARCHILYZER_BRANCH: "main" });
+ assert.equal(branchOnly.branch, "main");
+ assert.equal(branchOnly.commit, fromGit.commit);
+ const noRepo = { monorepoRoot: ROOT };
+ assert.deepEqual(await checkoutInfo(noRepo, {}), { commit: null, branch: null });
+ assert.deepEqual(await checkoutInfo(noRepo, { ARCHILYZER_COMMIT: "c1", ARCHILYZER_BRANCH: "main" }), {
+ commit: "c1",
+ branch: "main",
+ });
+});
+
+test("publish deploy all: only a private site and a site with no Pages project are skipped; any other refusal fails the run (exit 1) after the rest are tried", async () => {
+ const { publishDeploy } = await import("../bin/publish");
+ writeSite("mine", { audience: "private", cloudflareProject: "w3c-never-real" });
+ writeSite("noproj");
+ writeSite("unbuilt", { cloudflareProject: "w3c-never-real" });
+ const siteOut = path.join(ROOT, "builds", "site-all");
+ process.env.ARCHILYZER_SITE_OUT = siteOut;
+ const built = (await stamps.readBuiltStamp(paths, "jer"))!;
+ await stamps.writeBuiltStamp(paths, { ...built, branch: "main" });
+ try {
+ const lines: string[] = [];
+ const out = { log: (l: string) => lines.push(l), error: (l: string) => lines.push(l) };
+ // Pages: private and no-project are skipped quietly; jer and unbuilt are
+ // TRIED (jer: never deployed there, w3c-never-real is no real project —
+ // its deploy is refused at wrangler or before; unbuilt: exit 3).
+ const local = await publishDeploy({ target: "all", to: "local", force: true, paths, signal: new AbortController().signal }, out);
+ assert.equal(local, 1, lines.join("\n"));
+ assert.ok(lines.includes("[publish] mine: skipped — " + 'Site "mine" is private (audience: private): it is built for reading on this machine and is never deployed. Build it without deploying, or set its audience to public on its Settings tab'));
+ assert.ok(!lines.some((l) => l.startsWith("[publish] noproj: skipped")), "--to local needs no project");
+ assert.ok(existsSync(path.join(siteOut, "index.html")), "jer was still deployed");
+ const local2 = stamps.deployRecordFor(await stamps.readDeployedFile(paths, "unbuilt"), "local");
+ assert.equal(local2, null, "unbuilt was refused");
+ lines.length = 0;
+ // Only never-deployable sites skipped: an all of nothing but those is exit 0.
+ rmSync(path.join(PINNED.SITES_DIR, "unbuilt"), { recursive: true });
+ rmSync(path.join(PINNED.SITES_DIR, "noproj"), { recursive: true });
+ const onlyOk = await publishDeploy({ target: "all", to: "local", paths, signal: new AbortController().signal }, out);
+ assert.equal(onlyOk, 0, lines.join("\n"));
+ // To Pages: a site with no project is skipped too. (jer's build is taken
+ // away first, so its refusal comes before any wrangler.)
+ writeSite("noproj");
+ rmSync(stamps.builtStampPath(paths, "jer"));
+ lines.length = 0;
+ const pages = await publishDeploy({ target: "all", preview: "r18", paths, signal: new AbortController().signal }, out);
+ assert.equal(pages, 1, "jer, never built, is a failure");
+ assert.ok(lines.includes("[publish] noproj: skipped — no Cloudflare Pages project"), lines.join("\n"));
+ } finally {
+ delete process.env.ARCHILYZER_SITE_OUT;
+ for (const id of ["mine", "noproj", "unbuilt"]) rmSync(path.join(PINNED.SITES_DIR, id), { recursive: true, force: true });
+ }
+});
diff --git a/common/publish/stageRun.ts b/common/publish/stageRun.ts
@@ -0,0 +1,164 @@
+// Running one publish stage (release 18): under the publish lock, with the exit
+// codes every caller reads.
+//
+// 0 ran, or a no-op (the target was fresh)
+// 1 failed
+// 2 usage (a bad flag, an unknown site, a bad preview name)
+// 3 precondition not met ("update the index first", "no build of X", …)
+// 130 cancelled (SIGTERM / SIGINT, before or during the stage)
+//
+// Two ways in. The editor spawns `stageCommand(paths, req)` — the CLI's
+// internal `stage` row — as a `runManagedCommand` job on the `publish` queue:
+// that child (`stageMain`) traps SIGTERM so a Cancel unwinds the stage, and
+// turns on runChildIntoLog's tree-kill so `next build`, wrangler and docker
+// go with it. `archilyzer publish …` calls `runStage` in its own process.
+// Either way the stage takes `<exportBuildsDir>/.publish.lock` first.
+
+import path from "node:path";
+import { killChildTreesNow, setKillChildTrees } from "../jobs/runChild";
+import { getPaths, type Paths } from "../lib/paths";
+import { StageCancelled, StageFailure } from "./stageBodies";
+import { LockWaitCancelled, acquirePublishLock, type LockEnv } from "./stageLock";
+import { STAGES, type StageOutcome, type StageRequest } from "./stages";
+
+export const STAGE_EXIT = {
+ ok: 0,
+ failed: 1,
+ usage: 2,
+ precondition: 3,
+ cancelled: 130,
+} as const;
+
+// The update-index child's heap: the index and stats builds of a large corpus
+// (what export's `build:index` / `build:stats` scripts have always set).
+export const INDEX_HEAP_MB = 8192;
+
+export type StageCommand = {
+ command: string;
+ args: string[];
+ cwd: string;
+ env: Record<string, string | undefined>;
+};
+
+/**
+ * The child the editor spawns for `req`: `<common>/node_modules/.bin/tsx
+ * bin/archilyzer.ts stage <kind> <target> [flags]`, cwd `common/`, the
+ * editor's environment (+ the heap cap for update-index). S3's
+ * `enqueueStage` hands this to `runManagedCommand` as it is.
+ */
+export function stageCommand(
+ paths: Pick<Paths, "monorepoRoot">,
+ req: StageRequest,
+ baseEnv: NodeJS.ProcessEnv = process.env,
+): StageCommand {
+ const commonDir = path.join(paths.monorepoRoot, "common");
+ const env: Record<string, string | undefined> = { ...baseEnv };
+ if (req.kind === "update-index") {
+ env.NODE_OPTIONS = [baseEnv.NODE_OPTIONS?.trim(), `--max-old-space-size=${INDEX_HEAP_MB}`]
+ .filter(Boolean)
+ .join(" ");
+ }
+ return {
+ command: path.join(commonDir, "node_modules", ".bin", "tsx"),
+ args: ["bin/archilyzer.ts", ...STAGES[req.kind].argv(req)],
+ cwd: commonDir,
+ env,
+ };
+}
+
+export type StageRunResult = { code: number; outcome: StageOutcome | null; message: string | null };
+
+function terminal(line: string): void {
+ process.stdout.write(line.endsWith("\n") ? line : `${line}\n`);
+}
+
+/**
+ * Run `req` in this process under the publish lock (waiting for a live
+ * holder). Never throws: the result carries the exit code, the outcome on
+ * 0, and the one sentence a failure or refusal ended on.
+ */
+export async function runStage(
+ req: StageRequest,
+ opts: {
+ paths?: Paths;
+ onLog?: (line: string) => void;
+ signal?: AbortSignal;
+ lockEnv?: LockEnv & { pollMs?: number };
+ } = {},
+): Promise<StageRunResult> {
+ const paths = opts.paths ?? getPaths();
+ const onLog = opts.onLog ?? terminal;
+ const signal = opts.signal ?? new AbortController().signal;
+ const name = `${req.kind} ${req.target}`;
+ let lock;
+ try {
+ lock = await acquirePublishLock(paths, { kind: req.kind, target: req.target }, {
+ ...opts.lockEnv,
+ signal,
+ onLog,
+ });
+ } catch (err) {
+ if (err instanceof LockWaitCancelled) {
+ onLog(`[stage] ${name}: cancelled while waiting for the publish lock\n`);
+ return { code: STAGE_EXIT.cancelled, outcome: null, message: err.message };
+ }
+ const message = (err as Error).message;
+ onLog(`[stage] ${name}: FAILED — ${message}\n`);
+ return { code: STAGE_EXIT.failed, outcome: null, message };
+ }
+ try {
+ const outcome = await STAGES[req.kind].run({ paths, onLog, signal }, req);
+ if (signal.aborted) throw new StageCancelled();
+ onLog(`[stage] ${name}: ${outcome.status === "noop" ? "no-op" : "done"} — ${outcome.summary}\n`);
+ return { code: STAGE_EXIT.ok, outcome, message: null };
+ } catch (err) {
+ if (err instanceof StageCancelled || signal.aborted) {
+ onLog(`[stage] ${name}: cancelled\n`);
+ return { code: STAGE_EXIT.cancelled, outcome: null, message: "cancelled" };
+ }
+ const message = (err as Error).message;
+ if (err instanceof StageFailure) {
+ const word = err.exitCode === STAGE_EXIT.failed ? "FAILED" : "REFUSED";
+ onLog(`[stage] ${name}: ${word} — ${message}\n`);
+ return { code: err.exitCode, outcome: null, message };
+ }
+ onLog(`[stage] ${name}: FAILED — ${(err as Error).stack ?? message}\n`);
+ return { code: STAGE_EXIT.failed, outcome: null, message };
+ } finally {
+ await lock.release();
+ }
+}
+
+// After a first SIGTERM the stage unwinds (its children are signalled); a
+// stage still running this long after is left to the next taker's stale-lock
+// check, and the process exits.
+const CANCEL_GRACE_MS = 15_000;
+
+/**
+ * The stage child's entry (the `stage` row): SIGTERM / SIGINT cancel the stage
+ * — a second one exits at once — and every child it runs is killed as a tree.
+ * Returns the exit code.
+ */
+export async function stageMain(req: StageRequest, opts: { paths?: Paths } = {}): Promise<number> {
+ const ac = new AbortController();
+ // Exiting without the unwind (a second signal, or the grace run out) takes
+ // the detached process groups down first: no orphan `next build`.
+ const exitNow = () => {
+ killChildTreesNow("SIGKILL");
+ process.exit(STAGE_EXIT.cancelled);
+ };
+ const onSignal = () => {
+ if (ac.signal.aborted) exitNow();
+ ac.abort();
+ setTimeout(exitNow, CANCEL_GRACE_MS).unref();
+ };
+ process.on("SIGTERM", onSignal);
+ process.on("SIGINT", onSignal);
+ setKillChildTrees(true);
+ try {
+ return (await runStage(req, { paths: opts.paths, signal: ac.signal })).code;
+ } finally {
+ process.off("SIGTERM", onSignal);
+ process.off("SIGINT", onSignal);
+ }
+}
diff --git a/common/publish/stages.test.ts b/common/publish/stages.test.ts
@@ -0,0 +1,354 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { parseArgv } from "../bin/_parseFlags";
+import type { Paths } from "../lib/paths";
+import { builtStamp, indexStamp } from "./__fixtures__/stamps";
+import type { BuiltStamp, DeployedFile, DeployRecord } from "./stamps";
+import { STAGE_EXIT, stageCommand } from "./stageRun";
+import {
+ STAGES,
+ STAGE_FLAGS,
+ STAGE_KINDS,
+ builtCheckedAt,
+ deployKindOf,
+ parseStageArgs,
+ stageArgv,
+ type Freshness,
+ type NeedsInput,
+ type StageRequest,
+ type TargetState,
+} from "./stages";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common test
+//
+// The publish stages' needs() — one case (at least) per row of the plan's
+// "Stages" table (plans/release-18.md), plus `changedChannels`, `--force` and
+// "code newer is not stale" — and the child's argv, both ways.
+
+function target(over: Partial<TargetState> = {}): TargetState {
+ return {
+ built: null,
+ deployed: null,
+ changedChannels: [],
+ configChangedAt: null,
+ bundleProblem: null,
+ ...over,
+ };
+}
+
+function input(over: Partial<NeedsInput> = {}): NeedsInput {
+ return {
+ index: { stamp: indexStamp(), lastIngestDoneAt: null, configChangedAt: null },
+ sites: { jer: target({ built: builtStamp() }) },
+ hub: target({ built: builtStamp({ target: "_hub", kind: "hub", inputSig: "hub-1" }) }),
+ homepage: {
+ ...target({
+ built: builtStamp({ target: "_homepage", kind: "homepage", inputSig: "s1", sourceCommit: "main-1" }),
+ }),
+ mainHead: "main-1",
+ },
+ ...over,
+ };
+}
+
+const req = (kind: StageRequest["kind"], tgt: string, over: Partial<StageRequest> = {}): StageRequest => ({
+ kind,
+ target: tgt,
+ runId: "run-1",
+ ...over,
+});
+
+const needs = (s: NeedsInput, r: StageRequest): Freshness => STAGES[r.kind].needs(s, r);
+const state = (f: Freshness) => f.state;
+const reason = (f: Freshness) => (f.state === "fresh" ? "" : f.reason);
+
+function deployed(over: Partial<DeployedFile> = {}, rec: Partial<DeployRecord> = {}): DeployedFile {
+ const r: DeployRecord = {
+ builtStampId: "b1",
+ builtAt: 3_000,
+ kind: "production",
+ url: null,
+ at: 4_000,
+ liveCheck: null,
+ ...rec,
+ };
+ return { v: 1, target: "jer", production: r, previews: {}, ...over };
+}
+
+test("the table: seven stages, each a publish-<kind> job on the publish queue", () => {
+ assert.deepEqual(Object.keys(STAGES), [...STAGE_KINDS]);
+ for (const k of STAGE_KINDS) {
+ assert.equal(STAGES[k].kind, k);
+ assert.equal(STAGES[k].jobKind, `publish-${k}`);
+ assert.equal(STAGES[k].queueKey, "publish");
+ }
+});
+
+// --- update-index -------------------------------------------------------------
+
+test("update-index: fresh when a stamp exists and nothing is newer than its scan", () => {
+ assert.equal(state(needs(input(), req("update-index", "_index"))), "fresh");
+ const none = input({ index: { stamp: null, lastIngestDoneAt: null, configChangedAt: null } });
+ assert.equal(reason(needs(none, req("update-index", "_index"))), "no index stamp yet");
+ const data = input({ index: { stamp: indexStamp(), lastIngestDoneAt: 1_001, configChangedAt: null } });
+ assert.equal(reason(needs(data, req("update-index", "_index"))), "new data since the last index");
+ const older = input({ index: { stamp: indexStamp(), lastIngestDoneAt: 999, configChangedAt: 1_000 } });
+ assert.equal(state(needs(older, req("update-index", "_index"))), "fresh", "not NEWER than scannedAt");
+ const cfg = input({ index: { stamp: indexStamp(), lastIngestDoneAt: null, configChangedAt: 5_000 } });
+ assert.match(reason(needs(cfg, req("update-index", "_index"))), /a config file changed/);
+ assert.equal(reason(needs(input(), req("update-index", "_index", { force: true }))), "forced");
+});
+
+// --- build-site <id> ----------------------------------------------------------
+
+test("build-site: blocked 'update the index first' with no stamp — even forced", () => {
+ const none = input({ index: { stamp: null, lastIngestDoneAt: null, configChangedAt: null } });
+ assert.deepEqual(needs(none, req("build-site", "jer")), { state: "blocked", reason: "update the index first" });
+ assert.equal(state(needs(none, req("build-site", "jer", { force: true }))), "blocked");
+});
+
+test("build-site: fresh when the built inputSig is the stamp's, nothing changed and the bundle is sound", () => {
+ assert.equal(state(needs(input(), req("build-site", "jer"))), "fresh");
+});
+
+test("build-site: changed channels make it stale before any index runs, named", () => {
+ const s = input({ sites: { jer: target({ built: builtStamp(), changedChannels: ["a", "b", "c", "d", "e"] }) } });
+ assert.equal(reason(needs(s, req("build-site", "jer"))), "5 channels changed (a, b, c, d, …)");
+ const one = input({ sites: { jer: target({ built: builtStamp(), changedChannels: ["a"] }) } });
+ assert.equal(reason(needs(one, req("build-site", "jer"))), "1 channel changed (a)");
+});
+
+test("build-site: an inputSig mismatch is 'data changed'; a config change, a bad bundle, no build are stale", () => {
+ const sig = input({ sites: { jer: target({ built: builtStamp({ inputSig: "older" }) }) } });
+ assert.equal(reason(needs(sig, req("build-site", "jer"))), "data changed");
+ const cfg = input({ sites: { jer: target({ built: builtStamp(), configChangedAt: 3_001 }) } });
+ assert.equal(reason(needs(cfg, req("build-site", "jer"))), "config changed");
+ const bad = input({ sites: { jer: target({ built: builtStamp(), bundleProblem: "holds a build of x" }) } });
+ assert.equal(reason(needs(bad, req("build-site", "jer"))), "holds a build of x");
+ const never = input({ sites: { jer: target() } });
+ assert.equal(reason(needs(never, req("build-site", "jer"))), "never built");
+});
+
+test("build-site: --force builds a fresh site; a code change alone is NOT stale", () => {
+ assert.equal(reason(needs(input(), req("build-site", "jer", { force: true }))), "forced");
+ const code = input({ sites: { jer: target({ built: builtStamp({ commit: "an-older-commit", branch: "x" }) }) } });
+ assert.equal(state(needs(code, req("build-site", "jer"))), "fresh");
+});
+
+test("build-site: indexAfter waits for this run's index; an unknown site or one the index never saw is blocked", () => {
+ assert.equal(state(needs(input(), req("build-site", "jer", { indexAfter: 2_000 }))), "fresh");
+ assert.match(reason(needs(input(), req("build-site", "jer", { indexAfter: 2_001 }))), /waiting for the index update/);
+ assert.equal(reason(needs(input(), req("build-site", "nope"))), 'no site "nope"');
+ const unseen = input({ sites: { jer: target({ built: builtStamp() }), ani: target() } });
+ assert.match(reason(needs(unseen, req("build-site", "ani"))), /the index has not seen site "ani" — update the index first/);
+});
+
+// --- build-site _all --runner docker ------------------------------------------
+
+test("build-site _all: per site, as above", () => {
+ const s = input({
+ index: {
+ stamp: indexStamp({ sites: { jer: indexStamp().sites.jer, ani: { siteFp: null, statsFp: null, inputSig: "x" } } }),
+ lastIngestDoneAt: null,
+ configChangedAt: null,
+ },
+ sites: { jer: target({ built: builtStamp() }), ani: target() },
+ });
+ assert.equal(reason(needs(s, req("build-site", "_all", { runner: "docker" }))), "1 of 2 sites to build (ani)");
+ const allFresh = input();
+ assert.equal(state(needs(allFresh, req("build-site", "_all", { runner: "docker" }))), "fresh");
+ const none = input({ index: { stamp: null, lastIngestDoneAt: null, configChangedAt: null } });
+ assert.equal(state(needs(none, req("build-site", "_all"))), "blocked");
+});
+
+// --- deploy-site --------------------------------------------------------------
+
+test("deploy-site: fresh exactly when the record for that kind/branch names the built stamp", () => {
+ const prod = input({ sites: { jer: target({ built: builtStamp(), deployed: deployed() }) } });
+ assert.equal(state(needs(prod, req("deploy-site", "jer"))), "fresh");
+ const newer = input({ sites: { jer: target({ built: builtStamp({ stampId: "b2" }), deployed: deployed() }) } });
+ assert.equal(reason(needs(newer, req("deploy-site", "jer"))), "a newer build is not deployed");
+ // The production record says nothing about a preview, or the local copy.
+ assert.equal(reason(needs(prod, req("deploy-site", "jer", { preview: "r18" }))), 'never deployed to preview "r18"');
+ assert.equal(reason(needs(prod, req("deploy-site", "jer", { to: "local" }))), "never deployed (local)");
+ const pv = input({
+ sites: { jer: target({ built: builtStamp(), deployed: deployed({ previews: { r18: deployed().production! } }) }) },
+ });
+ assert.equal(state(needs(pv, req("deploy-site", "jer", { preview: "r18" }))), "fresh");
+ assert.equal(reason(needs(prod, req("deploy-site", "jer", { force: true }))), "forced");
+});
+
+test("deploy-site: refused with no build, a private site, no project (pages only), a bad bundle, builtAfter", () => {
+ const never = input({ sites: { jer: target() } });
+ assert.equal(reason(needs(never, req("deploy-site", "jer"))), "no build of jer — archilyzer publish build jer");
+ const priv = input({ sites: { jer: target({ built: builtStamp(), deployProblem: "Site jer is private" }) } });
+ assert.equal(reason(needs(priv, req("deploy-site", "jer", { to: "local" }))), "Site jer is private");
+ const noProject = input({ sites: { jer: target({ built: builtStamp(), pagesProblem: "no project" }) } });
+ assert.equal(reason(needs(noProject, req("deploy-site", "jer"))), "no project");
+ assert.equal(state(needs(noProject, req("deploy-site", "jer", { to: "local" }))), "stale", "local needs no project");
+ const bad = input({ sites: { jer: target({ built: builtStamp(), bundleProblem: "torn" }) } });
+ assert.deepEqual(needs(bad, req("deploy-site", "jer")), { state: "blocked", reason: "torn" });
+ assert.match(reason(needs(input(), req("deploy-site", "jer", { builtAfter: 3_001 }))), /waiting for the build of jer/);
+ assert.equal(state(needs(never, req("deploy-site", "jer", { force: true }))), "blocked", "force never deploys nothing");
+});
+
+test("deploy-site: production refuses a build made on another branch; a preview of it is fine", () => {
+ const off = input({ sites: { jer: target({ built: builtStamp({ branch: "r18/stage-core" }) }) } });
+ assert.match(reason(needs(off, req("deploy-site", "jer"))), /built from branch "r18\/stage-core"; production ships only a build of main/);
+ assert.equal(state(needs(off, req("deploy-site", "jer", { preview: "r18" }))), "stale");
+ const detached = input({ sites: { jer: target({ built: builtStamp({ branch: null }) }) } });
+ assert.match(
+ reason(needs(detached, req("deploy-site", "jer"))),
+ /built with no branch recorded \(a detached HEAD, or an image built without ARCHILYZER_BRANCH\); production ships only a build of main/,
+ "no branch is refused like another branch",
+ );
+ assert.equal(state(needs(detached, req("deploy-site", "jer", { preview: "r18" }))), "stale");
+});
+
+test("deploy-site under builtAfter: a run's NO-OP build (the bundle still matches the index this run updated) does not hold the deploy", () => {
+ // The run started at 1_500; its index ran at 2_000 (stamp.builtAt); the
+ // site's build was a no-op, so built.builtAt (500) predates the run.
+ const old = builtStamp({ builtAt: 500 });
+ const s = input({ sites: { jer: target({ built: old }) } });
+ assert.equal(state(needs(s, req("deploy-site", "jer", { builtAfter: 1_500 }))), "stale", "same inputs, same bundle");
+ // …but not when the bundle does NOT match the current index (the build failed, say).
+ const drifted = input({ sites: { jer: target({ built: builtStamp({ builtAt: 500, inputSig: "older" }) }) } });
+ assert.match(reason(needs(drifted, req("deploy-site", "jer", { builtAfter: 1_500 }))), /waiting for the build of jer/);
+ // …nor when the index itself has not run since the run began.
+ assert.match(reason(needs(s, req("deploy-site", "jer", { builtAfter: 2_500 }))), /waiting for the build of jer/);
+ // A no-op build's checkedAt counts as well.
+ const checked = input({ sites: { jer: target({ built: builtStamp({ builtAt: 500, checkedAt: 3_000, inputSig: "older" }) }) } });
+ assert.equal(state(needs(checked, req("deploy-site", "jer", { builtAfter: 2_500 }))), "stale");
+ // The hub and the homepage judge "current" by their own signatures.
+ const hub = input({ hub: target({ built: builtStamp({ target: "_hub", kind: "hub", inputSig: "hub-1", builtAt: 500 }) }) });
+ assert.equal(state(needs(hub, req("deploy-hub", "_hub", { builtAfter: 1_500 }))), "stale");
+ const home = input();
+ home.homepage.built = { ...(home.homepage.built as BuiltStamp), builtAt: 500 };
+ assert.equal(state(needs(home, req("deploy-homepage", "_homepage", { builtAfter: 1_500 }))), "stale");
+});
+
+test("changedChannels and config changes are measured against max(builtAt, checkedAt)", () => {
+ assert.equal(builtCheckedAt(builtStamp({ builtAt: 3_000 })), 3_000);
+ assert.equal(builtCheckedAt(builtStamp({ builtAt: 3_000, checkedAt: 9_000 })), 9_000);
+ const cfg = input({ sites: { jer: target({ built: builtStamp({ checkedAt: 9_000 }), configChangedAt: 5_000 }) } });
+ assert.equal(state(needs(cfg, req("build-site", "jer"))), "fresh", "a config change older than the last check");
+});
+
+// --- the hub ------------------------------------------------------------------
+
+test("build-hub: fresh when built.inputSig is the stamp's hubSig", () => {
+ assert.equal(state(needs(input(), req("build-hub", "_hub"))), "fresh");
+ const moved = input({ index: { stamp: indexStamp({ hubSig: "hub-2" }), lastIngestDoneAt: null, configChangedAt: null } });
+ assert.equal(reason(needs(moved, req("build-hub", "_hub"))), "the index or the pool it lists changed");
+ const ch = input({ hub: target({ built: builtStamp({ target: "_hub", kind: "hub", inputSig: "hub-1" }), changedChannels: ["x"] }) });
+ assert.equal(reason(needs(ch, req("build-hub", "_hub"))), "1 channel changed (x)");
+ const none = input({ index: { stamp: null, lastIngestDoneAt: null, configChangedAt: null } });
+ assert.equal(reason(needs(none, req("build-hub", "_hub"))), "update the index first");
+});
+
+test("deploy-hub: as deploy-site", () => {
+ const s = input({ hub: target({ built: builtStamp({ target: "_hub", kind: "hub" }), deployed: deployed({ target: "_hub" }) }) });
+ assert.equal(state(needs(s, req("deploy-hub", "_hub"))), "fresh");
+ assert.equal(reason(needs(input({ hub: target() }), req("deploy-hub", "_hub"))), "no build of the hub — archilyzer publish hub");
+ const noProject = input({ hub: target({ built: builtStamp(), pagesProblem: "The hub has no Cloudflare Pages project" }) });
+ assert.match(reason(needs(noProject, req("deploy-hub", "_hub"))), /no Cloudflare Pages project/);
+});
+
+// --- the homepage -------------------------------------------------------------
+
+test("build-homepage: fresh when built from the current index stamp and main has not moved", () => {
+ assert.equal(state(needs(input(), req("build-homepage", "_homepage"))), "fresh");
+ const idx = input({ index: { stamp: indexStamp({ stampId: "s2" }), lastIngestDoneAt: null, configChangedAt: null } });
+ assert.equal(reason(needs(idx, req("build-homepage", "_homepage"))), "the index was updated");
+ const moved = input();
+ moved.homepage.mainHead = "main-2";
+ assert.equal(reason(needs(moved, req("build-homepage", "_homepage"))), "main has moved since the source was published");
+ const noRepo = input();
+ noRepo.homepage.mainHead = null;
+ noRepo.homepage.built = { ...(noRepo.homepage.built as BuiltStamp), sourceCommit: null };
+ assert.equal(state(needs(noRepo, req("build-homepage", "_homepage"))), "fresh", "no repository: the index decides");
+});
+
+test("deploy-homepage: as deploy-site", () => {
+ const s = input();
+ s.homepage.deployed = deployed({ target: "_homepage" });
+ assert.equal(state(needs(s, req("deploy-homepage", "_homepage"))), "fresh");
+ s.homepage.built = null;
+ assert.equal(reason(needs(s, req("deploy-homepage", "_homepage"))), "no build of the homepage — archilyzer publish homepage");
+});
+
+test("deployKindOf: --to local, --preview, else production", () => {
+ assert.equal(deployKindOf({}), "production");
+ assert.equal(deployKindOf({ preview: "x" }), "preview");
+ assert.equal(deployKindOf({ to: "local" }), "local");
+ assert.equal(deployKindOf({ to: "pages" }), "production");
+});
+
+// --- argv -------------------------------------------------------------------
+
+test("argv: the child's command line, pinned, and parsed back to the same request", () => {
+ const r: StageRequest = {
+ kind: "deploy-site",
+ target: "jer",
+ runId: "run-9",
+ preview: "r18",
+ force: true,
+ builtAfter: 1_700_000_000_000,
+ };
+ assert.deepEqual(STAGES["deploy-site"].argv(r), [
+ "stage", "deploy-site", "jer", "--run-id", "run-9", "--preview", "r18", "--force", "--built-after", "1700000000000",
+ ]);
+ const booleans = Object.entries(STAGE_FLAGS).filter(([, k]) => k === "boolean").map(([n]) => n);
+ for (const sample of [
+ r,
+ { kind: "update-index", target: "_index", runId: "a" },
+ { kind: "build-site", target: "_all", runId: "b", runner: "docker", skipArchives: true, indexAfter: 5 },
+ { kind: "build-site", target: "ani", runId: "c", allowMissingMedia: true },
+ { kind: "deploy-homepage", target: "_homepage", runId: "d", to: "local" },
+ ] as StageRequest[]) {
+ const { positionals, flags } = parseArgv(stageArgv(sample), booleans);
+ assert.equal(positionals[0], "stage");
+ assert.deepEqual(parseStageArgs(positionals.slice(1), flags), sample);
+ }
+});
+
+test("parseStageArgs refuses what the child must never guess", () => {
+ const err = (pos: string[], flags: Record<string, string | boolean> = { "run-id": "r" }) => {
+ const out = parseStageArgs(pos, flags);
+ return "error" in out ? out.error : "";
+ };
+ assert.match(err(["build"]), /which stage\?/);
+ assert.match(err(["build-site"]), /which target\?/);
+ assert.match(err(["build-site", "jer"], {}), /--run-id is required/);
+ assert.match(err(["update-index", "jer"]), /the target is _index/);
+ assert.match(err(["build-hub", "_index"]), /the target is _hub/);
+ assert.match(err(["deploy-site", "_all"]), /the target is a site id/);
+ assert.match(err(["deploy-site", "jer"], { "run-id": "r", to: "ftp" }), /--to is pages or local/);
+ assert.match(err(["build-site", "_all"], { "run-id": "r", runner: "podman" }), /--runner is local or docker/);
+ assert.match(err(["build-site", "jer"], { "run-id": "r", "index-after": "soon" }), /--index-after is a time/);
+ assert.match(err(["deploy-site", "jer"], { "run-id": "r", preview: "x", to: "local" }), /two different deploys/);
+ assert.match(err(["build-site", "../etc"]), /which target\?/);
+});
+
+// --- the spawned command ------------------------------------------------------
+
+test("stageCommand: common's tsx running bin/archilyzer.ts stage …, cwd common/, the heap cap for update-index only", () => {
+ const paths = { monorepoRoot: "/repo" } as Paths;
+ const idx = stageCommand(paths, { kind: "update-index", target: "_index", runId: "r1" }, { PATH: "/bin" });
+ assert.deepEqual(idx, {
+ command: "/repo/common/node_modules/.bin/tsx",
+ args: ["bin/archilyzer.ts", "stage", "update-index", "_index", "--run-id", "r1"],
+ cwd: "/repo/common",
+ env: { PATH: "/bin", NODE_OPTIONS: "--max-old-space-size=8192" },
+ });
+ const kept = stageCommand(paths, { kind: "update-index", target: "_index", runId: "r1" }, { NODE_OPTIONS: "--trace-warnings" });
+ assert.equal(kept.env.NODE_OPTIONS, "--trace-warnings --max-old-space-size=8192");
+ const build = stageCommand(paths, { kind: "build-site", target: "jer", runId: "r1" }, { PATH: "/bin" });
+ assert.deepEqual(build.env, { PATH: "/bin" });
+ assert.deepEqual(build.args, ["bin/archilyzer.ts", "stage", "build-site", "jer", "--run-id", "r1"]);
+});
+
+test("the exit codes", () => {
+ assert.deepEqual(STAGE_EXIT, { ok: 0, failed: 1, usage: 2, precondition: 3, cancelled: 130 });
+});
diff --git a/common/publish/stages.ts b/common/publish/stages.ts
@@ -0,0 +1,419 @@
+// The publish stages (release 18) — the ONLY module that knows all of them.
+//
+// Publishing is seven independent, queueable stages driven by on-disk state
+// (publish/stamps.ts), like the ingest lanes: one index build shared by every
+// site build, then builds and deploys one at a time. Each stage is:
+//
+// needs(input, req) PURE: is the target fresh, stale (and why), or blocked?
+// argv(req) the child argv the editor spawns: ["stage", kind, target, …]
+// run(ctx, req) the body, in that child or in the CLI's own process
+//
+// `needs()` reads a `NeedsInput` — the minimal PublishStatus-shaped input
+// defined here. S3's status view (common/views/publishStatus.ts) satisfies it
+// from the stamps plus the job metas (`changedChannels`) and config mtimes; the
+// stage child builds one from disk alone (`readNeedsInput`, stageBodies.ts),
+// because ordering is enforced ON DISK: a stage whose precondition is not met
+// when it starts exits 3, whatever the queue believed when it enqueued it.
+//
+// Nothing here loads LMDB, next or the AWS SDK: the bodies are imported lazily.
+
+import type { Paths } from "../lib/paths";
+import {
+ ALL_TARGET,
+ HOMEPAGE_TARGET,
+ HUB_TARGET,
+ INDEX_TARGET,
+ deployRecordFor,
+ type BuiltStamp,
+ type DeployKind,
+ type DeployedFile,
+ type IndexStamp,
+} from "./stamps";
+
+export type StageKind =
+ | "update-index"
+ | "build-site"
+ | "deploy-site"
+ | "build-hub"
+ | "deploy-hub"
+ | "build-homepage"
+ | "deploy-homepage";
+
+export const STAGE_KINDS: readonly StageKind[] = [
+ "update-index",
+ "build-site",
+ "deploy-site",
+ "build-hub",
+ "deploy-hub",
+ "build-homepage",
+ "deploy-homepage",
+];
+
+export function isStageKind(v: unknown): v is StageKind {
+ return typeof v === "string" && (STAGE_KINDS as readonly string[]).includes(v);
+}
+
+export type StageRequest = {
+ kind: StageKind;
+ // "_index" | siteId | "_all" | "_hub" | "_homepage"
+ target: string;
+ runId: string;
+ preview?: string;
+ to?: "pages" | "local";
+ runner?: "local" | "docker";
+ force?: boolean;
+ skipArchives?: boolean;
+ // On-disk preconditions of a run: the index stamp (for a build) or the
+ // target's built stamp (for a deploy) must be at least this new (ms).
+ indexAfter?: number;
+ builtAfter?: number;
+ // `build site --allow-missing-media` (a report citation with no prepared
+ // media is let through compose). Not part of the plan's shape; optional.
+ allowMissingMedia?: boolean;
+};
+
+export type Freshness =
+ | { state: "fresh" }
+ | { state: "stale"; reason: string }
+ | { state: "blocked"; reason: string };
+
+export type StageOutcome = { status: "ran" | "noop"; stamp: string; summary: string };
+
+export type StageContext = {
+ paths: Paths;
+ onLog: (line: string) => void;
+ signal: AbortSignal;
+};
+
+export type Stage = {
+ kind: StageKind;
+ label: string;
+ jobKind: `publish-${StageKind}`;
+ queueKey: "publish";
+ needs(s: NeedsInput, r: StageRequest): Freshness;
+ argv(r: StageRequest): string[];
+ run(ctx: StageContext, r: StageRequest): Promise<StageOutcome>;
+};
+
+// ---------------------------------------------------------------------------
+// The input needs() reads (S3's PublishStatus satisfies it)
+// ---------------------------------------------------------------------------
+
+export type TargetState = {
+ built: BuiltStamp | null;
+ deployed: DeployedFile | null;
+ // Member channels (a site's, or every listed site's for the hub) with an
+ // ingest job ended `done` after `builtCheckedAt(built)` — the later of
+ // `builtAt` and `checkedAt`, so a no-op build clears the chip.
+ changedChannels: string[];
+ // The newest mtime (ms) of a config file this target's build reads (its
+ // site.json, tags.json, search-aliases.json, duplicates*.json), or null.
+ configChangedAt: number | null;
+ // What is wrong with the bundle on disk (builtBundleProblem / builtHubProblem
+ // / builtHomepageProblem), or null. Only asked when `built` is set.
+ bundleProblem: string | null;
+ // Why it is never deployed anywhere (a private site), or null.
+ deployProblem?: string | null;
+ // Why it cannot go to Cloudflare Pages (no project), or null.
+ pagesProblem?: string | null;
+};
+
+export type NeedsInput = {
+ index: {
+ stamp: IndexStamp | null;
+ // When the newest drainable ingest job ended `done` (ms), or null.
+ lastIngestDoneAt: number | null;
+ // The newest mtime (ms) of an index input config file (tags.json,
+ // search-aliases.json, duplicates*.json, sites/*/site.json,
+ // homepage.json, the settings file, the charts config), or null.
+ configChangedAt: number | null;
+ };
+ sites: Record<string, TargetState>;
+ hub: TargetState;
+ homepage: TargetState & {
+ // `main`'s HEAD where a repository is reachable, else null.
+ mainHead: string | null;
+ };
+};
+
+// ---------------------------------------------------------------------------
+// needs()
+// ---------------------------------------------------------------------------
+
+const FRESH: Freshness = { state: "fresh" };
+const stale = (reason: string): Freshness => ({ state: "stale", reason });
+const blocked = (reason: string): Freshness => ({ state: "blocked", reason });
+
+export const UPDATE_INDEX_FIRST = "update the index first";
+
+/** The deploy kind a request names: --to local, --preview <b>, else production. */
+export function deployKindOf(r: Pick<StageRequest, "to" | "preview">): DeployKind {
+ if (r.to === "local") return "local";
+ return r.preview ? "preview" : "production";
+}
+
+function namesList(slugs: string[], max = 4): string {
+ const shown = slugs.slice(0, max).join(", ");
+ return slugs.length > max ? `${shown}, …` : shown;
+}
+
+function needsIndex(s: NeedsInput, r: StageRequest): Freshness {
+ const stamp = s.index.stamp;
+ if (!stamp) return stale("no index stamp yet");
+ if (r.force) return stale("forced");
+ if (s.index.lastIngestDoneAt !== null && s.index.lastIngestDoneAt > stamp.scannedAt) {
+ return stale("new data since the last index");
+ }
+ if (s.index.configChangedAt !== null && s.index.configChangedAt > stamp.scannedAt) {
+ return stale("a config file changed since the last index");
+ }
+ return FRESH;
+}
+
+// The stamp a build needs, or why it is blocked.
+function indexGate(s: NeedsInput, r: StageRequest): Freshness | IndexStamp {
+ const stamp = s.index.stamp;
+ if (!stamp) return blocked(UPDATE_INDEX_FIRST);
+ if (r.indexAfter !== undefined && stamp.builtAt < r.indexAfter) {
+ return blocked("waiting for the index update this run started");
+ }
+ return stamp;
+}
+
+// Stale reasons shared by the three builds, after the target's own signature.
+/**
+ * When the bundle was last known to match its inputs: built, or found fresh
+ * by a later no-op build (`checkedAt`). What `changedChannels` and a config
+ * change are measured against.
+ */
+export function builtCheckedAt(built: BuiltStamp): number {
+ return Math.max(built.builtAt, built.checkedAt ?? 0);
+}
+
+function builtStale(t: TargetState, sigMatches: boolean, sigReason: string): Freshness {
+ const built = t.built!;
+ if (t.changedChannels.length > 0) {
+ const n = t.changedChannels.length;
+ return stale(`${n} channel${n === 1 ? "" : "s"} changed (${namesList(t.changedChannels)})`);
+ }
+ if (t.configChangedAt !== null && t.configChangedAt > builtCheckedAt(built)) return stale("config changed");
+ if (!sigMatches) return stale(sigReason);
+ if (t.bundleProblem) return stale(t.bundleProblem);
+ return FRESH;
+}
+
+function needsBuildSite(s: NeedsInput, r: StageRequest): Freshness {
+ const gate = indexGate(s, r);
+ if ("state" in gate) return gate;
+ if (r.target === ALL_TARGET) {
+ const ids = Object.keys(s.sites).sort();
+ const staleIds = ids.filter((id) => needsBuildSite(s, { ...r, target: id }).state !== "fresh");
+ if (staleIds.length === 0) return FRESH;
+ return stale(`${staleIds.length} of ${ids.length} sites to build (${namesList(staleIds)})`);
+ }
+ const t = s.sites[r.target];
+ if (!t) return blocked(`no site "${r.target}"`);
+ const entry = gate.sites[r.target];
+ if (!entry) return blocked(`the index has not seen site "${r.target}" — ${UPDATE_INDEX_FIRST}`);
+ if (r.force) return stale("forced");
+ if (!t.built) return stale("never built");
+ return builtStale(t, t.built.inputSig === entry.inputSig, "data changed");
+}
+
+function needsBuildHub(s: NeedsInput, r: StageRequest): Freshness {
+ const gate = indexGate(s, r);
+ if ("state" in gate) return gate;
+ if (r.force) return stale("forced");
+ if (!s.hub.built) return stale("never built");
+ return builtStale(s.hub, s.hub.built.inputSig === gate.hubSig, "the index or the pool it lists changed");
+}
+
+function needsBuildHomepage(s: NeedsInput, r: StageRequest): Freshness {
+ const gate = indexGate(s, r);
+ if ("state" in gate) return gate;
+ const h = s.homepage;
+ if (r.force) return stale("forced");
+ if (!h.built) return stale("never built");
+ if (h.built.indexStampId !== gate.stampId) return stale("the index was updated");
+ if (h.mainHead !== null && (h.built.sourceCommit ?? null) !== h.mainHead) {
+ return stale("main has moved since the source was published");
+ }
+ if (h.bundleProblem) return stale(h.bundleProblem);
+ return FRESH;
+}
+
+// Is `built` what the CURRENT index would build? (Same inputs, same bundle.)
+function builtFromCurrentIndex(s: NeedsInput, kind: "site" | "hub" | "homepage", id: string): boolean {
+ const stamp = s.index.stamp;
+ const built = kind === "site" ? s.sites[id]?.built : kind === "hub" ? s.hub.built : s.homepage.built;
+ if (!stamp || !built) return false;
+ if (kind === "site") return stamp.sites[id] !== undefined && built.inputSig === stamp.sites[id].inputSig;
+ if (kind === "hub") return built.inputSig === stamp.hubSig;
+ return built.indexStampId === stamp.stampId;
+}
+
+function needsDeploy(
+ t: TargetState | undefined,
+ name: string,
+ buildCmd: string,
+ r: StageRequest,
+ // The bundle matches the current index, and that index ran at or after
+ // `builtAfter` (a run's no-op build: nothing was rebuilt because nothing
+ // needed to be).
+ currentSince: (after: number) => boolean,
+): Freshness {
+ if (!t) return blocked(`no site "${name}"`);
+ const kind = deployKindOf(r);
+ const never = t.deployProblem ?? (kind === "local" ? null : (t.pagesProblem ?? null));
+ if (never) return blocked(never);
+ const built = t.built;
+ if (!built) return blocked(`no build of ${name} — ${buildCmd}`);
+ if (
+ r.builtAfter !== undefined &&
+ builtCheckedAt(built) < r.builtAfter &&
+ !currentSince(r.builtAfter)
+ ) {
+ return blocked(`waiting for the build of ${name} this run started`);
+ }
+ if (t.bundleProblem) return blocked(t.bundleProblem);
+ // Production ships only a build of main. A null branch (a detached HEAD, or
+ // an image built without ARCHILYZER_BRANCH) is refused the same way.
+ if (kind === "production" && built.branch !== "main") {
+ return blocked(
+ built.branch === null
+ ? `${name} was built with no branch recorded (a detached HEAD, or an image built without ARCHILYZER_BRANCH); production ships only a build of main (deploy it as a preview)`
+ : `${name} was built from branch "${built.branch}"; production ships only a build of main (deploy it as a preview)`,
+ );
+ }
+ if (r.force) return stale("forced");
+ const rec = deployRecordFor(t.deployed, kind, r.preview);
+ if (!rec) return stale(kind === "preview" ? `never deployed to preview "${r.preview}"` : `never deployed (${kind})`);
+ if (rec.builtStampId !== built.stampId) return stale("a newer build is not deployed");
+ return FRESH;
+}
+
+// ---------------------------------------------------------------------------
+// argv (the child's command line) and its parser
+// ---------------------------------------------------------------------------
+
+export function stageArgv(r: StageRequest): string[] {
+ const out = ["stage", r.kind, r.target, "--run-id", r.runId];
+ if (r.preview) out.push("--preview", r.preview);
+ if (r.to) out.push("--to", r.to);
+ if (r.runner) out.push("--runner", r.runner);
+ if (r.force) out.push("--force");
+ if (r.skipArchives) out.push("--skip-archives");
+ if (r.allowMissingMedia) out.push("--allow-missing-media");
+ if (r.indexAfter !== undefined) out.push("--index-after", String(r.indexAfter));
+ if (r.builtAfter !== undefined) out.push("--built-after", String(r.builtAfter));
+ return out;
+}
+
+/** The flags the `stage` row accepts (archilyzer.ts), by kind. */
+export const STAGE_FLAGS = {
+ "run-id": "string",
+ preview: "string",
+ to: "string",
+ runner: "string",
+ force: "boolean",
+ "skip-archives": "boolean",
+ "allow-missing-media": "boolean",
+ "index-after": "string",
+ "built-after": "string",
+} as const;
+
+const TARGET_RE = /^(?:_index|_all|_hub|_homepage|[a-z0-9][a-z0-9-]*)$/;
+
+/** The default target of a kind that has only one. */
+export function fixedTarget(kind: StageKind): string | null {
+ if (kind === "update-index") return INDEX_TARGET;
+ if (kind === "build-hub" || kind === "deploy-hub") return HUB_TARGET;
+ if (kind === "build-homepage" || kind === "deploy-homepage") return HOMEPAGE_TARGET;
+ return null;
+}
+
+/**
+ * The StageRequest a `stage <kind> <target> [flags]` command line names, or
+ * the usage problem. The inverse of `stageArgv`.
+ */
+export function parseStageArgs(
+ positionals: string[],
+ flags: Record<string, string | boolean | undefined>,
+): StageRequest | { error: string } {
+ const [kind, target] = positionals;
+ if (!isStageKind(kind)) return { error: `stage: which stage? one of ${STAGE_KINDS.join(", ")}` };
+ if (!target || !TARGET_RE.test(target)) return { error: `stage ${kind}: which target?` };
+ const fixed = fixedTarget(kind);
+ if (fixed !== null && target !== fixed) return { error: `stage ${kind}: the target is ${fixed}` };
+ if (fixed === null && (target.startsWith("_") && !(kind === "build-site" && target === ALL_TARGET))) {
+ return { error: `stage ${kind}: the target is a site id` };
+ }
+ const runId = typeof flags["run-id"] === "string" ? flags["run-id"] : "";
+ if (!runId) return { error: `stage ${kind}: --run-id is required` };
+ const r: StageRequest = { kind, target, runId };
+ if (typeof flags.preview === "string") r.preview = flags.preview;
+ if (flags.to !== undefined) {
+ if (flags.to !== "pages" && flags.to !== "local") return { error: `stage ${kind}: --to is pages or local` };
+ r.to = flags.to;
+ }
+ if (flags.runner !== undefined) {
+ if (flags.runner !== "local" && flags.runner !== "docker") return { error: `stage ${kind}: --runner is local or docker` };
+ r.runner = flags.runner;
+ }
+ if (flags.force === true) r.force = true;
+ if (flags["skip-archives"] === true) r.skipArchives = true;
+ if (flags["allow-missing-media"] === true) r.allowMissingMedia = true;
+ for (const [flag, key] of [
+ ["index-after", "indexAfter"],
+ ["built-after", "builtAfter"],
+ ] as const) {
+ const v = flags[flag];
+ if (v === undefined) continue;
+ const n = typeof v === "string" ? Number(v) : NaN;
+ if (!Number.isFinite(n)) return { error: `stage ${kind}: --${flag} is a time in ms` };
+ r[key] = n;
+ }
+ if (r.preview && r.to === "local") return { error: `stage ${kind}: --preview and --to local are two different deploys` };
+ return r;
+}
+
+// ---------------------------------------------------------------------------
+// The table
+// ---------------------------------------------------------------------------
+
+function stage(
+ kind: StageKind,
+ label: string,
+ needs: (s: NeedsInput, r: StageRequest) => Freshness,
+): Stage {
+ return {
+ kind,
+ label,
+ jobKind: `publish-${kind}`,
+ queueKey: "publish",
+ needs,
+ argv: stageArgv,
+ run: async (ctx, r) => (await import("./stageBodies")).runStageBody(ctx, r),
+ };
+}
+
+export const STAGES: Record<StageKind, Stage> = {
+ "update-index": stage("update-index", "Update the index", needsIndex),
+ "build-site": stage("build-site", "Build site", needsBuildSite),
+ "deploy-site": stage("deploy-site", "Deploy site", (s, r) =>
+ needsDeploy(s.sites[r.target], r.target, `archilyzer publish build ${r.target}`, r, (after) =>
+ builtFromCurrentIndex(s, "site", r.target) && (s.index.stamp?.builtAt ?? 0) >= after)),
+ "build-hub": stage("build-hub", "Build hub", needsBuildHub),
+ "deploy-hub": stage("deploy-hub", "Deploy hub", (s, r) =>
+ needsDeploy(s.hub, "the hub", "archilyzer publish hub", r, (after) =>
+ builtFromCurrentIndex(s, "hub", "_hub") && (s.index.stamp?.builtAt ?? 0) >= after)),
+ "build-homepage": stage("build-homepage", "Build homepage", needsBuildHomepage),
+ "deploy-homepage": stage("deploy-homepage", "Deploy homepage", (s, r) =>
+ needsDeploy(s.homepage, "the homepage", "archilyzer publish homepage", r, (after) =>
+ builtFromCurrentIndex(s, "homepage", "_homepage") && (s.index.stamp?.builtAt ?? 0) >= after)),
+};
+
+/** The job kind a stage runs as on the editor's `publish` queue. */
+export function stageJobKind(kind: StageKind): `publish-${StageKind}` {
+ return STAGES[kind].jobKind;
+}
diff --git a/common/publish/stamps.test.ts b/common/publish/stamps.test.ts
@@ -0,0 +1,138 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import {
+ asBuiltStamp,
+ asIndexStamp,
+ builtStampPath,
+ deployRecordFor,
+ deployedPath,
+ imageBuildFacts,
+ indexStampPath,
+ newStampId,
+ readBuiltStamp,
+ readDeployedFile,
+ readIndexStamp,
+ recordDeploy,
+ writeBuiltStamp,
+ writeIndexStamp,
+ type DeployRecord,
+} from "./stamps";
+import { builtStamp, indexStamp } from "./__fixtures__/stamps";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common test
+//
+// The publish stamps (release 18): a round trip through disk, and the
+// tolerance every `needs()` relies on — a missing, unparseable or
+// wrongly-shaped stamp reads as null (stale), never as half a stamp.
+
+function tmpPaths(): { root: string; paths: Paths } {
+ const root = mkdtempSync(path.join(os.tmpdir(), "stamps-"));
+ return {
+ root,
+ paths: {
+ exportIndexDir: path.join(root, ".export-index"),
+ exportBuildsDir: path.join(root, ".export-builds"),
+ } as Paths,
+ };
+}
+
+test("the three stamps live where the plan says", () => {
+ const p = { exportIndexDir: "/e/.export-index", exportBuildsDir: "/e/.export-builds" } as Paths;
+ assert.equal(indexStampPath(p), "/e/.export-index/stamp.json");
+ assert.equal(builtStampPath(p, "jer"), "/e/.export-builds/jer/built.json");
+ assert.equal(builtStampPath(p, "_hub"), "/e/.export-builds/_hub/built.json");
+ assert.equal(deployedPath(p, "_homepage"), "/e/.export-builds/_homepage/deployed.json");
+});
+
+test("stamps round-trip through disk", async () => {
+ const { root, paths } = tmpPaths();
+ try {
+ assert.equal(await readIndexStamp(paths), null, "no stamp yet");
+ await writeIndexStamp(paths, indexStamp());
+ assert.deepEqual(await readIndexStamp(paths), indexStamp());
+ await writeBuiltStamp(paths, builtStamp({ sourceCommit: null }));
+ assert.deepEqual(await readBuiltStamp(paths, "jer"), builtStamp({ sourceCommit: null }));
+ assert.equal(await readBuiltStamp(paths, "other"), null);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("a malformed or wrongly-shaped stamp reads as null", async () => {
+ const { root, paths } = tmpPaths();
+ try {
+ mkdirSync(paths.exportIndexDir, { recursive: true });
+ writeFileSync(indexStampPath(paths), "{not json");
+ assert.equal(await readIndexStamp(paths), null);
+ writeFileSync(indexStampPath(paths), JSON.stringify({ ...indexStamp(), v: 2 }));
+ assert.equal(await readIndexStamp(paths), null, "another version");
+ writeFileSync(indexStampPath(paths), JSON.stringify({ ...indexStamp(), hubSig: 3 }));
+ assert.equal(await readIndexStamp(paths), null, "a field of the wrong type");
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+ assert.equal(asIndexStamp(null), null);
+ assert.equal(asIndexStamp([]), null);
+ assert.equal(asIndexStamp({ ...indexStamp(), sites: { x: { inputSig: 1 } } }), null);
+ assert.equal(asIndexStamp({ ...indexStamp(), index: { ...indexStamp().index, heldChannels: [1] } }), null);
+ assert.equal(asBuiltStamp({ ...builtStamp(), kind: "export" }), null);
+ assert.equal(asBuiltStamp({ ...builtStamp(), runner: "podman" }), null);
+ assert.equal(asBuiltStamp({ ...builtStamp(), files: "10" }), null);
+ assert.equal(asBuiltStamp({ ...builtStamp(), sourceCommit: 5 }), null);
+ const { stampId: _drop, ...noId } = builtStamp();
+ assert.equal(asBuiltStamp(noId), null);
+});
+
+test("recordDeploy keeps every other record, and deployRecordFor finds each kind", async () => {
+ const { root, paths } = tmpPaths();
+ const rec = (over: Partial<DeployRecord>): DeployRecord => ({
+ builtStampId: "b1",
+ builtAt: 3_000,
+ kind: "production",
+ url: "https://x.pages.dev",
+ at: 4_000,
+ liveCheck: null,
+ ...over,
+ });
+ try {
+ await recordDeploy(paths, "jer", rec({}));
+ await recordDeploy(paths, "jer", rec({ kind: "preview", branch: "r18", url: "https://r18.x.pages.dev" }));
+ await recordDeploy(paths, "jer", rec({ kind: "local", url: null }));
+ await recordDeploy(paths, "jer", rec({ kind: "preview", branch: "r19", builtStampId: "b2" }));
+ const file = await readDeployedFile(paths, "jer");
+ assert.equal(deployRecordFor(file, "production")?.url, "https://x.pages.dev");
+ assert.equal(deployRecordFor(file, "preview", "r18")?.url, "https://r18.x.pages.dev");
+ assert.equal(deployRecordFor(file, "preview", "r19")?.builtStampId, "b2");
+ assert.equal(deployRecordFor(file, "preview", "nope"), null);
+ assert.equal(deployRecordFor(file, "preview"), null);
+ assert.equal(deployRecordFor(file, "local")?.url, null);
+ assert.equal(deployRecordFor(null, "production"), null);
+ // A malformed file is replaced, not merged.
+ writeFileSync(deployedPath(paths, "jer"), "[]");
+ assert.equal(await readDeployedFile(paths, "jer"), null);
+ await recordDeploy(paths, "jer", rec({ builtStampId: "b3" }));
+ const again = JSON.parse(readFileSync(deployedPath(paths, "jer"), "utf8"));
+ assert.deepEqual(Object.keys(again.previews), []);
+ assert.equal(again.production.builtStampId, "b3");
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("imageBuildFacts: the image's commit and branch, empty = null", () => {
+ assert.deepEqual(imageBuildFacts({ ARCHILYZER_COMMIT: " abc ", ARCHILYZER_BRANCH: "" }), { commit: "abc", branch: null });
+ assert.deepEqual(imageBuildFacts({}), { commit: null, branch: null });
+ assert.deepEqual(imageBuildFacts({ ARCHILYZER_COMMIT: "c", ARCHILYZER_BRANCH: "main" }), { commit: "c", branch: "main" });
+});
+
+test("newStampId is unique and sorts by time", () => {
+ const a = newStampId(1_000);
+ const b = newStampId(2_000);
+ assert.notEqual(newStampId(1_000), a);
+ assert.ok(a < b);
+});
diff --git a/common/publish/stamps.ts b/common/publish/stamps.ts
@@ -0,0 +1,282 @@
+// The publish stages' on-disk state (release 18): three stamp files, read by
+// every stage's `needs()` and written only by the stage that owns them.
+//
+// <exportIndexDir>/stamp.json IndexStamp (update-index)
+// <exportBuildsDir>/<target>/built.json BuiltStamp (build-site/hub/homepage)
+// <exportBuildsDir>/<target>/deployed.json DeployedFile (deploy-*)
+//
+// `target` is a site id, "_hub" or "_homepage". Every write is atomic (temp +
+// rename, lib/jsonFile-server.ts); every read is TOLERANT: a missing,
+// unparseable or wrongly-shaped file is null, and null means "stale" to every
+// `needs()` — a stage never trusts half a stamp.
+//
+// The shapes are the plan's ("Model", plans/release-18.md) and other slices
+// code against them: S2's live check fills `DeployRecord.liveCheck` with the
+// `LiveCheck` / `Probe` types declared here, S3's status view reads all three.
+
+import { randomBytes } from "node:crypto";
+import path from "node:path";
+import { readJsonFile, writeJsonAtomic } from "../lib/jsonFile-server";
+import type { Paths } from "../lib/paths";
+
+export const HUB_TARGET = "_hub";
+export const HOMEPAGE_TARGET = "_homepage";
+export const INDEX_TARGET = "_index";
+export const ALL_TARGET = "_all";
+
+export type IndexStamp = {
+ v: 1;
+ stampId: string;
+ // LMDB meta `generation` after the build (buildIndex.ts).
+ generation: number;
+ // LMDB meta INDEX_SCANNED_AT_KEY: when the completed scan began (ms).
+ scannedAt: number;
+ // When the stage finished (ms).
+ builtAt: number;
+ // When `build templates` finished (ms).
+ templatesAt: number;
+ commit: string | null;
+ index: {
+ shortCircuited: boolean;
+ added: number;
+ changed: number;
+ removed: number;
+ heldChannels: string[];
+ };
+ stats: { shortCircuited: boolean; notIndexedYet: number; notIndexable: number };
+ // Per site: sha1 of the LMDB `siteFp:<id>` / `statsFp:<id>` fingerprints
+ // (null when absent), and the inputSig compose's skip rule is computed from.
+ sites: Record<string, { siteFp: string | null; statsFp: string | null; inputSig: string }>;
+ hubSig: string;
+};
+
+export type BuiltKind = "site" | "hub" | "homepage";
+export type Runner = "local" | "docker";
+
+export type BuiltStamp = {
+ v: 1;
+ stampId: string;
+ target: string;
+ kind: BuiltKind;
+ // The IndexStamp the bundle was built from (null: none on disk then).
+ indexStampId: string | null;
+ inputSig: string;
+ builtAt: number;
+ // When a later no-op build last found this bundle still matching its inputs
+ // (absent: never). `changedChannels` is measured against max(builtAt, it).
+ checkedAt?: number;
+ commit: string | null;
+ branch: string | null;
+ runner: Runner;
+ audience: "public" | "private";
+ // The bundle's corpus.json `generatedAt` (what the live check compares).
+ corpusGeneratedAt: string | null;
+ files: number;
+ bytes: number;
+ // Oversize archives staged for R2 beside the bundle.
+ archivesStaged: number;
+ // The homepage only: the commit its published source was cut from.
+ sourceCommit?: string | null;
+};
+
+export type Probe = {
+ status: number | null;
+ generatedAt?: string;
+ cfCacheStatus?: string;
+ age?: number;
+ cacheControl?: string;
+ error?: string;
+};
+
+export type LiveCheck = {
+ at: number;
+ url: string;
+ plain: Probe;
+ busted: Probe;
+ expected: string | null;
+ verdict: "ok" | "stale-edge" | "mismatch" | "unreachable" | "skipped";
+ tombstones?: { path: string; plain: Probe; busted: Probe; ok: boolean }[];
+};
+
+export type DeployKind = "production" | "preview" | "local";
+
+export type DeployRecord = {
+ builtStampId: string;
+ builtAt: number;
+ kind: DeployKind;
+ branch?: string;
+ url: string | null;
+ alias?: string;
+ at: number;
+ wrangler?: string;
+ liveCheck: LiveCheck | null;
+};
+
+export type DeployedFile = {
+ v: 1;
+ target: string;
+ production?: DeployRecord;
+ local?: DeployRecord;
+ previews: Record<string, DeployRecord>;
+};
+
+// --- paths --------------------------------------------------------------------
+
+export function indexStampPath(paths: Pick<Paths, "exportIndexDir">): string {
+ return path.join(paths.exportIndexDir, "stamp.json");
+}
+
+export function targetDir(paths: Pick<Paths, "exportBuildsDir">, target: string): string {
+ return path.join(paths.exportBuildsDir, target);
+}
+
+export function builtStampPath(paths: Pick<Paths, "exportBuildsDir">, target: string): string {
+ return path.join(targetDir(paths, target), "built.json");
+}
+
+export function deployedPath(paths: Pick<Paths, "exportBuildsDir">, target: string): string {
+ return path.join(targetDir(paths, target), "deployed.json");
+}
+
+// The runtime image's build facts (Dockerfile build args → ENV): the stamps'
+// `commit` / `branch` where there is no .git to ask. Release 18 S5 declares
+// the two names and exports the same helper (`imageBuildFacts`,
+// lib/envVars.ts); this local one is swapped for it in one line once both
+// slices are merged. Empty = null.
+const IMAGE_COMMIT_NAME = "ARCHILYZER_COMMIT";
+const IMAGE_BRANCH_NAME = "ARCHILYZER_BRANCH";
+
+export function imageBuildFacts(env: NodeJS.ProcessEnv = process.env): {
+ commit: string | null;
+ branch: string | null;
+} {
+ const v = (k: string) => env[k]?.trim() || null;
+ return { commit: v(IMAGE_COMMIT_NAME), branch: v(IMAGE_BRANCH_NAME) };
+}
+
+/** A fresh, sortable, unique stamp id. */
+export function newStampId(now = Date.now()): string {
+ return `${now.toString(36).padStart(9, "0")}-${randomBytes(4).toString("hex")}`;
+}
+
+// --- shape checks (pure; exported for the tests) -------------------------------
+
+type Obj = Record<string, unknown>;
+const isObj = (v: unknown): v is Obj => typeof v === "object" && v !== null && !Array.isArray(v);
+const isStr = (v: unknown): v is string => typeof v === "string";
+const isNum = (v: unknown): v is number => typeof v === "number" && Number.isFinite(v);
+const isStrOrNull = (v: unknown) => v === null || isStr(v);
+const isBool = (v: unknown): v is boolean => typeof v === "boolean";
+
+export function asIndexStamp(v: unknown): IndexStamp | null {
+ if (!isObj(v) || v.v !== 1) return null;
+ if (!isStr(v.stampId) || !isNum(v.generation) || !isNum(v.scannedAt)) return null;
+ if (!isNum(v.builtAt) || !isNum(v.templatesAt) || !isStrOrNull(v.commit)) return null;
+ const ix = v.index;
+ if (!isObj(ix) || !isBool(ix.shortCircuited) || !isNum(ix.added) || !isNum(ix.changed)) return null;
+ if (!isNum(ix.removed) || !Array.isArray(ix.heldChannels) || !ix.heldChannels.every(isStr)) return null;
+ const st = v.stats;
+ if (!isObj(st) || !isBool(st.shortCircuited) || !isNum(st.notIndexedYet) || !isNum(st.notIndexable)) {
+ return null;
+ }
+ if (!isObj(v.sites) || !isStr(v.hubSig)) return null;
+ for (const s of Object.values(v.sites)) {
+ if (!isObj(s) || !isStr(s.inputSig) || !isStrOrNull(s.siteFp) || !isStrOrNull(s.statsFp)) return null;
+ }
+ return v as unknown as IndexStamp;
+}
+
+export function asBuiltStamp(v: unknown): BuiltStamp | null {
+ if (!isObj(v) || v.v !== 1) return null;
+ if (!isStr(v.stampId) || !isStr(v.target) || !isStr(v.inputSig) || !isNum(v.builtAt)) return null;
+ if (v.kind !== "site" && v.kind !== "hub" && v.kind !== "homepage") return null;
+ if (v.runner !== "local" && v.runner !== "docker") return null;
+ if (v.audience !== "public" && v.audience !== "private") return null;
+ if (!isStrOrNull(v.indexStampId) || !isStrOrNull(v.commit) || !isStrOrNull(v.branch)) return null;
+ if (!isStrOrNull(v.corpusGeneratedAt)) return null;
+ if (!isNum(v.files) || !isNum(v.bytes) || !isNum(v.archivesStaged)) return null;
+ if (v.sourceCommit !== undefined && !isStrOrNull(v.sourceCommit)) return null;
+ if (v.checkedAt !== undefined && !isNum(v.checkedAt)) return null;
+ return v as unknown as BuiltStamp;
+}
+
+function asDeployRecord(v: unknown): DeployRecord | null {
+ if (!isObj(v)) return null;
+ if (!isStr(v.builtStampId) || !isNum(v.builtAt) || !isNum(v.at)) return null;
+ if (v.kind !== "production" && v.kind !== "preview" && v.kind !== "local") return null;
+ if (!isStrOrNull(v.url)) return null;
+ if (v.liveCheck !== null && !isObj(v.liveCheck)) return null;
+ return v as unknown as DeployRecord;
+}
+
+export function asDeployedFile(v: unknown): DeployedFile | null {
+ if (!isObj(v) || v.v !== 1 || !isStr(v.target) || !isObj(v.previews)) return null;
+ for (const key of ["production", "local"] as const) {
+ if (v[key] !== undefined && asDeployRecord(v[key]) === null) return null;
+ }
+ for (const r of Object.values(v.previews)) if (asDeployRecord(r) === null) return null;
+ return v as unknown as DeployedFile;
+}
+
+// --- read / write -------------------------------------------------------------
+
+async function readAs<T>(file: string, as: (v: unknown) => T | null): Promise<T | null> {
+ const r = await readJsonFile(file);
+ return r.ok ? as(r.value) : null;
+}
+
+export function readIndexStamp(paths: Pick<Paths, "exportIndexDir">): Promise<IndexStamp | null> {
+ return readAs(indexStampPath(paths), asIndexStamp);
+}
+
+export function writeIndexStamp(paths: Pick<Paths, "exportIndexDir">, stamp: IndexStamp): Promise<void> {
+ return writeJsonAtomic(indexStampPath(paths), stamp, { mkdir: true });
+}
+
+export function readBuiltStamp(
+ paths: Pick<Paths, "exportBuildsDir">,
+ target: string,
+): Promise<BuiltStamp | null> {
+ return readAs(builtStampPath(paths, target), asBuiltStamp);
+}
+
+export function writeBuiltStamp(paths: Pick<Paths, "exportBuildsDir">, stamp: BuiltStamp): Promise<void> {
+ return writeJsonAtomic(builtStampPath(paths, stamp.target), stamp, { mkdir: true });
+}
+
+export function readDeployedFile(
+ paths: Pick<Paths, "exportBuildsDir">,
+ target: string,
+): Promise<DeployedFile | null> {
+ return readAs(deployedPath(paths, target), asDeployedFile);
+}
+
+/** The record a deploy of `kind` (and `branch`, for a preview) left, or null. */
+export function deployRecordFor(
+ file: DeployedFile | null,
+ kind: DeployKind,
+ branch?: string,
+): DeployRecord | null {
+ if (!file) return null;
+ if (kind === "production") return file.production ?? null;
+ if (kind === "local") return file.local ?? null;
+ return branch ? (file.previews[branch] ?? null) : null;
+}
+
+/**
+ * Record one deploy into `<target>/deployed.json`, keeping every other record
+ * (a malformed file is replaced). Returns the file written.
+ */
+export async function recordDeploy(
+ paths: Pick<Paths, "exportBuildsDir">,
+ target: string,
+ record: DeployRecord,
+): Promise<DeployedFile> {
+ const prev = (await readDeployedFile(paths, target)) ?? { v: 1 as const, target, previews: {} };
+ const next: DeployedFile = { ...prev, v: 1, target, previews: { ...prev.previews } };
+ if (record.kind === "production") next.production = record;
+ else if (record.kind === "local") next.local = record;
+ else if (record.branch) next.previews[record.branch] = record;
+ await writeJsonAtomic(deployedPath(paths, target), next, { mkdir: true });
+ return next;
+}
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,10 @@
# Changelog
## [Unreleased]
+- **Publishing is stages, from the command line: `archilyzer publish`.** `publish index` updates the index — the LMDB index, the stats datasets and the chart templates, in one child process with an 8 GB heap — and writes an index stamp (`export/.export-index/stamp.json`) naming, for each site, a signature of everything that site's build reads. `publish build <id|all>` builds a site from that index (no data phase of its own) into its own bundle, `export/.export-builds/<id>/out`, and stamps it (`built.json`); a site whose bundle already matches the index is a no-op unless `--force`. `publish deploy <id|all> [--preview <branch>] [--to local]` ships that bundle — to Cloudflare Pages, or with `--to local` into the directory the docker `site` service serves — and records the deploy (`deployed.json`); deploying the same build again is a no-op unless `--force`. `all` passes over private sites and, to Pages, sites with no Pages project; any other site it cannot deploy is a failure, said after the rest are tried. `publish hub [--deploy]` and `publish homepage [--deploy]` do the same for the hub (`_hub/out`) and the homepage. A stage whose input is not there says so and exits 3: "update the index first", "no build of jeralyzer — archilyzer publish build jeralyzer". Production refuses a bundle built on a branch other than `main`, or with no branch recorded (a detached checkout; an image sets `ARCHILYZER_BRANCH`) — a preview of it is fine. Exit codes: 0 done or nothing to do, 1 failed, 2 usage, 3 precondition not met, 130 cancelled.
+- **One publish at a time on a machine.** Every stage takes `export/.export-builds/.publish.lock`; a second one — an `archilyzer publish` beside the editor, say — waits for it, saying once whom it waits for, and Ctrl-C ends the wait. A lock left by a process that is gone is taken over. A cancelled stage takes the whole process tree it started with it (`next build`'s workers, wrangler, docker).
+- **`export/out` is now a link to the bundle built last.** Each site, and the hub, keeps its own bundle, so building one site no longer replaces another's; `export/out` points at whichever was built most recently, so `serve out` and anything else that read it keeps working.
+- **`build site`, `build all` and `deploy site` are aliases of the publish commands** and print what they run: `build site <id>` is `publish index` (skipped with `--nodata`) then `publish build <id> --force`; `build all` is `publish index` then `publish build all --runner auto` (containers when an engine answers, else one site at a time on the host); `deploy site <id>` is `publish deploy <id>`, which now ships the site's own bundle and refuses a site never built that way. `publish build all --runner docker` builds every stale site in containers on a Linux host and refuses with "the docker runner needs an engine on this host" where there is none.
- **Substitute your own yt-dlp in Docker.** Point `YTDLP_BIN` at a zipapp you built, or set `YTDLP_SOURCE_HOST_DIR` to a yt-dlp checkout and start with `docker-compose.ytdlp.yml`: the image runs it with its own python, and nothing is rebuilt. Every editor boot logs `yt-dlp: <path> <version> (image|override)` (`MISSING` when it does not run; the editor still starts), and `YTDLP_AUTO_UPDATE` updates the image's yt-dlp only, warning instead of touching yours.
- **The Docker image can publish.** It carries python, `pipx` and a pinned `git-filter-repo`, so the homepage's `/source` mirror builds in the container; `docker-compose.source.yml` mounts your repository read-only for it, and the scrub rules and denylist live in the config volume (`/data/config/archilyzer`). Cloudflare and R2 credentials come from `.env`. Run publish commands with `docker compose exec editor pnpm archilyzer …`, not `run --rm`. The `homepage` service serves a local deploy from the builds volume once there is one. RUNNING_IN_DOCKER.md has a Windows checklist.
- **`archilyzer doctor` checks what a publish needs.** Which yt-dlp runs (the image's, the host's or an override, and whether it runs), whether the Cloudflare token and the R2 keys are set (never their values; R2 only when a bucket is configured), free space for the site bundles, the repository the source mirror reads, and the private config dir.
diff --git a/plans/release-18.md b/plans/release-18.md
@@ -353,6 +353,194 @@ Probe = { status|null; generatedAt?; cfCacheStatus?; age?; cacheControl?; error?
(Each slice adds a "### Slice <X>, as shipped" section here, before "## Rollout".)
+### Slice S1, as shipped — the stage contract, the stamps, the lock, per-target bundles and the CLI (2026-10-06)
+
+Branch `r18/stage-core` off `ce66f2d3` (the plan commit on `r18/integration`), worktree `~/Projects/r18-stage-core`
+(editor 7201, test 7211, export 7210 — `pnpm wt list`'s #42), one Opus implementer, beside S2 and S5's image half.
+Scratch files `s1-*` in the job's `tmp`. The plan is "Model", "Stages" and "CLI" above; the deploy bodies are S2's to
+rewire, the status view, the lane and `publish status|now` S3's.
+
+**What it does.**
+- **`common/publish/stages.ts`** — the only module that knows every stage: `StageKind`, `StageRequest`, `Freshness`,
+ `Stage`, `StageOutcome` as "Model" lists them (+ an optional `allowMissingMedia` on the request, for `build site
+ --allow-missing-media`); `STAGES` (seven, `jobKind: publish-<kind>`, `queueKey: "publish"`); a PURE `needs()` per
+ stage over **`NeedsInput`** — the minimal PublishStatus-shaped input S3's view satisfies: `index: {stamp,
+ lastIngestDoneAt, configChangedAt}`, per site / `hub` / `homepage` a `TargetState` `{built, deployed,
+ changedChannels, configChangedAt, bundleProblem, deployProblem?, pagesProblem?}`, and the homepage's `mainHead`.
+ `stageArgv` / `parseStageArgs` (inverse, round-trip tested) and `STAGE_FLAGS`.
+- **`needs()`, row by row.** update-index: stale with no stamp, an ingest ended `done` after `stamp.scannedAt`, or a
+ config newer than it. build-site: BLOCKED "update the index first" with no stamp (forced too), "waiting for the
+ index update this run started" under `indexAfter`, "the index has not seen site X" when the stamp has no entry;
+ then stale by `changedChannels` ("N channels changed (a, b, …)"), a config change after `builtCheckedAt(built)`, an
+ `inputSig` mismatch ("data changed"), a bundle problem, never built; `--force` stale; a `built.commit` that differs
+ is NOT stale. `_all`: per site. deploy-*: blocked with no build ("no build of X — archilyzer publish build X"), a
+ private target (every kind), no Pages project (pages kinds only), a bundle problem, `builtAfter` (unless the
+ bundle matches the index this run updated — see the review fixes), and a PRODUCTION deploy of a bundle whose
+ `branch` is not `main` — a null branch (a detached HEAD, an image built without `ARCHILYZER_BRANCH`) is refused the
+ same way; fresh exactly when `deployed[kind/branch].builtStampId === built.stampId`. build-hub: `built.inputSig ===
+ stamp.hubSig` (+ `changedChannels`). build-homepage: `indexStampId` current and `sourceCommit === mainHead` when a
+ repository answers.
+- **`common/publish/stamps.ts`** — `IndexStamp`, `BuiltStamp`, `DeployRecord`, `DeployedFile`, `LiveCheck`, `Probe`
+ exactly as listed; paths (`<exportIndexDir>/stamp.json`, `<exportBuildsDir>/<target>/{built,deployed}.json`); atomic
+ writes (`writeJsonAtomic`); tolerant reads (missing, unparseable or wrongly shaped = null); `recordDeploy` keeps
+ every other record; `newStampId` sorts by time; `imageBuildFacts` (the image's `ARCHILYZER_COMMIT` /
+ `ARCHILYZER_BRANCH`, empty = null — S5's names, read by name here until S5's helper of the same name replaces it).
+ `BuiltStamp.checkedAt?` (review fix): when a later no-op build last found the bundle still matching its inputs.
+- **Commit and branch in a stamp (the S4 seam):** `ARCHILYZER_COMMIT` / `ARCHILYZER_BRANCH`, when set, WIN over git,
+ each on its own — the runtime image bakes them (no `.git`), and S4's e2e webServer sets `ARCHILYZER_BRANCH=main`,
+ because a worktree's branch is never `main` and production refuses any other. Else git's HEAD and branch; a
+ detached HEAD records `branch: null`.
+- **`common/publish/stageLock.ts`** — `<exportBuildsDir>/.publish.lock` `{pid, host, kind, target, since, pidStart}`,
+ `open(…, "wx")`; the host is `ARCHILYZER_HOST_ID` when set (S5 fixes one in compose), else `os.hostname()`; stale
+ when same host and the pid is dead (`processIsAlive`), answers with another `/proc/<pid>/stat` start time, or names
+ a process that STARTED AFTER the lock's `since` (starttime/100 + `/proc/stat` btime, 2 s slack; a recreated
+ container's pid 1) — with no `/proc`, pid-alive alone; another host's is never stolen, and its wait line names
+ both hosts and how to clear it; a torn file is taken over after 60 s; a live holder is waited for (5 s poll, ONE log
+ line, the signal cancels the wait); release removes only its own.
+- **`common/publish/stageRun.ts`** — `runStage(req)` (in-process, under the lock, never throws: `{code, outcome,
+ message}`), exit codes 0 / 1 / 2 / 3 / 130, `stageMain` (the child: SIGTERM/SIGINT abort the stage, a second one
+ SIGKILLs the child process groups and exits, tree-kill on), and **`stageCommand(paths, req)`** — `<common>/node_modules/.bin/tsx bin/archilyzer.ts stage
+ <kind> <target> [flags]`, cwd `common/`, the caller's env + `NODE_OPTIONS=… --max-old-space-size=8192` for
+ update-index only — what S3's `enqueueStage` hands `runManagedCommand`.
+- **`common/publish/stageBodies.ts`** — every body but update-index first asks its `needs()` over the state ON DISK
+ (`readNeedsInput`: the stamps, the bundles, `siteDeployProblem`, the Pages project, `main`'s head; no job metas):
+ blocked → exit 3, fresh and not forced → a no-op (a build's no-op writes `checkedAt`). **update-index**: `buildIndex` → `buildStats` → the chart
+ templates in ONE process, then the stamp — `generation`, `scannedAt` and each site's `siteFp`/`statsFp` (sha1 of
+ the LMDB keys, read-only), each site's `inputSig` and the `hubSig` (`common/publish/inputSig.ts`). An index that
+ rebuilt nothing and whose every signature is unchanged KEEPS its stamp id (status `noop`), so the builds made from
+ it stay current. **build-site**: `buildSiteBundle`, then `built.json`; `_all` local = each stale site in turn
+ (failures collected); `_all --runner docker` = no engine → exit 3 "the docker runner needs an engine on this host",
+ else `build archives` on the host → `ensureBuildImage` → `runDockerBuildOne` per stale site at
+ `maxParallelBuilds` → `builtBundleProblem` → `built.json` (`runner: "docker"`). **build-hub** / **build-homepage**:
+ `buildHubBundle` / `buildHomepage` (source mirror included; `sourceCommit` from `homepage/out/source/manifest.json`).
+ **deploy-site / hub / homepage**: today's `deploySite` / `deployHub` / `deployHomepage` over the TARGET'S bundle
+ (`outDir` / `stagingDir` options added), then `deployed.json` (`url` = the deployment URL the log line names,
+ `alias` = the preview alias, `liveCheck: null` — S2's); `--to local` copies the bundle's contents into
+ `ARCHILYZER_SITE_OUT` (a site) or `ARCHILYZER_HOMEPAGE_OUT` (the homepage — S5's name), private refused.
+- **`inputSig`** (`common/publish/inputSig.ts`) signs, with compose's `dirSignature`: the site's whole
+ `.export-index/sites/<id>/` tree (chart-templates.json by its BYTES — `build templates` rewrites it every run), each
+ PUBLISHED member's shared transcripts/subs/posts/digests tree (`manifest.json` ignored, a manifest-only tree as
+ compose's constant), `site.json`'s bytes, the `sites/<id>/` dir, the global aliases, curated tags and duplicates
+ files (size + mtime), `archiveStorage` + `social.x.visibility` + `buildArchives`, and — what the export BUILD
+ renders beyond compose (review fix) — the resolved social links, the resolved hub url and the footer's sibling
+ sites (`resolveRelatedSites(site, listSites())`: each sibling's url, title and listing). The rule: a site is fresh
+ exactly when compose AND the export build would produce the same bundle. A superset of their inputs:
+ conservative. `hubSig` = sha1(stampId, homepage.json, each listed site's id + siteUrl + title).
+- **`common/lib/dirSignature.ts`** — compose's `dirSignature`, moved unchanged; `compose-site.ts` imports it (that
+ line, and its now-unused `createHash` import removed, are the only compose-site edits).
+- **`common/publish/build.ts`** — per-target bundles: `bundleDir(paths, target)` (= `dockerSiteOutDir` for a site),
+ `installBundle(src, <target>/out)` (rename into `out.next`, `out → out.prev`, `out.next → out`, `out.prev` removed, a
+ leftover `out.next` deleted first; EXDEV → `fs.cp` + remove, injectable `BundleFs`), `recoverInterruptedInstall` (an
+ `out.prev` with no `out` is renamed back before a build or an install touches the target; the old `built.json` is
+ removed before the swap and the new one written last), `unlinkExportOut` (before a
+ build: a link at export/out is removed so a failed build cannot leave an older bundle there), `pointExportOutAt`
+ (export/out → a RELATIVE symlink, replaced atomically), `stageSiteArchives` (the host compose's `.r2-staging/<id>`
+ moved to `dockerSiteStagingDir`, a no-op where they are one place), `bundleCounts`, `corpusGeneratedAtIn`,
+ `buildSiteBundle`, `buildHubBundle`. `ensureBuildImage`, `runDockerBuildOne`, `runHostScript`,
+ `runWithConcurrency` are exported. A build container gets `EXPORT_BUILDS_DIR=/tmp/archilyzer-builds`
+ (`CONTAINER_BUILDS_DIR`) so its own `publish build` lock never lands on the host mount. Every existing export is
+ unchanged; the editor's actions still build into `export/out` (S4 rewires them).
+- **`common/bin/archilyzer.ts` + `common/bin/publish.ts`** — rows `publish index` (the SAME child the editor spawns,
+ for its heap), `publish build <id|all> [--runner local|docker|auto] [--force] [--skip-archives]`, `publish deploy
+ <id|all> [--preview b] [--to local] [--force]` (`all` passes over, one line each, only a private site and — to
+ Pages — a site with no project; every other refusal is a failure, the rest are still tried and the run exits 1), `publish hub [--deploy] [--preview b] [--force]`, `publish homepage [--deploy]
+ [--preview b] [--to local] [--force]`, and the internal `stage <kind> <target> --run-id …`. A comment marks where
+ S3's `publish status` / `publish now` rows go. **Aliases, printed first**: `build site <id>` = `publish index` (not
+ with `--nodata`) + `publish build <id> --force`; `build all` = `publish index` + `publish build all --runner auto`;
+ `deploy site <id>` = `publish deploy <id>`.
+
+**Deviations from the plan** (one sentence each):
+1. `publish index` runs the stage child (`stageCommand`, the 8 GB heap) rather than the body in the CLI's process: the
+ index and stats builds of the real corpus have always run with that cap (export's `build:index`), and the CLI's own
+ node has the default heap.
+2. `build all`'s alias runs `publish index` first — the old row always ran the data phase, and building every site
+ from a stale index would not be what it said.
+3. Tree-kill lives in `common/jobs/runChild.ts` (`setKillChildTrees`, off by default; a stage child and the publish
+ CLI turn it on): each child leads its own process group and a cancel signals the group, then SIGKILLs what is left
+ once the leader exits. The editor's in-process jobs are unchanged.
+4. `.gitignore` gains `/export/out` (no trailing slash): `**/out/` matches only a directory, and the link showed as
+ untracked — which `release cut --commit` refuses.
+5. Inside the docker per-site build container (`ARCHIVES_READONLY=1`, set by `docker/build-site.sh`) `publish build`
+ builds IN PLACE and stamps nothing — the container hands export/out back and the host stamps it — and asks no
+ stamp, so the editor's existing Build all (host `build:data`, no stamp) keeps working until S4 rewires it. The
+ container still writes `<id>/out` with build-site.sh's `rm` + `cp`, not through `out.next` (S5's file).
+6. `StageRequest.allowMissingMedia` (optional) carries `build site --allow-missing-media` through the stage.
+7. The bundle-layout tests are a new `common/publish/bundle.test.ts`, not `build.test.ts`, which S2 also edits.
+8. `ARCHILYZER_COMMIT` / `ARCHILYZER_BRANCH` / `ARCHILYZER_HOMEPAGE_OUT` are read by name through an `env[name]`
+ helper: they are declared in `lib/envVars.ts` on S5's branch, not on this one (`envVars.test` stays green here;
+ after the merge the helper can be S5's `imageBuildFacts`).
+
+**Open question 3, settled: yes — a `generation` bump rewrites EVERY site's aggregates.** `generation` is bumped
+whenever any record anywhere is added, changed or removed (`buildIndex.ts` ~:2036), and every site's fingerprint
+carries `gen` (~:2083), so every site is rebuilt: its summary pages (`writeJsonAtomic`, no sha1 skip for site pages),
+its four manifests (fresh `generatedAt`) and its `tag-counts.json`. Their mtimes move, compose would re-copy the
+summaries, and every site's `inputSig` changes. So `inputSig` is conservative as the plan expected: any data change
+anywhere makes every site stale (more rebuilds, never a wrong skip); `changedChannels` is the precise per-site signal.
+A follow-up could sign the site's summaries by content less `generatedAt`.
+
+**Found, not fixed (not this slice's files).**
+- **Every hub bundle is refused since `5c09cd7b` (2026-10-05).** `builtHubProblem` (`common/lib/builtExport.ts`)
+ refuses a hub `out/` that holds `reports/` or `m/` ("still carries a site's data (reports, m)"), but the export app's
+ own `/reports/` and `/m/[...moment]` routes render `out/reports/index.html` (+ `__next.*.txt`) and `out/m/…` in
+ EVERY build, the hub's included — `export/public` had neither when measured. So `deployHub` (old path and new) and
+ `publish hub` refuse every hub; rollout step 5 is blocked until the check looks for report DATA (e.g.
+ `reports/index.json`, a `reports/<id>/page.json`) or the hub stops rendering those routes. Measured in the S1 smoke
+ (a scratch corpus; `publish hub` exit 1 after a 119 s build).
+- `pnpm --filter … exec` (and so `pnpm archilyzer`) reports a stage's exit 2 / 3 / 130 as 1; the editor spawns tsx
+ directly and sees the real code.
+- The worktree export build gate needs a FULL site's compose in `export/public`; the primary's held a cited site's
+ (no `summaries/`) at the time, and `/` failed to prerender against it. The gate was run over a scratch site
+ composed into the worktree's own `public/` (then cleaned and re-linked).
+
+**Left for the other slices.** S2: rewire the three deploy bodies (`stageBodies.ts` `deployStage`) to its deploy
+stage, import `LiveCheck`/`Probe` from `stamps.ts`, fill `DeployRecord.liveCheck`/`wrangler`. S3: build
+`PublishStatus` to satisfy `NeedsInput` (job metas → `lastIngestDoneAt` / `changedChannels`, config mtimes), spawn
+`stageCommand`, the `publish status|now` rows, the `publish-*` job kinds. S4: the editor's actions still call
+`buildSite` / `deploySite` on export/out. S5: `docker/publish-site.sh` and `build-site.sh` (the swap), and the
+envVars names above.
+
+| commit | what |
+|---|---|
+| `ffa95b73` | `dirSignature` moves to `lib/dirSignature.ts`, unchanged; compose imports it |
+| `0122cd1b` | the stamp files and the publish lock (+ tests) |
+| `75403df3` | per-target bundles: install, link, archive staging; `buildSiteBundle`/`buildHubBundle`; deploy `outDir`; tree-kill |
+| `0a7d88ac` | the seven stages, the runner, `inputSig`, the CLI rows and the aliases |
+| `e040bb3f` | tests: `needs()` per row, argv, bundle install (EXDEV), inputSig, stages over a scratch corpus, tree-kill, CLI |
+| `3033d9f2` | stamps fall back to the image's commit/branch; `deploy-homepage --to local` → `ARCHILYZER_HOMEPAGE_OUT` |
+| `89b16716` | `.gitignore`: `/export/out` |
+
+**Review fixes** (review `s1-review.md`: SHIP AFTER FIXES; the coordinator's list, plus S5's host-id note):
+
+| sev | fix | commit |
+|---|---|---|
+| HIGH | `inputSig` also signs the resolved social links, the hub url, `buildArchives` and the footer's sibling sites; one test each | `b88cbe8d` |
+| MEDIUM | the lock's host is `ARCHILYZER_HOST_ID` (else the hostname); a pid whose process started after the lock's `since` is not its holder (`/proc` start time, injectable; no `/proc` = pid-alive alone); a foreign host's wait line names both hosts and how to clear it | `99a8a939` |
+| MEDIUM | a run's no-op build no longer holds its deploy: under `builtAfter` the bundle counts as current when it matches the current index (`inputSig` / `hubSig` / `indexStampId`) and that index ran at or after `builtAfter`; a no-op build writes `built.checkedAt`, and `changedChannels` / config changes are measured against `builtCheckedAt` = max(builtAt, checkedAt) — S3's chip uses the same | `d1d19add` |
+| LOW | a detached HEAD records `branch: null`, and production refuses null like any branch but `main` | `d1d19add` |
+| SEAM | `ARCHILYZER_BRANCH` / `ARCHILYZER_COMMIT` win over git in the stamps (S4's e2e sets `ARCHILYZER_BRANCH=main`) | `d1d19add` |
+| MEDIUM | `publish deploy all` skips only private and (to Pages) project-less sites; every other refusal fails the run (exit 1) after the rest | `32f8a37d` |
+| LOW | an interrupted install (`out.prev`, no `out`) is restored before anything else; `built.json` is removed before the swap, written last | `32f8a37d` |
+| LOW | a second SIGTERM / Ctrl-C (CLI and stage child) SIGKILLs the detached process groups before exiting (`killChildTreesNow`) | `32f8a37d` |
+| LOW | the `stage` usage names `--allow-missing-media` | `32f8a37d` |
+
+Not taken (the review's other lows, left for S6's list): the read-then-`rm` race in `removeIfUnchanged`; the lock's
+place beside the checkout's builds dir while a worktree's `export/public` links into the primary's.
+
+**Gates** (all from the worktree root): tsc clean at every commit; common **3219 passed** (58 new: stamps 6,
+stageLock 8, stages 21, bundle 7, stageRun 6, inputSig 4, runChild 2, `_cli` +4); editor unit **142**;
+`test:scripts` **596 + 3 skipped**; mcp **289**; export unit **116**; homepage unit **23**; `pnpm --filter editor exec
+next build` ok (402 s, the machine busy); `pnpm --filter export exec next build` ok (59 s, over a scratch full site —
+see "Found"); `pnpm --filter homepage run build:nodata` ok (44 s); umtool's capped build ok (38 s). e2e (editor
+suite, `s1-specs.txt`: build, deploy-page, site-publish-preview, sites-homepage, duplicate-shorts, cut-release,
+ops-api): **52 passed, 0 failed, 2.5 min** — after a first launch died on "Timed out waiting 120000ms from
+config.webServer" (the linked primary `export/public` held a cited site's compose, so the export dev server 500ed on
+`summaries/manifest.json`); re-run with a scratch FULL site composed into the worktree's own `public/` (FACTS :3485's
+"Copy a composed fixture site into it"), cleaned after. **After the review fixes:** tsc clean; common
+**3229 passed** (10 new: inputSig +1, stageLock +3, stages +2, stageRun +2, bundle +1, runChild +1); e2e (same seven, same seeding) **52 passed, 0 failed, 2.1 min**. Numbers tool: none. Live smoke over a scratch corpus (`s1-smoke-build.sh`): `publish index`
+(9 s) → `publish build smoke` (85 s, bundle installed by rename, export/out a relative link) → again: no-op →
+`stage deploy-site … --to local` without the env: refused → `publish deploy smoke --to local`: copied + recorded →
+`publish hub`: refused by `builtHubProblem` (above) → `build site smoke --nodata`: alias printed, forced rebuild.
+
### Slice S5, as shipped — the image half: the container can publish, yt-dlp can be substituted, the doctor checks it (2026-10-06)
Branch `r18/docker-publish` off `r18/integration` `ce66f2d3`, worktree `~/Projects/r18-docker-publish`
