commit a15670934874afb3c93201053f6e9ef01fcbac94
parent 90345b517bbf52739b9b0942ce0e1b4f624dc988
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 6 Oct 2026 08:18:13 -0400
publish: the stamp files and the publish lock (release 18 stage core)
stamps.ts: IndexStamp / BuiltStamp / DeployedFile (+ LiveCheck, Probe) as the
plan's Model lists them, atomic writes, tolerant reads (malformed = null).
stageLock.ts: <exportBuildsDir>/.publish.lock taken O_EXCL; a holder on this
host whose pid is gone (or reused by a later process) is taken over, a live
one waited for (5 s poll, one line, cancellable), another host's never stolen.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
5 files changed, 839 insertions(+), 0 deletions(-)
diff --git a/common/publish/__fixtures__/stamps.ts b/common/publish/__fixtures__/stamps.ts
@@ -0,0 +1,40 @@
+// Stamp builders shared by the publish tests (stamps, stages).
+import type { BuiltStamp, IndexStamp } from "../stamps";
+
+export function indexStamp(over: Partial<IndexStamp> = {}): IndexStamp {
+ return {
+ v: 1,
+ stampId: "s1",
+ generation: 7,
+ scannedAt: 1_000,
+ builtAt: 2_000,
+ templatesAt: 1_900,
+ commit: "abc",
+ index: { shortCircuited: false, added: 1, changed: 2, removed: 0, heldChannels: [] },
+ stats: { shortCircuited: true, notIndexedYet: 0, notIndexable: 3 },
+ sites: { jer: { siteFp: "f1", statsFp: null, inputSig: "sig-jer" } },
+ hubSig: "hub-1",
+ ...over,
+ };
+}
+
+export function builtStamp(over: Partial<BuiltStamp> = {}): BuiltStamp {
+ return {
+ v: 1,
+ stampId: "b1",
+ target: "jer",
+ kind: "site",
+ indexStampId: "s1",
+ inputSig: "sig-jer",
+ builtAt: 3_000,
+ commit: "abc",
+ branch: "main",
+ runner: "local",
+ audience: "public",
+ corpusGeneratedAt: "2026-10-06T00:00:00.000Z",
+ files: 10,
+ bytes: 1234,
+ archivesStaged: 0,
+ ...over,
+ };
+}
diff --git a/common/publish/stageLock.test.ts b/common/publish/stageLock.test.ts
@@ -0,0 +1,172 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import {
+ LockWaitCancelled,
+ acquirePublishLock,
+ holderIsGone,
+ pidStartOf,
+ publishLockPath,
+ readLockHolder,
+ withPublishLock,
+ type LockHolder,
+} from "./stageLock";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common test
+//
+// The publish lock (release 18): a stale holder on this host is taken over, a
+// live one is waited for (cancellably), and a holder on another host is never
+// stolen. The process probes are injected; nothing here signals a real pid.
+
+function tmp(): { root: string; paths: Paths } {
+ const root = mkdtempSync(path.join(os.tmpdir(), "stage-lock-"));
+ return { root, paths: { exportBuildsDir: path.join(root, ".export-builds") } as Paths };
+}
+
+function plant(paths: Paths, holder: Partial<LockHolder>): void {
+ mkdirSync(paths.exportBuildsDir, { recursive: true });
+ writeFileSync(
+ publishLockPath(paths),
+ JSON.stringify({ pid: 4242, host: "here", kind: "build-site", target: "jer", since: 1, ...holder }),
+ );
+}
+
+const here = { host: "here", pid: 100, startOf: () => null };
+
+test("holderIsGone: same host and a dead pid, or a pid reused by a later process", () => {
+ const h: LockHolder = { pid: 7, host: "here", kind: "k", target: "t", since: 1, pidStart: "500" };
+ assert.equal(holderIsGone(h, { host: "here", isAlive: () => false }), true);
+ assert.equal(holderIsGone(h, { host: "here", isAlive: () => true, startOf: () => "500" }), false);
+ assert.equal(holderIsGone(h, { host: "here", isAlive: () => true, startOf: () => "900" }), true, "pid reused");
+ assert.equal(holderIsGone(h, { host: "here", isAlive: () => true, startOf: () => null }), false);
+ assert.equal(holderIsGone(h, { host: "there", isAlive: () => false }), false, "another host: never");
+ assert.equal(holderIsGone({ ...h, pidStart: null }, { host: "here", isAlive: () => true, startOf: () => "1" }), false);
+});
+
+test("pidStartOf reads this process's start time on Linux and is null for no such pid", () => {
+ if (process.platform === "linux") {
+ assert.match(pidStartOf(process.pid) ?? "", /^\d+$/);
+ assert.equal(pidStartOf(process.pid), pidStartOf(process.pid));
+ }
+ assert.equal(pidStartOf(2 ** 30), null);
+});
+
+test("the lock is taken, names its holder, and is released", async () => {
+ const { root, paths } = tmp();
+ try {
+ const lock = await acquirePublishLock(paths, { kind: "update-index", target: "_index" }, here);
+ assert.deepEqual(await readLockHolder(paths), lock.holder);
+ assert.equal(lock.holder.pid, 100);
+ assert.equal(lock.holder.kind, "update-index");
+ await lock.release();
+ assert.equal(existsSync(publishLockPath(paths)), false);
+ // withPublishLock releases on a throw too.
+ await assert.rejects(
+ withPublishLock(paths, { kind: "k", target: "t" }, async () => {
+ throw new Error("boom");
+ }, here),
+ /boom/,
+ );
+ assert.equal(existsSync(publishLockPath(paths)), false);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("a stale holder on this host is taken over, and said so", async () => {
+ const { root, paths } = tmp();
+ try {
+ plant(paths, { pid: 4242, host: "here" });
+ const logs: string[] = [];
+ const lock = await acquirePublishLock(paths, { kind: "build-site", target: "ani" }, {
+ ...here,
+ isAlive: (pid) => pid !== 4242,
+ onLog: (l) => logs.push(l),
+ });
+ assert.equal(lock.holder.target, "ani");
+ assert.match(logs.join(""), /taking over a stale publish lock: build-site jer \(pid 4242 on here/);
+ await lock.release();
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("a live holder is waited for, with one log line, until it releases", async () => {
+ const { root, paths } = tmp();
+ try {
+ plant(paths, { pid: 4242, host: "here" });
+ const logs: string[] = [];
+ let polls = 0;
+ const taking = acquirePublishLock(paths, { kind: "build-site", target: "ani" }, {
+ ...here,
+ isAlive: () => true,
+ pollMs: 5,
+ onLog: (l) => {
+ logs.push(l);
+ },
+ now: () => {
+ polls++;
+ // The holder releases after a few polls.
+ if (polls === 6) rmSync(publishLockPath(paths));
+ return polls;
+ },
+ });
+ const lock = await taking;
+ assert.equal(lock.holder.target, "ani");
+ assert.equal(logs.length, 1, logs.join(""));
+ assert.match(logs[0], /waiting for the publish lock — held by build-site jer \(pid 4242 on here/);
+ await lock.release();
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("a holder on another host is never stolen; the wait is cancellable", async () => {
+ const { root, paths } = tmp();
+ try {
+ plant(paths, { pid: 4242, host: "elsewhere" });
+ const ac = new AbortController();
+ const taking = acquirePublishLock(paths, { kind: "k", target: "t" }, {
+ ...here,
+ isAlive: () => false, // dead HERE means nothing for a pid over there
+ pollMs: 5,
+ signal: ac.signal,
+ });
+ setTimeout(() => ac.abort(), 40);
+ await assert.rejects(taking, LockWaitCancelled);
+ assert.equal(JSON.parse(readFileSync(publishLockPath(paths), "utf8")).host, "elsewhere", "untouched");
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("a torn lock file is waited for, then taken over after the grace", async () => {
+ const { root, paths } = tmp();
+ try {
+ mkdirSync(paths.exportBuildsDir, { recursive: true });
+ writeFileSync(publishLockPath(paths), "");
+ const old = new Date(Date.now() - 120_000);
+ utimesSync(publishLockPath(paths), old, old);
+ const lock = await acquirePublishLock(paths, { kind: "k", target: "t" }, { ...here, pollMs: 5 });
+ assert.equal(lock.holder.kind, "k");
+ await lock.release();
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("release never removes a lock somebody else holds now", async () => {
+ const { root, paths } = tmp();
+ try {
+ const lock = await acquirePublishLock(paths, { kind: "k", target: "t" }, here);
+ plant(paths, { pid: 9, host: "here", since: 99 });
+ await lock.release();
+ assert.equal(JSON.parse(readFileSync(publishLockPath(paths), "utf8")).pid, 9);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
diff --git a/common/publish/stageLock.ts b/common/publish/stageLock.ts
@@ -0,0 +1,234 @@
+// The publish lock (release 18): ONE publish stage at a time on this machine,
+// whoever started it — a stage child the editor spawned, or `archilyzer
+// publish …` run by hand (`docker compose exec editor …` included). The
+// editor's `publish` queue already runs its stages one by one; this file is
+// what keeps a CLI started beside it from building into the same shared
+// export/public at once.
+//
+// <exportBuildsDir>/.publish.lock {pid, host, kind, target, since, pidStart}
+//
+// Taken with O_EXCL (`open(…, "wx")`). A holder is STALE — and its lock taken
+// over — only when it is on THIS host and its process is gone: the pid does
+// not answer (`processIsAlive`, jobs/bootQueuedJobs.ts), or it answers with a
+// different start time (`/proc/<pid>/stat`, where there is one: a container's
+// editor comes back with the same small pids on every restart). A lock naming
+// another host is never stolen — a pid means nothing across a namespace. A
+// live holder makes the taker WAIT: a poll every 5 s, one log line, and a
+// cancel (the AbortSignal) gives up the wait.
+
+import { readFileSync } from "node:fs";
+import { mkdir, open, readFile, rm, stat } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { processIsAlive } from "../jobs/bootQueuedJobs";
+import type { Paths } from "../lib/paths";
+
+export type LockHolder = {
+ pid: number;
+ host: string;
+ kind: string;
+ target: string;
+ since: number;
+ // The holder process's start time (/proc/<pid>/stat field 22), when known.
+ pidStart?: string | null;
+};
+
+export const LOCK_POLL_MS = 5_000;
+// A lock file that stays unreadable this long was left by a taker that died
+// between creating it and writing it.
+export const LOCK_TORN_GRACE_MS = 60_000;
+
+export class LockWaitCancelled extends Error {
+ constructor() {
+ super("cancelled while waiting for the publish lock");
+ this.name = "LockWaitCancelled";
+ }
+}
+
+export function publishLockPath(paths: Pick<Paths, "exportBuildsDir">): string {
+ return path.join(paths.exportBuildsDir, ".publish.lock");
+}
+
+/** A process's start time from /proc (Linux), or null where there is none. */
+export function pidStartOf(pid: number): string | null {
+ try {
+ const raw = readFileSync(`/proc/${pid}/stat`, "utf8");
+ // `pid (comm) state ppid …` — comm may hold spaces and parens, so split
+ // after the LAST ')'. Field 22 (starttime) is index 19 of the rest.
+ const rest = raw.slice(raw.lastIndexOf(")") + 2).split(" ");
+ return rest[19] ?? null;
+ } catch {
+ return null;
+ }
+}
+
+export type LockEnv = {
+ host?: string;
+ pid?: number;
+ isAlive?: (pid: number) => boolean;
+ startOf?: (pid: number) => string | null;
+ now?: () => number;
+};
+
+function env(e: LockEnv = {}) {
+ return {
+ host: e.host ?? os.hostname(),
+ pid: e.pid ?? process.pid,
+ isAlive: e.isAlive ?? processIsAlive,
+ startOf: e.startOf ?? pidStartOf,
+ now: e.now ?? Date.now,
+ };
+}
+
+/**
+ * True when `holder`'s process is certainly gone: same host, and its pid is
+ * dead or now belongs to a process that started at another time. Pure over
+ * the injected probes.
+ */
+export function holderIsGone(holder: LockHolder, e: LockEnv = {}): boolean {
+ const { host, isAlive, startOf } = env(e);
+ if (holder.host !== host) return false;
+ if (!isAlive(holder.pid)) return true;
+ if (holder.pidStart) {
+ const now = startOf(holder.pid);
+ if (now !== null && now !== holder.pidStart) return true;
+ }
+ return false;
+}
+
+export function parseLockHolder(text: string): LockHolder | null {
+ try {
+ const v = JSON.parse(text) as Partial<LockHolder>;
+ if (
+ typeof v?.pid !== "number" ||
+ typeof v.host !== "string" ||
+ typeof v.kind !== "string" ||
+ typeof v.target !== "string" ||
+ typeof v.since !== "number"
+ ) {
+ return null;
+ }
+ return v as LockHolder;
+ } catch {
+ return null;
+ }
+}
+
+/** Who holds the publish lock now, or null (none, or unreadable). */
+export async function readLockHolder(paths: Pick<Paths, "exportBuildsDir">): Promise<LockHolder | null> {
+ try {
+ return parseLockHolder(await readFile(publishLockPath(paths), "utf8"));
+ } catch {
+ return null;
+ }
+}
+
+export function describeHolder(h: LockHolder): string {
+ return `${h.kind} ${h.target} (pid ${h.pid} on ${h.host}, since ${new Date(h.since).toISOString()})`;
+}
+
+export type PublishLock = { holder: LockHolder; release: () => Promise<void> };
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+ return new Promise((resolve) => {
+ const t = setTimeout(done, ms);
+ function done() {
+ clearTimeout(t);
+ signal?.removeEventListener("abort", done);
+ resolve();
+ }
+ signal?.addEventListener("abort", done, { once: true });
+ });
+}
+
+/**
+ * Take the publish lock for `who`, waiting while a live holder has it.
+ * Throws LockWaitCancelled when `signal` aborts first.
+ */
+export async function acquirePublishLock(
+ paths: Pick<Paths, "exportBuildsDir">,
+ who: { kind: string; target: string },
+ opts: LockEnv & { signal?: AbortSignal; onLog?: (line: string) => void; pollMs?: number } = {},
+): Promise<PublishLock> {
+ const e = env(opts);
+ const file = publishLockPath(paths);
+ await mkdir(path.dirname(file), { recursive: true });
+ let said = false;
+ for (;;) {
+ if (opts.signal?.aborted) throw new LockWaitCancelled();
+ const holder: LockHolder = {
+ pid: e.pid,
+ host: e.host,
+ kind: who.kind,
+ target: who.target,
+ since: e.now(),
+ pidStart: e.startOf(e.pid),
+ };
+ try {
+ const fh = await open(file, "wx");
+ try {
+ await fh.writeFile(JSON.stringify(holder) + "\n");
+ await fh.sync().catch(() => {});
+ } finally {
+ await fh.close();
+ }
+ return { holder, release: () => releaseLock(file, holder) };
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
+ }
+ // Someone holds it (or held it).
+ let text = "";
+ try {
+ text = await readFile(file, "utf8");
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code === "ENOENT") continue; // released meanwhile
+ throw err;
+ }
+ const current = parseLockHolder(text);
+ if (current === null) {
+ // Torn: a taker between its create and its write — or one that died
+ // there. Give it the grace, then take the lock over.
+ const age = await stat(file).then((s) => e.now() - s.mtimeMs, () => 0);
+ if (age > LOCK_TORN_GRACE_MS) {
+ await removeIfUnchanged(file, text);
+ continue;
+ }
+ } else if (holderIsGone(current, e)) {
+ opts.onLog?.(`[publish] taking over a stale publish lock: ${describeHolder(current)} is gone\n`);
+ await removeIfUnchanged(file, text);
+ continue;
+ } else if (!said) {
+ said = true;
+ opts.onLog?.(`[publish] waiting for the publish lock — held by ${describeHolder(current)}\n`);
+ }
+ await sleep(opts.pollMs ?? LOCK_POLL_MS, opts.signal);
+ }
+}
+
+// Remove the lock only if it still holds exactly what was judged stale.
+async function removeIfUnchanged(file: string, judged: string): Promise<void> {
+ const now = await readFile(file, "utf8").catch(() => null);
+ if (now === judged) await rm(file, { force: true });
+}
+
+async function releaseLock(file: string, holder: LockHolder): Promise<void> {
+ const current = parseLockHolder(await readFile(file, "utf8").catch(() => ""));
+ if (current && current.pid === holder.pid && current.host === holder.host && current.since === holder.since) {
+ await rm(file, { force: true });
+ }
+}
+
+/** Run `fn` holding the publish lock; always released after. */
+export async function withPublishLock<T>(
+ paths: Pick<Paths, "exportBuildsDir">,
+ who: { kind: string; target: string },
+ fn: () => Promise<T>,
+ opts: LockEnv & { signal?: AbortSignal; onLog?: (line: string) => void; pollMs?: number } = {},
+): Promise<T> {
+ const lock = await acquirePublishLock(paths, who, opts);
+ try {
+ return await fn();
+ } finally {
+ await lock.release();
+ }
+}
diff --git a/common/publish/stamps.test.ts b/common/publish/stamps.test.ts
@@ -0,0 +1,131 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import {
+ asBuiltStamp,
+ asIndexStamp,
+ builtStampPath,
+ deployRecordFor,
+ deployedPath,
+ indexStampPath,
+ newStampId,
+ readBuiltStamp,
+ readDeployedFile,
+ readIndexStamp,
+ recordDeploy,
+ writeBuiltStamp,
+ writeIndexStamp,
+ type DeployRecord,
+} from "./stamps";
+import { builtStamp, indexStamp } from "./__fixtures__/stamps";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common test
+//
+// The publish stamps (release 18): a round trip through disk, and the
+// tolerance every `needs()` relies on — a missing, unparseable or
+// wrongly-shaped stamp reads as null (stale), never as half a stamp.
+
+function tmpPaths(): { root: string; paths: Paths } {
+ const root = mkdtempSync(path.join(os.tmpdir(), "stamps-"));
+ return {
+ root,
+ paths: {
+ exportIndexDir: path.join(root, ".export-index"),
+ exportBuildsDir: path.join(root, ".export-builds"),
+ } as Paths,
+ };
+}
+
+test("the three stamps live where the plan says", () => {
+ const p = { exportIndexDir: "/e/.export-index", exportBuildsDir: "/e/.export-builds" } as Paths;
+ assert.equal(indexStampPath(p), "/e/.export-index/stamp.json");
+ assert.equal(builtStampPath(p, "jer"), "/e/.export-builds/jer/built.json");
+ assert.equal(builtStampPath(p, "_hub"), "/e/.export-builds/_hub/built.json");
+ assert.equal(deployedPath(p, "_homepage"), "/e/.export-builds/_homepage/deployed.json");
+});
+
+test("stamps round-trip through disk", async () => {
+ const { root, paths } = tmpPaths();
+ try {
+ assert.equal(await readIndexStamp(paths), null, "no stamp yet");
+ await writeIndexStamp(paths, indexStamp());
+ assert.deepEqual(await readIndexStamp(paths), indexStamp());
+ await writeBuiltStamp(paths, builtStamp({ sourceCommit: null }));
+ assert.deepEqual(await readBuiltStamp(paths, "jer"), builtStamp({ sourceCommit: null }));
+ assert.equal(await readBuiltStamp(paths, "other"), null);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("a malformed or wrongly-shaped stamp reads as null", async () => {
+ const { root, paths } = tmpPaths();
+ try {
+ mkdirSync(paths.exportIndexDir, { recursive: true });
+ writeFileSync(indexStampPath(paths), "{not json");
+ assert.equal(await readIndexStamp(paths), null);
+ writeFileSync(indexStampPath(paths), JSON.stringify({ ...indexStamp(), v: 2 }));
+ assert.equal(await readIndexStamp(paths), null, "another version");
+ writeFileSync(indexStampPath(paths), JSON.stringify({ ...indexStamp(), hubSig: 3 }));
+ assert.equal(await readIndexStamp(paths), null, "a field of the wrong type");
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+ assert.equal(asIndexStamp(null), null);
+ assert.equal(asIndexStamp([]), null);
+ assert.equal(asIndexStamp({ ...indexStamp(), sites: { x: { inputSig: 1 } } }), null);
+ assert.equal(asIndexStamp({ ...indexStamp(), index: { ...indexStamp().index, heldChannels: [1] } }), null);
+ assert.equal(asBuiltStamp({ ...builtStamp(), kind: "export" }), null);
+ assert.equal(asBuiltStamp({ ...builtStamp(), runner: "podman" }), null);
+ assert.equal(asBuiltStamp({ ...builtStamp(), files: "10" }), null);
+ assert.equal(asBuiltStamp({ ...builtStamp(), sourceCommit: 5 }), null);
+ const { stampId: _drop, ...noId } = builtStamp();
+ assert.equal(asBuiltStamp(noId), null);
+});
+
+test("recordDeploy keeps every other record, and deployRecordFor finds each kind", async () => {
+ const { root, paths } = tmpPaths();
+ const rec = (over: Partial<DeployRecord>): DeployRecord => ({
+ builtStampId: "b1",
+ builtAt: 3_000,
+ kind: "production",
+ url: "https://x.pages.dev",
+ at: 4_000,
+ liveCheck: null,
+ ...over,
+ });
+ try {
+ await recordDeploy(paths, "jer", rec({}));
+ await recordDeploy(paths, "jer", rec({ kind: "preview", branch: "r18", url: "https://r18.x.pages.dev" }));
+ await recordDeploy(paths, "jer", rec({ kind: "local", url: null }));
+ await recordDeploy(paths, "jer", rec({ kind: "preview", branch: "r19", builtStampId: "b2" }));
+ const file = await readDeployedFile(paths, "jer");
+ assert.equal(deployRecordFor(file, "production")?.url, "https://x.pages.dev");
+ assert.equal(deployRecordFor(file, "preview", "r18")?.url, "https://r18.x.pages.dev");
+ assert.equal(deployRecordFor(file, "preview", "r19")?.builtStampId, "b2");
+ assert.equal(deployRecordFor(file, "preview", "nope"), null);
+ assert.equal(deployRecordFor(file, "preview"), null);
+ assert.equal(deployRecordFor(file, "local")?.url, null);
+ assert.equal(deployRecordFor(null, "production"), null);
+ // A malformed file is replaced, not merged.
+ writeFileSync(deployedPath(paths, "jer"), "[]");
+ assert.equal(await readDeployedFile(paths, "jer"), null);
+ await recordDeploy(paths, "jer", rec({ builtStampId: "b3" }));
+ const again = JSON.parse(readFileSync(deployedPath(paths, "jer"), "utf8"));
+ assert.deepEqual(Object.keys(again.previews), []);
+ assert.equal(again.production.builtStampId, "b3");
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("newStampId is unique and sorts by time", () => {
+ const a = newStampId(1_000);
+ const b = newStampId(2_000);
+ assert.notEqual(newStampId(1_000), a);
+ assert.ok(a < b);
+});
diff --git a/common/publish/stamps.ts b/common/publish/stamps.ts
@@ -0,0 +1,262 @@
+// The publish stages' on-disk state (release 18): three stamp files, read by
+// every stage's `needs()` and written only by the stage that owns them.
+//
+// <exportIndexDir>/stamp.json IndexStamp (update-index)
+// <exportBuildsDir>/<target>/built.json BuiltStamp (build-site/hub/homepage)
+// <exportBuildsDir>/<target>/deployed.json DeployedFile (deploy-*)
+//
+// `target` is a site id, "_hub" or "_homepage". Every write is atomic (temp +
+// rename, lib/jsonFile-server.ts); every read is TOLERANT: a missing,
+// unparseable or wrongly-shaped file is null, and null means "stale" to every
+// `needs()` — a stage never trusts half a stamp.
+//
+// The shapes are the plan's ("Model", plans/release-18.md) and other slices
+// code against them: S2's live check fills `DeployRecord.liveCheck` with the
+// `LiveCheck` / `Probe` types declared here, S3's status view reads all three.
+
+import { randomBytes } from "node:crypto";
+import path from "node:path";
+import { readJsonFile, writeJsonAtomic } from "../lib/jsonFile-server";
+import type { Paths } from "../lib/paths";
+
+export const HUB_TARGET = "_hub";
+export const HOMEPAGE_TARGET = "_homepage";
+export const INDEX_TARGET = "_index";
+export const ALL_TARGET = "_all";
+
+export type IndexStamp = {
+ v: 1;
+ stampId: string;
+ // LMDB meta `generation` after the build (buildIndex.ts).
+ generation: number;
+ // LMDB meta INDEX_SCANNED_AT_KEY: when the completed scan began (ms).
+ scannedAt: number;
+ // When the stage finished (ms).
+ builtAt: number;
+ // When `build templates` finished (ms).
+ templatesAt: number;
+ commit: string | null;
+ index: {
+ shortCircuited: boolean;
+ added: number;
+ changed: number;
+ removed: number;
+ heldChannels: string[];
+ };
+ stats: { shortCircuited: boolean; notIndexedYet: number; notIndexable: number };
+ // Per site: sha1 of the LMDB `siteFp:<id>` / `statsFp:<id>` fingerprints
+ // (null when absent), and the inputSig compose's skip rule is computed from.
+ sites: Record<string, { siteFp: string | null; statsFp: string | null; inputSig: string }>;
+ hubSig: string;
+};
+
+export type BuiltKind = "site" | "hub" | "homepage";
+export type Runner = "local" | "docker";
+
+export type BuiltStamp = {
+ v: 1;
+ stampId: string;
+ target: string;
+ kind: BuiltKind;
+ // The IndexStamp the bundle was built from (null: none on disk then).
+ indexStampId: string | null;
+ inputSig: string;
+ builtAt: number;
+ commit: string | null;
+ branch: string | null;
+ runner: Runner;
+ audience: "public" | "private";
+ // The bundle's corpus.json `generatedAt` (what the live check compares).
+ corpusGeneratedAt: string | null;
+ files: number;
+ bytes: number;
+ // Oversize archives staged for R2 beside the bundle.
+ archivesStaged: number;
+ // The homepage only: the commit its published source was cut from.
+ sourceCommit?: string | null;
+};
+
+export type Probe = {
+ status: number | null;
+ generatedAt?: string;
+ cfCacheStatus?: string;
+ age?: number;
+ cacheControl?: string;
+ error?: string;
+};
+
+export type LiveCheck = {
+ at: number;
+ url: string;
+ plain: Probe;
+ busted: Probe;
+ expected: string | null;
+ verdict: "ok" | "stale-edge" | "mismatch" | "unreachable" | "skipped";
+ tombstones?: { path: string; plain: Probe; busted: Probe; ok: boolean }[];
+};
+
+export type DeployKind = "production" | "preview" | "local";
+
+export type DeployRecord = {
+ builtStampId: string;
+ builtAt: number;
+ kind: DeployKind;
+ branch?: string;
+ url: string | null;
+ alias?: string;
+ at: number;
+ wrangler?: string;
+ liveCheck: LiveCheck | null;
+};
+
+export type DeployedFile = {
+ v: 1;
+ target: string;
+ production?: DeployRecord;
+ local?: DeployRecord;
+ previews: Record<string, DeployRecord>;
+};
+
+// --- paths --------------------------------------------------------------------
+
+export function indexStampPath(paths: Pick<Paths, "exportIndexDir">): string {
+ return path.join(paths.exportIndexDir, "stamp.json");
+}
+
+export function targetDir(paths: Pick<Paths, "exportBuildsDir">, target: string): string {
+ return path.join(paths.exportBuildsDir, target);
+}
+
+export function builtStampPath(paths: Pick<Paths, "exportBuildsDir">, target: string): string {
+ return path.join(targetDir(paths, target), "built.json");
+}
+
+export function deployedPath(paths: Pick<Paths, "exportBuildsDir">, target: string): string {
+ return path.join(targetDir(paths, target), "deployed.json");
+}
+
+/** A fresh, sortable, unique stamp id. */
+export function newStampId(now = Date.now()): string {
+ return `${now.toString(36).padStart(9, "0")}-${randomBytes(4).toString("hex")}`;
+}
+
+// --- shape checks (pure; exported for the tests) -------------------------------
+
+type Obj = Record<string, unknown>;
+const isObj = (v: unknown): v is Obj => typeof v === "object" && v !== null && !Array.isArray(v);
+const isStr = (v: unknown): v is string => typeof v === "string";
+const isNum = (v: unknown): v is number => typeof v === "number" && Number.isFinite(v);
+const isStrOrNull = (v: unknown) => v === null || isStr(v);
+const isBool = (v: unknown): v is boolean => typeof v === "boolean";
+
+export function asIndexStamp(v: unknown): IndexStamp | null {
+ if (!isObj(v) || v.v !== 1) return null;
+ if (!isStr(v.stampId) || !isNum(v.generation) || !isNum(v.scannedAt)) return null;
+ if (!isNum(v.builtAt) || !isNum(v.templatesAt) || !isStrOrNull(v.commit)) return null;
+ const ix = v.index;
+ if (!isObj(ix) || !isBool(ix.shortCircuited) || !isNum(ix.added) || !isNum(ix.changed)) return null;
+ if (!isNum(ix.removed) || !Array.isArray(ix.heldChannels) || !ix.heldChannels.every(isStr)) return null;
+ const st = v.stats;
+ if (!isObj(st) || !isBool(st.shortCircuited) || !isNum(st.notIndexedYet) || !isNum(st.notIndexable)) {
+ return null;
+ }
+ if (!isObj(v.sites) || !isStr(v.hubSig)) return null;
+ for (const s of Object.values(v.sites)) {
+ if (!isObj(s) || !isStr(s.inputSig) || !isStrOrNull(s.siteFp) || !isStrOrNull(s.statsFp)) return null;
+ }
+ return v as unknown as IndexStamp;
+}
+
+export function asBuiltStamp(v: unknown): BuiltStamp | null {
+ if (!isObj(v) || v.v !== 1) return null;
+ if (!isStr(v.stampId) || !isStr(v.target) || !isStr(v.inputSig) || !isNum(v.builtAt)) return null;
+ if (v.kind !== "site" && v.kind !== "hub" && v.kind !== "homepage") return null;
+ if (v.runner !== "local" && v.runner !== "docker") return null;
+ if (v.audience !== "public" && v.audience !== "private") return null;
+ if (!isStrOrNull(v.indexStampId) || !isStrOrNull(v.commit) || !isStrOrNull(v.branch)) return null;
+ if (!isStrOrNull(v.corpusGeneratedAt)) return null;
+ if (!isNum(v.files) || !isNum(v.bytes) || !isNum(v.archivesStaged)) return null;
+ if (v.sourceCommit !== undefined && !isStrOrNull(v.sourceCommit)) return null;
+ return v as unknown as BuiltStamp;
+}
+
+function asDeployRecord(v: unknown): DeployRecord | null {
+ if (!isObj(v)) return null;
+ if (!isStr(v.builtStampId) || !isNum(v.builtAt) || !isNum(v.at)) return null;
+ if (v.kind !== "production" && v.kind !== "preview" && v.kind !== "local") return null;
+ if (!isStrOrNull(v.url)) return null;
+ if (v.liveCheck !== null && !isObj(v.liveCheck)) return null;
+ return v as unknown as DeployRecord;
+}
+
+export function asDeployedFile(v: unknown): DeployedFile | null {
+ if (!isObj(v) || v.v !== 1 || !isStr(v.target) || !isObj(v.previews)) return null;
+ for (const key of ["production", "local"] as const) {
+ if (v[key] !== undefined && asDeployRecord(v[key]) === null) return null;
+ }
+ for (const r of Object.values(v.previews)) if (asDeployRecord(r) === null) return null;
+ return v as unknown as DeployedFile;
+}
+
+// --- read / write -------------------------------------------------------------
+
+async function readAs<T>(file: string, as: (v: unknown) => T | null): Promise<T | null> {
+ const r = await readJsonFile(file);
+ return r.ok ? as(r.value) : null;
+}
+
+export function readIndexStamp(paths: Pick<Paths, "exportIndexDir">): Promise<IndexStamp | null> {
+ return readAs(indexStampPath(paths), asIndexStamp);
+}
+
+export function writeIndexStamp(paths: Pick<Paths, "exportIndexDir">, stamp: IndexStamp): Promise<void> {
+ return writeJsonAtomic(indexStampPath(paths), stamp, { mkdir: true });
+}
+
+export function readBuiltStamp(
+ paths: Pick<Paths, "exportBuildsDir">,
+ target: string,
+): Promise<BuiltStamp | null> {
+ return readAs(builtStampPath(paths, target), asBuiltStamp);
+}
+
+export function writeBuiltStamp(paths: Pick<Paths, "exportBuildsDir">, stamp: BuiltStamp): Promise<void> {
+ return writeJsonAtomic(builtStampPath(paths, stamp.target), stamp, { mkdir: true });
+}
+
+export function readDeployedFile(
+ paths: Pick<Paths, "exportBuildsDir">,
+ target: string,
+): Promise<DeployedFile | null> {
+ return readAs(deployedPath(paths, target), asDeployedFile);
+}
+
+/** The record a deploy of `kind` (and `branch`, for a preview) left, or null. */
+export function deployRecordFor(
+ file: DeployedFile | null,
+ kind: DeployKind,
+ branch?: string,
+): DeployRecord | null {
+ if (!file) return null;
+ if (kind === "production") return file.production ?? null;
+ if (kind === "local") return file.local ?? null;
+ return branch ? (file.previews[branch] ?? null) : null;
+}
+
+/**
+ * Record one deploy into `<target>/deployed.json`, keeping every other record
+ * (a malformed file is replaced). Returns the file written.
+ */
+export async function recordDeploy(
+ paths: Pick<Paths, "exportBuildsDir">,
+ target: string,
+ record: DeployRecord,
+): Promise<DeployedFile> {
+ const prev = (await readDeployedFile(paths, target)) ?? { v: 1 as const, target, previews: {} };
+ const next: DeployedFile = { ...prev, v: 1, target, previews: { ...prev.previews } };
+ if (record.kind === "production") next.production = record;
+ else if (record.kind === "local") next.local = record;
+ else if (record.branch) next.previews[record.branch] = record;
+ await writeJsonAtomic(deployedPath(paths, target), next, { mkdir: true });
+ return next;
+}
