commit 44a98e5f715f8745cdb4914e2eec29edd97b5961
parent c36a19c846d57f29b34ebb1e217ceb4623b9600e
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 6 Oct 2026 09:39:13 -0400
publish: the lock's host is ARCHILYZER_HOST_ID (else the hostname); a pid whose process started after the lock was taken is not its holder; a foreign host's lock says how to clear it (review MEDIUM 2)
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
3 files changed, 100 insertions(+), 10 deletions(-)
diff --git a/common/publish/stageLock.test.ts b/common/publish/stageLock.test.ts
@@ -7,8 +7,11 @@ import type { Paths } from "../lib/paths";
import {
LockWaitCancelled,
acquirePublishLock,
+ START_SLACK_MS,
holderIsGone,
+ lockHostId,
pidStartOf,
+ processStartedAtMs,
publishLockPath,
readLockHolder,
withPublishLock,
@@ -35,16 +38,41 @@ function plant(paths: Paths, holder: Partial<LockHolder>): void {
);
}
-const here = { host: "here", pid: 100, startOf: () => null };
+const here = { host: "here", pid: 100, startOf: () => null, startedAtMs: () => null };
test("holderIsGone: same host and a dead pid, or a pid reused by a later process", () => {
const h: LockHolder = { pid: 7, host: "here", kind: "k", target: "t", since: 1, pidStart: "500" };
assert.equal(holderIsGone(h, { host: "here", isAlive: () => false }), true);
- assert.equal(holderIsGone(h, { host: "here", isAlive: () => true, startOf: () => "500" }), false);
- assert.equal(holderIsGone(h, { host: "here", isAlive: () => true, startOf: () => "900" }), true, "pid reused");
- assert.equal(holderIsGone(h, { host: "here", isAlive: () => true, startOf: () => null }), false);
+ assert.equal(holderIsGone(h, { host: "here", isAlive: () => true, startOf: () => "500", startedAtMs: () => null }), false);
+ assert.equal(holderIsGone(h, { host: "here", isAlive: () => true, startOf: () => "900", startedAtMs: () => null }), true, "pid reused");
+ assert.equal(holderIsGone(h, { host: "here", isAlive: () => true, startOf: () => null, startedAtMs: () => null }), false);
assert.equal(holderIsGone(h, { host: "there", isAlive: () => false }), false, "another host: never");
- assert.equal(holderIsGone({ ...h, pidStart: null }, { host: "here", isAlive: () => true, startOf: () => "1" }), false);
+ assert.equal(holderIsGone({ ...h, pidStart: null }, { host: "here", isAlive: () => true, startOf: () => "1", startedAtMs: () => null }), false);
+});
+
+test("holderIsGone: a live pid whose process started AFTER the lock was taken is not the holder (a recreated container's pid 1)", () => {
+ const since = 1_000_000;
+ const h: LockHolder = { pid: 1, host: "archilyzer-editor", kind: "k", target: "t", since };
+ const probe = (startedAt: number | null) =>
+ holderIsGone(h, { host: "archilyzer-editor", isAlive: () => true, startOf: () => null, startedAtMs: () => startedAt });
+ assert.equal(probe(since + 60_000), true, "started a minute after the lock: stale");
+ assert.equal(probe(since - 60_000), false, "started before the lock: may be the holder");
+ assert.equal(probe(since + 1_000), false, "within btime's slack: not judged");
+ assert.equal(probe(null), false, "no /proc: pid-alive alone decides");
+});
+
+test("the host identity is ARCHILYZER_HOST_ID when set, else the hostname", () => {
+ assert.equal(lockHostId({ ARCHILYZER_HOST_ID: " archilyzer-editor " }), "archilyzer-editor");
+ assert.equal(lockHostId({ ARCHILYZER_HOST_ID: "" }), os.hostname());
+ assert.equal(lockHostId({}), os.hostname());
+});
+
+test("processStartedAtMs: this process started before now, and no such pid is null", () => {
+ if (process.platform === "linux") {
+ const t = processStartedAtMs(process.pid);
+ assert.ok(t !== null && t <= Date.now() + START_SLACK_MS && t > Date.now() - 86_400_000 * 365, String(t));
+ }
+ assert.equal(processStartedAtMs(2 ** 30), null);
});
test("pidStartOf reads this process's start time on Linux and is null for no such pid", () => {
@@ -130,14 +158,19 @@ test("a holder on another host is never stolen; the wait is cancellable", async
try {
plant(paths, { pid: 4242, host: "elsewhere" });
const ac = new AbortController();
+ const logs: string[] = [];
const taking = acquirePublishLock(paths, { kind: "k", target: "t" }, {
...here,
isAlive: () => false, // dead HERE means nothing for a pid over there
pollMs: 5,
signal: ac.signal,
+ onLog: (l) => logs.push(l),
});
setTimeout(() => ac.abort(), 40);
await assert.rejects(taking, LockWaitCancelled);
+ assert.equal(logs.length, 1);
+ assert.match(logs[0], /on ANOTHER host \("elsewhere"; this one is "here"\)/);
+ assert.match(logs[0], /If no publish stage is running there, remove .*\.publish\.lock/);
assert.equal(JSON.parse(readFileSync(publishLockPath(paths), "utf8")).host, "elsewhere", "untouched");
} finally {
rmSync(root, { recursive: true, force: true });
diff --git a/common/publish/stageLock.ts b/common/publish/stageLock.ts
@@ -11,10 +11,20 @@
// over — only when it is on THIS host and its process is gone: the pid does
// not answer (`processIsAlive`, jobs/bootQueuedJobs.ts), or it answers with a
// different start time (`/proc/<pid>/stat`, where there is one: a container's
-// editor comes back with the same small pids on every restart). A lock naming
+// editor comes back with the same small pids on every restart), or the
+// process that pid names now STARTED AFTER the lock was taken (`since`) — so
+// it cannot be the holder, whatever `pidStart` the lock carries. A lock naming
// another host is never stolen — a pid means nothing across a namespace. A
// live holder makes the taker WAIT: a poll every 5 s, one log line, and a
// cancel (the AbortSignal) gives up the wait.
+//
+// THE HOST is `ARCHILYZER_HOST_ID` when set, else `os.hostname()`. In the
+// container the hostname is the container id, new on every recreate — which
+// would make the last container's lock "another host's" for ever — so the
+// compose file sets a fixed ARCHILYZER_HOST_ID (release 18 S5). With a fixed
+// id a recreated container's editor is pid 1 again, alive: the start-time
+// rules above are what tell it from the holder. Where there is no /proc (not
+// Linux) neither start-time rule can be asked, and staleness is pid-alive alone.
import { readFileSync } from "node:fs";
import { mkdir, open, readFile, rm, stat } from "node:fs/promises";
@@ -49,6 +59,40 @@ export function publishLockPath(paths: Pick<Paths, "exportBuildsDir">): string {
return path.join(paths.exportBuildsDir, ".publish.lock");
}
+// /proc reports starttime in USER_HZ ticks, which Linux fixes at 100 for
+// every userspace interface whatever the kernel's HZ.
+const USER_HZ = 100;
+
+/**
+ * When `pid`'s process started, in ms since the epoch — /proc/<pid>/stat's
+ * starttime (ticks since boot) plus /proc/stat's `btime` — or null where
+ * there is no /proc. `btime` is whole seconds, so this is good to about 1 s.
+ */
+export function processStartedAtMs(pid: number): number | null {
+ const raw = pidStartOf(pid);
+ const ticks = raw === null ? NaN : Number(raw);
+ if (!Number.isFinite(ticks)) return null;
+ try {
+ const btime = /^btime (\d+)$/m.exec(readFileSync("/proc/stat", "utf8"));
+ if (!btime) return null;
+ return Number(btime[1]) * 1000 + (ticks * 1000) / USER_HZ;
+ } catch {
+ return null;
+ }
+}
+
+// The slack on "started after the lock was taken": btime's whole seconds.
+export const START_SLACK_MS = 2_000;
+
+// The host identity's override (release 18 S5 declares it in lib/envVars.ts;
+// read by name here until both slices are merged).
+const HOST_ID_NAME = "ARCHILYZER_HOST_ID";
+
+/** This machine's identity for the lock: ARCHILYZER_HOST_ID, else the hostname. */
+export function lockHostId(envVars: NodeJS.ProcessEnv = process.env): string {
+ return envVars[HOST_ID_NAME]?.trim() || os.hostname();
+}
+
/** A process's start time from /proc (Linux), or null where there is none. */
export function pidStartOf(pid: number): string | null {
try {
@@ -67,15 +111,18 @@ export type LockEnv = {
pid?: number;
isAlive?: (pid: number) => boolean;
startOf?: (pid: number) => string | null;
+ // When the process `pid` names now started (ms), or null when unknown.
+ startedAtMs?: (pid: number) => number | null;
now?: () => number;
};
function env(e: LockEnv = {}) {
return {
- host: e.host ?? os.hostname(),
+ host: e.host ?? lockHostId(),
pid: e.pid ?? process.pid,
isAlive: e.isAlive ?? processIsAlive,
startOf: e.startOf ?? pidStartOf,
+ startedAtMs: e.startedAtMs ?? processStartedAtMs,
now: e.now ?? Date.now,
};
}
@@ -86,13 +133,16 @@ function env(e: LockEnv = {}) {
* the injected probes.
*/
export function holderIsGone(holder: LockHolder, e: LockEnv = {}): boolean {
- const { host, isAlive, startOf } = env(e);
+ const { host, isAlive, startOf, startedAtMs } = env(e);
if (holder.host !== host) return false;
if (!isAlive(holder.pid)) return true;
if (holder.pidStart) {
const now = startOf(holder.pid);
if (now !== null && now !== holder.pidStart) return true;
}
+ // The pid's process started after the lock was taken: not the holder.
+ const started = startedAtMs(holder.pid);
+ if (started !== null && started > holder.since + START_SLACK_MS) return true;
return false;
}
@@ -199,7 +249,14 @@ export async function acquirePublishLock(
continue;
} else if (!said) {
said = true;
- opts.onLog?.(`[publish] waiting for the publish lock — held by ${describeHolder(current)}\n`);
+ opts.onLog?.(
+ current.host === e.host
+ ? `[publish] waiting for the publish lock — held by ${describeHolder(current)}\n`
+ : `[publish] waiting for the publish lock — held by ${describeHolder(current)}, on ANOTHER host ` +
+ `("${current.host}"; this one is "${e.host}"), which this one can never judge stale. If no ` +
+ `publish stage is running there, remove ${file} (or give both the same ARCHILYZER_HOST_ID ` +
+ `when they are one machine)\n`,
+ );
}
await sleep(opts.pollMs ?? LOCK_POLL_MS, opts.signal);
}
diff --git a/common/publish/stageRun.test.ts b/common/publish/stageRun.test.ts
@@ -204,7 +204,7 @@ test("a live holder of the publish lock is waited for; a cancel during the wait
mkdirSync(path.dirname(publishLockPath(paths)), { recursive: true });
writeFileSync(
publishLockPath(paths),
- JSON.stringify({ pid: process.pid, host: os.hostname(), kind: "build-site", target: "x", since: 1 }),
+ JSON.stringify({ pid: process.pid, host: os.hostname(), kind: "build-site", target: "x", since: Date.now() }),
);
try {
const ac = new AbortController();
