// The publish lock (release 18): ONE publish stage at a time on this machine, // whoever started it — a stage child the editor spawned, or `archilyzer // publish …` run by hand (`docker compose exec editor …` included). The // editor's `publish` queue already runs its stages one by one; this file is // what keeps a CLI started beside it from building into the same shared // export/public at once. // // /.publish.lock {pid, host, kind, target, since, pidStart} // // Taken with O_EXCL (`open(…, "wx")`). A holder is STALE — and its lock taken // over — only when it is on THIS host and its process is gone: the pid does // not answer (`processIsAlive`, jobs/bootQueuedJobs.ts), or it answers with a // different start time (`/proc//stat`, where there is one: a container's // editor comes back with the same small pids on every restart), or the // process that pid names now STARTED AFTER the lock was taken (`since`) — so // it cannot be the holder, whatever `pidStart` the lock carries. A lock naming // another host is never stolen — a pid means nothing across a namespace. A // live holder makes the taker WAIT: a poll every 5 s, one log line, and a // cancel (the AbortSignal) gives up the wait. // // THE HOST is `ARCHILYZER_HOST_ID` when set, else `os.hostname()`. In the // container the hostname is the container id, new on every recreate — which // would make the last container's lock "another host's" for ever — so the // compose file sets a fixed ARCHILYZER_HOST_ID (release 18 S5). With a fixed // id a recreated container's editor is pid 1 again, alive: the start-time // rules above are what tell it from the holder. Where there is no /proc (not // Linux) neither start-time rule can be asked, and staleness is pid-alive alone. import { readFileSync } from "node:fs"; import { mkdir, open, readFile, rm, stat } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { processIsAlive } from "../jobs/bootQueuedJobs"; import type { Paths } from "../lib/paths"; export type LockHolder = { pid: number; host: string; kind: string; target: string; since: number; // The holder process's start time (/proc//stat field 22), when known. pidStart?: string | null; }; export const LOCK_POLL_MS = 5_000; // A lock file that stays unreadable this long was left by a taker that died // between creating it and writing it. export const LOCK_TORN_GRACE_MS = 60_000; export class LockWaitCancelled extends Error { constructor() { super("cancelled while waiting for the publish lock"); this.name = "LockWaitCancelled"; } } export function publishLockPath(paths: Pick): string { return path.join(paths.exportBuildsDir, ".publish.lock"); } // /proc reports starttime in USER_HZ ticks, which Linux fixes at 100 for // every userspace interface whatever the kernel's HZ. const USER_HZ = 100; /** * When `pid`'s process started, in ms since the epoch — /proc//stat's * starttime (ticks since boot) plus /proc/stat's `btime` — or null where * there is no /proc. `btime` is whole seconds, so this is good to about 1 s. */ export function processStartedAtMs(pid: number): number | null { const raw = pidStartOf(pid); const ticks = raw === null ? NaN : Number(raw); if (!Number.isFinite(ticks)) return null; try { const btime = /^btime (\d+)$/m.exec(readFileSync("/proc/stat", "utf8")); if (!btime) return null; return Number(btime[1]) * 1000 + (ticks * 1000) / USER_HZ; } catch { return null; } } // The slack on "started after the lock was taken": btime's whole seconds. export const START_SLACK_MS = 2_000; // The host identity's override (release 18 S5 declares it in lib/envVars.ts; // read by name here until both slices are merged). const HOST_ID_NAME = "ARCHILYZER_HOST_ID"; /** This machine's identity for the lock: ARCHILYZER_HOST_ID, else the hostname. */ export function lockHostId(envVars: NodeJS.ProcessEnv = process.env): string { return envVars[HOST_ID_NAME]?.trim() || os.hostname(); } /** A process's start time from /proc (Linux), or null where there is none. */ export function pidStartOf(pid: number): string | null { try { const raw = readFileSync(`/proc/${pid}/stat`, "utf8"); // `pid (comm) state ppid …` — comm may hold spaces and parens, so split // after the LAST ')'. Field 22 (starttime) is index 19 of the rest. const rest = raw.slice(raw.lastIndexOf(")") + 2).split(" "); return rest[19] ?? null; } catch { return null; } } export type LockEnv = { host?: string; pid?: number; isAlive?: (pid: number) => boolean; startOf?: (pid: number) => string | null; // When the process `pid` names now started (ms), or null when unknown. startedAtMs?: (pid: number) => number | null; now?: () => number; }; function env(e: LockEnv = {}) { return { host: e.host ?? lockHostId(), pid: e.pid ?? process.pid, isAlive: e.isAlive ?? processIsAlive, startOf: e.startOf ?? pidStartOf, startedAtMs: e.startedAtMs ?? processStartedAtMs, now: e.now ?? Date.now, }; } /** * True when `holder`'s process is certainly gone: same host, and its pid is * dead or now belongs to a process that started at another time. Pure over * the injected probes. */ export function holderIsGone(holder: LockHolder, e: LockEnv = {}): boolean { const { host, isAlive, startOf, startedAtMs } = env(e); if (holder.host !== host) return false; if (!isAlive(holder.pid)) return true; if (holder.pidStart) { const now = startOf(holder.pid); if (now !== null && now !== holder.pidStart) return true; } // The pid's process started after the lock was taken: not the holder. const started = startedAtMs(holder.pid); if (started !== null && started > holder.since + START_SLACK_MS) return true; return false; } export function parseLockHolder(text: string): LockHolder | null { try { const v = JSON.parse(text) as Partial; if ( typeof v?.pid !== "number" || typeof v.host !== "string" || typeof v.kind !== "string" || typeof v.target !== "string" || typeof v.since !== "number" ) { return null; } return v as LockHolder; } catch { return null; } } /** Who holds the publish lock now, or null (none, or unreadable). */ export async function readLockHolder(paths: Pick): Promise { try { return parseLockHolder(await readFile(publishLockPath(paths), "utf8")); } catch { return null; } } export function describeHolder(h: LockHolder): string { return `${h.kind} ${h.target} (pid ${h.pid} on ${h.host}, since ${new Date(h.since).toISOString()})`; } export type PublishLock = { holder: LockHolder; release: () => Promise }; function sleep(ms: number, signal?: AbortSignal): Promise { return new Promise((resolve) => { const t = setTimeout(done, ms); function done() { clearTimeout(t); signal?.removeEventListener("abort", done); resolve(); } signal?.addEventListener("abort", done, { once: true }); }); } /** * Take the publish lock for `who`, waiting while a live holder has it. * Throws LockWaitCancelled when `signal` aborts first. */ export async function acquirePublishLock( paths: Pick, who: { kind: string; target: string }, opts: LockEnv & { signal?: AbortSignal; onLog?: (line: string) => void; pollMs?: number } = {}, ): Promise { const e = env(opts); const file = publishLockPath(paths); await mkdir(path.dirname(file), { recursive: true }); let said = false; for (;;) { if (opts.signal?.aborted) throw new LockWaitCancelled(); const holder: LockHolder = { pid: e.pid, host: e.host, kind: who.kind, target: who.target, since: e.now(), pidStart: e.startOf(e.pid), }; try { const fh = await open(file, "wx"); try { await fh.writeFile(JSON.stringify(holder) + "\n"); await fh.sync().catch(() => {}); } finally { await fh.close(); } return { holder, release: () => releaseLock(file, holder) }; } catch (err) { if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err; } // Someone holds it (or held it). let text = ""; try { text = await readFile(file, "utf8"); } catch (err) { if ((err as NodeJS.ErrnoException).code === "ENOENT") continue; // released meanwhile throw err; } const current = parseLockHolder(text); if (current === null) { // Torn: a taker between its create and its write — or one that died // there. Give it the grace, then take the lock over. const age = await stat(file).then((s) => e.now() - s.mtimeMs, () => 0); if (age > LOCK_TORN_GRACE_MS) { await removeIfUnchanged(file, text); continue; } } else if (holderIsGone(current, e)) { opts.onLog?.(`[publish] taking over a stale publish lock: ${describeHolder(current)} is gone\n`); await removeIfUnchanged(file, text); continue; } else if (!said) { said = true; opts.onLog?.( current.host === e.host ? `[publish] waiting for the publish lock — held by ${describeHolder(current)}\n` : `[publish] waiting for the publish lock — held by ${describeHolder(current)}, on ANOTHER host ` + `("${current.host}"; this one is "${e.host}"), which this one can never judge stale. If no ` + `publish stage is running there, remove ${file} (or give both the same ARCHILYZER_HOST_ID ` + `when they are one machine)\n`, ); } await sleep(opts.pollMs ?? LOCK_POLL_MS, opts.signal); } } // Remove the lock only if it still holds exactly what was judged stale. async function removeIfUnchanged(file: string, judged: string): Promise { const now = await readFile(file, "utf8").catch(() => null); if (now === judged) await rm(file, { force: true }); } async function releaseLock(file: string, holder: LockHolder): Promise { const current = parseLockHolder(await readFile(file, "utf8").catch(() => "")); if (current && current.pid === holder.pid && current.host === holder.host && current.since === holder.since) { await rm(file, { force: true }); } } /** Run `fn` holding the publish lock; always released after. */ export async function withPublishLock( paths: Pick, who: { kind: string; target: string }, fn: () => Promise, opts: LockEnv & { signal?: AbortSignal; onLog?: (line: string) => void; pollMs?: number } = {}, ): Promise { const lock = await acquirePublishLock(paths, who, opts); try { return await fn(); } finally { await lock.release(); } }