// Tests for the global e2e queue and the heavy slot (scripts/queue-lock.mjs). // // Every test drives the real CLI against throwaway lock files via // E2E_QUEUE_LOCK_FILE / HEAVY_LOCK_FILE, so none of them can touch the actual // .git/e2e-queue.lock or .git/heavy-queue.lock or any real port. The e2e-queue // tests run with HEAVY=0 (they are about the queue); the heavy tests turn it // on with their own lock and a fake /proc/meminfo. Run with: pnpm test:scripts import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import fs from "node:fs"; import net from "node:net"; import os from "node:os"; import path from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; import { parseMeminfo, waitForMemory } from "./queue-lock.mjs"; const SCRIPT = fileURLToPath(new URL("./queue-lock.mjs", import.meta.url)); function tmpDir() { return fs.mkdtempSync(path.join(os.tmpdir(), "queue-lock-test-")); } // The environment a run sees: `env` over the current one. E2E_PORT_CHECK // defaults off so tests that are not about the preflight never probe a port; // HEAVY defaults off so the e2e-queue tests never take a heavy slot. A // pass-through marker inherited from whatever runs this suite (a // `pnpm heavy -- pnpm test:scripts`) is dropped unless the test sets it. function lockEnv(env) { const out = { E2E_PORT_CHECK: "0", HEAVY: "0", ...process.env, ...env }; for (const k of ["HEAVY_HELD", "QUEUE_LOCK_HELD", "E2E_QUEUE"]) { if (!(k in env)) delete out[k]; } return out; } // Start the wrapper; `done` resolves at exit with the captured output, and // `waitFor(re)` resolves once stderr matches — how a test knows a contender is // queued (its banner is out) rather than guessing with a delay. function startLock(args, env = {}) { const child = spawn(process.execPath, [SCRIPT, ...args], { env: lockEnv(env), stdio: ["ignore", "pipe", "pipe"], }); let stdout = ""; let stderr = ""; const waiters = []; child.stdout.on("data", (d) => (stdout += d)); child.stderr.on("data", (d) => { stderr += d; for (const w of waiters) if (w.re.test(stderr)) w.resolve(); }); const done = new Promise((resolve) => child.on("exit", (code) => resolve({ code, stdout, stderr })), ); const waitFor = (re) => new Promise((resolve, reject) => { if (re.test(stderr)) return resolve(); waiters.push({ re, resolve }); done.then(() => reject(new Error(`exited before stderr matched ${re}: ${stderr}`))); }); return { child, done, waitFor }; } function runLock(args, env = {}) { return startLock(args, env).done; } // Resolves once `check()` is true (a holder.json written: a run has acquired). async function until(check, ms = 10_000) { const t0 = Date.now(); while (!check()) { if (Date.now() - t0 > ms) throw new Error("timed out waiting"); await delay(20); } } // A command that records "S" when it starts and "E" when it ends, so // overlapping runs are visible as interleaving in the log. function markerCmd(logFile, id, holdMs) { return [ process.execPath, "-e", `const fs=require("fs");` + `fs.appendFileSync(${JSON.stringify(logFile)},"S${id}");` + `setTimeout(()=>fs.appendFileSync(${JSON.stringify(logFile)},"E${id}"),${holdMs});`, ]; } const delay = (ms) => new Promise((r) => setTimeout(r, ms)); test("serializes concurrent invocations", async () => { const dir = tmpDir(); const lock = path.join(dir, "q.lock"); const log = path.join(dir, "order.log"); const env = { E2E_QUEUE_LOCK_FILE: lock }; const runs = ["1", "2", "3"].map((id) => runLock(["--", ...markerCmd(log, id, 250)], env), ); for (const r of await Promise.all(runs)) assert.equal(r.code, 0); const order = fs.readFileSync(log, "utf8"); // Each run must fully close before the next opens: S1E1S2E2... in some // permutation of ids, with no S following an unmatched S. assert.match(order, /^(S(\d)E\2){3}$/, `interleaved: ${order}`); }); test("serves waiters in arrival order (FIFO)", async () => { const dir = tmpDir(); const lock = path.join(dir, "q.lock"); const log = path.join(dir, "fifo.log"); const env = { E2E_QUEUE_LOCK_FILE: lock }; // Each arrival waits until the one before it is in line: the first holds // (its holder.json is written), the next two have printed their banner and // had a moment to block in flock. A fixed stagger raced a loaded machine. const first = startLock(["--", ...markerCmd(log, "1", 600)], env); await until(() => fs.existsSync(`${lock}.holder.json`)); const runs = [first.done]; for (const id of ["2", "3"]) { const r = startLock(["--", ...markerCmd(log, id, 300)], env); await r.waitFor(/waiting for the e2e queue/); await delay(150); runs.push(r.done); } await Promise.all(runs); assert.equal(fs.readFileSync(log, "utf8"), "S1E1S2E2S3E3"); }); test("prints a banner naming the holder while waiting", async () => { const dir = tmpDir(); const lock = path.join(dir, "q.lock"); const env = { E2E_QUEUE_LOCK_FILE: lock }; const first = runLock(["--", process.execPath, "-e", "setTimeout(()=>{},600)"], env); await until(() => fs.existsSync(`${lock}.holder.json`)); const second = await runLock(["--", process.execPath, "-e", "0"], env); await first; assert.equal(second.code, 0); assert.match(second.stderr, /waiting for the e2e queue/); assert.match(second.stderr, /pid \d+/); // holder.json was read back assert.match(second.stderr, /acquired after/); }); test("QUEUE_LOCK_HELD passes straight through without deadlocking", async () => { const dir = tmpDir(); const lock = path.join(dir, "q.lock"); const env = { E2E_QUEUE_LOCK_FILE: lock }; // Hold the lock for a good while, then prove a nested invocation (the shape // of root `pnpm e2e` -> `pnpm --filter editor run e2e`) does not wait for it. const held = runLock(["--", process.execPath, "-e", "setTimeout(()=>{},1500)"], env); await delay(300); const t0 = Date.now(); const nested = await runLock(["--", process.execPath, "-e", "0"], { ...env, QUEUE_LOCK_HELD: "e2e", }); const elapsed = Date.now() - t0; assert.equal(nested.code, 0); assert.ok(elapsed < 1000, `nested run waited ${elapsed}ms — it deadlocked on its parent's lock`); assert.doesNotMatch(nested.stderr, /waiting for the e2e queue/); await held; }); test("E2E_QUEUE=0 bypasses the queue entirely", async () => { const dir = tmpDir(); const lock = path.join(dir, "q.lock"); const env = { E2E_QUEUE_LOCK_FILE: lock }; const held = runLock(["--", process.execPath, "-e", "setTimeout(()=>{},1500)"], env); await delay(300); const t0 = Date.now(); const bypass = await runLock(["--", process.execPath, "-e", "0"], { ...env, E2E_QUEUE: "0" }); assert.equal(bypass.code, 0); assert.ok(Date.now() - t0 < 1000, "E2E_QUEUE=0 still waited for the lock"); await held; }); test("a SIGKILLed run releases the lock immediately", async () => { const dir = tmpDir(); const lock = path.join(dir, "q.lock"); const env = lockEnv({ E2E_QUEUE_LOCK_FILE: lock }); const victim = spawn( process.execPath, [SCRIPT, "--", process.execPath, "-e", "setTimeout(()=>{},30000)"], { env, stdio: "ignore" }, ); await delay(500); // let it acquire victim.kill("SIGKILL"); await new Promise((r) => victim.on("exit", r)); // No stale-lock recovery code exists, by design: the kernel drops the lock // when the holder's fd closes. If that ever stops being true, this hangs. const t0 = Date.now(); const next = await runLock(["--", process.execPath, "-e", "0"], { E2E_QUEUE_LOCK_FILE: lock }); assert.equal(next.code, 0); assert.ok(Date.now() - t0 < 5000, "lock survived a SIGKILLed holder"); }); test("a leaked shell-spawned stray does not keep the lock", async (t) => { // The regression test for why the lock is held by a dedicated child rather // than by wrapping the command in `flock` directly: flock(1) does not set // FD_CLOEXEC on its lock fd, so a stray started through a shell — which is // exactly how playwright's webServer launches `pnpm dev:test` — inherits it // and would hold the queue until that stray dies. This suite leaks such // strays for real (see editor/e2e/fixtureProcs.ts). const dir = tmpDir(); const lock = path.join(dir, "q.lock"); const pidFile = path.join(dir, "stray.pid"); await runLock( ["--", "sh", "-c", `setsid sleep 30 >/dev/null 2>&1 & echo $! > ${pidFile}; sleep 0.2`], { E2E_QUEUE_LOCK_FILE: lock }, ); const strayPid = Number(fs.readFileSync(pidFile, "utf8").trim()); t.after(() => { try { process.kill(strayPid, "SIGKILL"); } catch { /* already gone */ } }); assert.doesNotThrow(() => process.kill(strayPid, 0), "stray should still be running"); const t0 = Date.now(); const next = await runLock(["--", process.execPath, "-e", "0"], { E2E_QUEUE_LOCK_FILE: lock }); assert.equal(next.code, 0); assert.ok(Date.now() - t0 < 5000, "a leaked stray held the queue lock"); }); test("the port preflight aborts non-zero, naming the port and its env var", async () => { const dir = tmpDir(); const lock = path.join(dir, "q.lock"); const server = net.createServer(); await new Promise((r) => server.listen(0, "127.0.0.1", r)); const port = server.address().port; const result = await runLock( ["--ports", `TEST_PORT:${port}`, "--port-grace", "0", "--", process.execPath, "-e", "0"], { E2E_QUEUE_LOCK_FILE: lock, E2E_PORT_CHECK: "1" }, ); await new Promise((r) => server.close(r)); assert.notEqual(result.code, 0, "a bound port must abort the run"); assert.match(result.stderr, new RegExp(`TEST_PORT=${port}`)); assert.match(result.stderr, /E2E_PORT_CHECK=0/); // the documented override }); test("the port preflight passes when the port is free", async () => { const dir = tmpDir(); const lock = path.join(dir, "q.lock"); // Bind and release, so we know the port was free at the moment we checked. const server = net.createServer(); await new Promise((r) => server.listen(0, "127.0.0.1", r)); const port = server.address().port; await new Promise((r) => server.close(r)); const result = await runLock( ["--ports", `TEST_PORT:${port}`, "--", process.execPath, "-e", 'console.log("ran")'], { E2E_QUEUE_LOCK_FILE: lock, E2E_PORT_CHECK: "1" }, ); assert.equal(result.code, 0); assert.match(result.stdout, /ran/); }); test("propagates the command's exit code", async () => { const dir = tmpDir(); const lock = path.join(dir, "q.lock"); const result = await runLock(["--", process.execPath, "-e", "process.exit(42)"], { E2E_QUEUE_LOCK_FILE: lock, }); assert.equal(result.code, 42); }); test("removes its holder.json when the run finishes", async () => { const dir = tmpDir(); const lock = path.join(dir, "q.lock"); await runLock(["--", process.execPath, "-e", "0"], { E2E_QUEUE_LOCK_FILE: lock }); assert.equal(fs.existsSync(`${lock}.holder.json`), false); }); // ------------------------------------------------------------- the heavy slot // A fake /proc/meminfo: `availableMb` free of `totalMb`. function meminfo(file, availableMb, totalMb = 32_000) { fs.writeFileSync( file, `MemTotal: ${totalMb * 1024} kB\nMemFree: 1024 kB\nMemAvailable: ${availableMb * 1024} kB\n`, ); } // The heavy slot on, against its own lock and a roomy fake meminfo. function heavyEnv(dir, extra = {}) { const mem = path.join(dir, "meminfo"); if (!fs.existsSync(mem)) meminfo(mem, 20_000); return { HEAVY: "1", HEAVY_LOCK_FILE: path.join(dir, "heavy.lock"), HEAVY_MEMINFO_FILE: mem, HEAVY_POLL_MS: "50", ...extra, }; } test("parseMeminfo reads MemAvailable and MemTotal in MB", () => { assert.deepEqual( parseMeminfo("MemTotal: 32768000 kB\nMemFree: 1 kB\nMemAvailable: 6144000 kB\n"), { availableMb: 6000, totalMb: 32000 }, ); assert.equal(parseMeminfo("nothing here"), null); }); test("waitForMemory waits for the floor, polling the injected reader", async () => { const readings = [2000, 4000, 5999, 6000]; const lines = []; let clock = 0; const res = await waitForMemory({ minMb: 6000, read: () => ({ availableMb: readings.shift() ?? 6000, totalMb: 32_000 }), pollMs: 1000, tickMs: 2000, log: (l) => lines.push(l), now: () => clock, sleep: async (ms) => { clock += ms; }, }); assert.equal(res.waitedMs, 3000); assert.match(lines[0], /waiting for memory — 2000 MB available, the floor is 6000 MB/); assert.ok(lines.some((l) => /still waiting for memory \(5999 MB/.test(l)), lines.join("")); assert.match(lines.at(-1), /6000 MB available after 3s/); }); test("waitForMemory: no wait above the floor, at 0, or under a MemTotal that can never reach it", async () => { const never = () => { throw new Error("must not sleep"); }; const read = (a, t = 32_000) => () => ({ availableMb: a, totalMb: t }); assert.deepEqual(await waitForMemory({ minMb: 6000, read: read(9000), sleep: never }), { waitedMs: 0 }); assert.deepEqual(await waitForMemory({ minMb: 0, read: read(10), sleep: never }), { skipped: "off" }); const lines = []; assert.deepEqual( await waitForMemory({ minMb: 6000, read: read(100, 4000), sleep: never, log: (l) => lines.push(l) }), { skipped: "total" }, ); assert.match(lines.join(""), /4000 MB in all, under the 6000 MB floor/); assert.deepEqual( await waitForMemory({ minMb: 6000, read: () => null, sleep: never, log: () => {} }), { skipped: "unreadable" }, ); }); test("waitForMemory gives up past its timeout", async () => { let clock = 0; await assert.rejects( waitForMemory({ minMb: 6000, read: () => ({ availableMb: 100, totalMb: 32_000 }), pollMs: 1000, timeoutMs: 3000, log: () => {}, now: () => clock, sleep: async (ms) => { clock += ms; }, }), (err) => err.timeout === true, ); }); test("two heavy contenders run one at a time; the second names what it waits behind", async () => { const dir = tmpDir(); const log = path.join(dir, "order.log"); const env = heavyEnv(dir); const first = startLock(["--heavy", "--", ...markerCmd(log, "1", 600)], env); await until(() => fs.existsSync(`${env.HEAVY_LOCK_FILE}.holder.json`)); const second = await runLock(["--heavy", "--", ...markerCmd(log, "2", 50)], env); assert.equal((await first.done).code, 0); assert.equal(second.code, 0); assert.equal(fs.readFileSync(log, "utf8"), "S1E1S2E2"); assert.match(second.stderr, /waiting for the heavy slot — held by .*pid \d+.*: .*appendFileSync/); assert.equal(fs.existsSync(`${env.HEAVY_LOCK_FILE}.holder.json`), false); }); test("an e2e run takes the heavy slot too: it waits for a heavy job, then runs", async () => { const dir = tmpDir(); const log = path.join(dir, "order.log"); const env = heavyEnv(dir, { E2E_QUEUE_LOCK_FILE: path.join(dir, "q.lock") }); const build = startLock(["--heavy", "--", ...markerCmd(log, "b", 600)], env); await until(() => fs.existsSync(`${env.HEAVY_LOCK_FILE}.holder.json`)); const e2e = await runLock(["--", ...markerCmd(log, "e", 50)], env); await build.done; assert.equal(e2e.code, 0); assert.equal(fs.readFileSync(log, "utf8"), "SbEbSeEe"); assert.match(e2e.stderr, /waiting for the heavy slot/); }); test("a heavy run inside a heavy run passes through (no self-deadlock), and so does an e2e run inside one", async () => { const dir = tmpDir(); const env = heavyEnv(dir, { E2E_QUEUE_LOCK_FILE: path.join(dir, "q.lock") }); const inner = `${JSON.stringify(process.execPath)} ${JSON.stringify(SCRIPT)}`; const res = await runLock( ["--heavy", "--", "sh", "-c", `${inner} --heavy -- true && ${inner} -- true && echo nested-ok`], env, ); assert.equal(res.code, 0, res.stderr); assert.match(res.stdout, /nested-ok/); assert.doesNotMatch(res.stderr, /waiting for the heavy slot/); }); test("a heavy run waits under the memory floor and starts once memory is back", async () => { const dir = tmpDir(); const env = heavyEnv(dir); meminfo(env.HEAVY_MEMINFO_FILE, 1500); const run = startLock(["--heavy", "--", process.execPath, "-e", 'console.log("ran")'], env); await run.waitFor(/waiting for memory — 1500 MB available, the floor is 6000 MB/); meminfo(env.HEAVY_MEMINFO_FILE, 7000); const res = await run.done; assert.equal(res.code, 0); assert.match(res.stdout, /ran/); assert.match(res.stderr, /7000 MB available after/); }); test("HEAVY_MIN_FREE_MB moves the floor; HEAVY=0 skips slot and floor", async () => { const dir = tmpDir(); const env = heavyEnv(dir); meminfo(env.HEAVY_MEMINFO_FILE, 1500); const lowered = await runLock(["--heavy", "--", "true"], { ...env, HEAVY_MIN_FREE_MB: "1000" }); assert.equal(lowered.code, 0); assert.doesNotMatch(lowered.stderr, /waiting for memory/); const off = await runLock(["--heavy", "--", "true"], { ...env, HEAVY: "0" }); assert.equal(off.code, 0); assert.equal(off.stderr, ""); }); test("a SIGKILLed heavy holder hands the slot on at once (the stale holder)", async () => { const dir = tmpDir(); const env = heavyEnv(dir); const victim = startLock(["--heavy", "--", process.execPath, "-e", "setTimeout(()=>{},30000)"], env); await until(() => fs.existsSync(`${env.HEAVY_LOCK_FILE}.holder.json`)); victim.child.kill("SIGKILL"); await victim.done; // Its holder.json is left behind (SIGKILL runs no cleanup); the lock is not. const t0 = Date.now(); const next = await runLock(["--heavy", "--", "true"], env); assert.equal(next.code, 0); assert.ok(Date.now() - t0 < 5000, "the heavy slot survived a SIGKILLed holder"); }); test("pnpm's `--` separator is accepted before a heavy command", async () => { const dir = tmpDir(); const res = await runLock( ["--heavy", "--", "--", process.execPath, "-e", 'console.log("ran")'], heavyEnv(dir), ); assert.equal(res.code, 0, res.stderr); assert.match(res.stdout, /ran/); }); test("SIGTERM to a heavy wrapper alone stops its command (a stage's Cancel)", async () => { const dir = tmpDir(); const pidFile = path.join(dir, "cmd.pid"); const run = startLock( [ "--heavy", "--", process.execPath, "-e", `require("fs").writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)); setTimeout(()=>{},30000)`, ], heavyEnv(dir), ); await until(() => fs.existsSync(pidFile) && fs.readFileSync(pidFile, "utf8") !== ""); const cmdPid = Number(fs.readFileSync(pidFile, "utf8")); const t0 = Date.now(); run.child.kill("SIGTERM"); const res = await run.done; assert.ok(Date.now() - t0 < 5000, "the wrapper outlived its SIGTERM"); assert.equal(res.code, 128 + os.constants.signals.SIGTERM); assert.throws(() => process.kill(cmdPid, 0), "the command survived the wrapper's SIGTERM"); });