commit 388bbeb6a543244523bac0c907089a937b292f2b
parent 6e395f005c779f4516f23ae0da98f86f6b1fb579
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 6 Oct 2026 10:11:08 -0400
Merge r18/integration (S5 second half) into r18/deploy-hardening
stamps.ts: S5's one-line imageBuildFacts re-export (and its comment) kept.
envVars.ts merged clean: one row per name, readBy combined; ENVIRONMENT.md
regenerates unchanged. The two shared md files merged clean.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
5 files changed, 286 insertions(+), 15 deletions(-)
diff --git a/common/bin/doctor.test.ts b/common/bin/doctor.test.ts
@@ -69,6 +69,7 @@ function checkout(): { root: string; bin: string; paths: Paths } {
parakeetCliBin: "parakeet-cli",
configDir: path.join(root, ".config"),
exportBuildsDir: path.join(root, "export", ".export-builds"),
+ exportIndexDir: path.join(root, "export", ".export-index"),
sourceScrubFile: path.join(root, ".config", "source-scrub.txt"),
sourceDenylistFile: path.join(root, ".config", "source-denylist.txt"),
// Outside the checkout, as the XDG cache is: the tests that compare the
@@ -809,3 +810,116 @@ test("source publish: source-repo — the variable naming nothing fails, a reada
assert.match(find(r, "source publish", "source-repo")!.detail, /^\/data\/source\.git \(ARCHILYZER_SOURCE_REPO\): main does not read — fatal: detected dubious ownership$/);
assert.deepEqual(tree(c.root), before);
});
+
+// ── release 18, second half: the lock, the index stamp, the wrangler floor ─
+
+function indexStampJson(stampId: string, builtAt: number) {
+ return {
+ v: 1, stampId, generation: 7, scannedAt: builtAt - 60_000, builtAt, templatesAt: builtAt, commit: null,
+ index: { shortCircuited: false, added: 1, changed: 0, removed: 0, heldChannels: [] },
+ stats: { shortCircuited: false, notIndexedYet: 0, notIndexable: 0 },
+ sites: {}, hubSig: "h",
+ };
+}
+
+function builtJson(target: string, indexStampId: string | null, bytes: number) {
+ return {
+ v: 1, stampId: `b-${target}`, target, kind: target === "_hub" ? "hub" : "site", indexStampId, inputSig: "s",
+ builtAt: Date.UTC(2026, 9, 6, 10), commit: null, branch: null, runner: "local", audience: "public",
+ corpusGeneratedAt: null, files: 3, bytes, archivesStaged: 0,
+ };
+}
+
+test("publish: publish-lock — free is ok, a running holder a note, a dead one stale with the rm, another host's named and never stale", async () => {
+ const c = checkout();
+ const h = home(c);
+ const builds = c.paths.exportBuildsDir;
+ mkdirSync(builds, { recursive: true });
+ const file = path.join(builds, ".publish.lock");
+ let r = await run(c, { HOME: h });
+ assert.equal(find(r, "publish", "publish-lock")?.status, "ok");
+ const { pidStartOf } = await import("../publish/stageLock");
+ const hold = (holder: Record<string, unknown>) => writeFileSync(file, JSON.stringify(holder));
+ // This process holds it: alive, same host.
+ hold({ pid: process.pid, host: "doctor-host", kind: "build-site", target: "alpha", since: Date.now() - 5_000, pidStart: pidStartOf(process.pid) });
+ r = await run(c, { HOME: h, ARCHILYZER_HOST_ID: "doctor-host" });
+ assert.equal(find(r, "publish", "publish-lock")?.status, "info");
+ assert.match(find(r, "publish", "publish-lock")!.detail, /^held: build-site alpha \(pid \d+ on doctor-host/);
+ // A pid that is not running, same host: stale.
+ hold({ pid: 2 ** 22 - 3, host: "doctor-host", kind: "deploy-site", target: "alpha", since: Date.now() - 60_000 });
+ r = await run(c, { HOME: h, ARCHILYZER_HOST_ID: "doctor-host" });
+ assert.equal(find(r, "publish", "publish-lock")?.status, "warn");
+ assert.match(find(r, "publish", "publish-lock")!.detail, new RegExp(`^stale: deploy-site alpha .* the next stage takes it over, or clear it, when nothing is publishing: rm ${file.replace(/[.]/g, "\\.")}$`));
+ // The same dead pid on ANOTHER host: named, never judged.
+ r = await run(c, { HOME: h, ARCHILYZER_HOST_ID: "this-host" });
+ assert.equal(find(r, "publish", "publish-lock")?.status, "warn");
+ assert.match(find(r, "publish", "publish-lock")!.detail, /^held by ANOTHER host: deploy-site alpha \(pid \d+ on doctor-host.*this host is "this-host"/);
+ // A lock that does not parse: being written, then (past the grace) torn.
+ writeFileSync(file, "{");
+ r = await run(c, { HOME: h });
+ assert.equal(find(r, "publish", "publish-lock")?.status, "info");
+ const before = tree(c.root);
+ r = await run(c, { HOME: h }, { now: new Date(Date.now() + 10 * 60_000) });
+ assert.equal(find(r, "publish", "publish-lock")?.status, "warn");
+ assert.match(find(r, "publish", "publish-lock")!.detail, /does not parse.*a taker died writing it/);
+ assert.equal(r.ok, true, "a warning, never a failure");
+ assert.deepEqual(tree(c.root), before, "the doctor never clears a lock");
+});
+
+test("publish: index-stamp — none is a note (a warning once something is built); its age; targets built from an older stamp; export-builds counts built.json bytes", async () => {
+ const c = checkout();
+ const h = home(c);
+ const builds = c.paths.exportBuildsDir;
+ let r = await run(c, { HOME: h });
+ assert.equal(find(r, "publish", "index-stamp")?.status, "info");
+ assert.match(find(r, "publish", "index-stamp")!.detail, /no index stamp at .*stamp\.json — update the index first: archilyzer publish index$/);
+ // Built bundles with no stamp: a warning.
+ for (const [target, id, bytes] of [["alpha", "old-stamp", 7_000_000], ["_hub", "cur-stamp", 2_000_000]] as const) {
+ mkdirSync(path.join(builds, target, "out"), { recursive: true });
+ writeFileSync(path.join(builds, target, "out", "index.html"), "x");
+ writeFileSync(path.join(builds, target, "built.json"), JSON.stringify(builtJson(target, id, bytes)));
+ }
+ r = await run(c, { HOME: h }, { freeBytes: () => 5e9 });
+ assert.equal(find(r, "publish", "index-stamp")?.status, "warn");
+ // export-builds sums the stamps' bytes, not the one-byte files on disk.
+ assert.match(find(r, "publish", "export-builds")!.detail, /: 2 bundles, 9 MB; 5\.00 GB free$/);
+ // A stamp: alpha was built from an older one.
+ const now = Date.UTC(2026, 9, 6, 12);
+ mkdirSync(c.paths.exportIndexDir, { recursive: true });
+ writeFileSync(path.join(c.paths.exportIndexDir, "stamp.json"), JSON.stringify(indexStampJson("cur-stamp", now - 3 * 3_600_000)));
+ const before = tree(c.root);
+ r = await run(c, { HOME: h }, { now: new Date(now) });
+ assert.equal(find(r, "publish", "index-stamp")?.status, "warn");
+ assert.match(find(r, "publish", "index-stamp")!.detail, /^cur-stamp, built 2026-10-06 09:00 \(3 hours ago\), generation 7; built from an older stamp: alpha — archilyzer publish build <id> rebuilds each$/);
+ // Everything from the current stamp: ok.
+ writeFileSync(path.join(builds, "alpha", "built.json"), JSON.stringify(builtJson("alpha", "cur-stamp", 7_000_000)));
+ const before2 = tree(c.root);
+ r = await run(c, { HOME: h }, { now: new Date(now) });
+ assert.equal(find(r, "publish", "index-stamp")?.status, "ok");
+ assert.match(find(r, "publish", "index-stamp")!.detail, /generation 7; 2 bundles built from it$/);
+ assert.equal(r.ok, true);
+ assert.notDeepEqual(before, before2);
+ assert.deepEqual(tree(c.root), before2);
+});
+
+test("workspace: node is graded against the pinned wrangler's engines floor when wrangler is installed — a warning below it, never a failure", async () => {
+ const c = checkout();
+ // No wrangler installed: next's floor only.
+ let r = await run(c, {}, { nodeVersion: "20.11.0" });
+ assert.equal(find(r, "workspace", "node")?.status, "ok");
+ assert.equal(find(r, "workspace", "node")!.detail, "v20.11.0 (needs >= 20.9.0)");
+ const pkg = path.join(c.root, "common", "node_modules", "wrangler");
+ mkdirSync(pkg, { recursive: true });
+ writeFileSync(path.join(pkg, "package.json"), JSON.stringify({ name: "wrangler", version: "4.147.0", engines: { node: ">=22.0.0" } }));
+ const before = tree(c.root);
+ r = await run(c, {}, { nodeVersion: "20.11.0" });
+ assert.equal(find(r, "workspace", "node")?.status, "warn");
+ assert.match(find(r, "workspace", "node")!.detail, /^v20\.11\.0 — runs the apps .* the pinned wrangler 4\.147\.0 needs node >=22\.0\.0: every deploy refuses; use Node 22$/);
+ assert.equal(r.ok, true);
+ r = await run(c, {}, { nodeVersion: "22.23.3" });
+ assert.equal(find(r, "workspace", "node")?.status, "ok");
+ assert.equal(find(r, "workspace", "node")!.detail, "v22.23.3 (needs >= 20.9.0; deploys: >= 22.0.0, wrangler 4.147.0)");
+ r = await run(c, {}, { nodeVersion: "18.20.0" });
+ assert.equal(find(r, "workspace", "node")?.status, "fail");
+ assert.deepEqual(tree(c.root), before);
+});
diff --git a/common/bin/doctor.ts b/common/bin/doctor.ts
@@ -9,7 +9,9 @@
// there and older than its Dockerfile (common/publish/build.ts), and what a
// publish needs from this machine (release 18): which yt-dlp (the image's or
// an override), whether deploy credentials are SET (never their values), room
-// for the bundles, and the repository the source mirror reads.
+// for the bundles, the publish lock (free, held, stale — never cleared here),
+// the index stamp and what was built from an older one, the repository the
+// source mirror reads, and node against the pinned wrangler's floor.
//
// STRICTLY READ-ONLY. It stats, reads and runs version flags, plus the engine's
// `image inspect`, a lock-free `git status` / `git log` and a `git rev-parse`
@@ -134,9 +136,21 @@ export async function collectDoctorReport(deps: DoctorDeps): Promise<DoctorRepor
// ── workspace ────────────────────────────────────────────────────────────
const W = "workspace";
const nodeV = deps.nodeVersion ?? process.versions.node;
- add(W, "node", versionAtLeast(nodeV, MIN_NODE) ? "ok" : "fail",
- `v${nodeV} (needs >= ${MIN_NODE.join(".")})`);
const root = paths.monorepoRoot;
+ // Two floors: next's (a failure — nothing runs below it) and the pinned
+ // wrangler's engines (a warning — every deploy refuses below it; wrangler 4
+ // wants 22). The wrangler floor is read from its package.json when it is
+ // installed, as the image's drift test reads it.
+ const wranglerFloor = wranglerNodeFloor(root);
+ if (!versionAtLeast(nodeV, MIN_NODE)) {
+ add(W, "node", "fail", `v${nodeV} (needs >= ${MIN_NODE.join(".")})`);
+ } else if (wranglerFloor && !versionAtLeast(nodeV, wranglerFloor.min)) {
+ add(W, "node", "warn",
+ `v${nodeV} — runs the apps (>= ${MIN_NODE.join(".")}), but the pinned wrangler ${wranglerFloor.version} needs node ${wranglerFloor.range}: every deploy refuses; use Node ${wranglerFloor.min[0]}`);
+ } else {
+ add(W, "node", "ok",
+ `v${nodeV} (needs >= ${MIN_NODE.join(".")}${wranglerFloor ? `; deploys: >= ${wranglerFloor.min.join(".")}, wrangler ${wranglerFloor.version}` : ""})`);
+ }
if (existsSync(path.join(root, "pnpm-workspace.yaml"))) {
add(W, "checkout", "ok", root);
} else {
@@ -509,7 +523,7 @@ export async function collectDoctorReport(deps: DoctorDeps): Promise<DoctorRepor
} else if (!writable(builds)) {
add(PB, "export-builds", "warn", `${builds} is not writable — every site build fails (${freeText})`);
} else {
- const bundles = await bundleBytes(builds);
+ const bundles = await bundleBytes(paths);
const need = Math.ceil(bundles.bytes * 1.5);
const what = `${builds}: ${bundles.count} bundle${bundles.count === 1 ? "" : "s"}, ${gigabytes(bundles.bytes)}; ${freeText}`;
if (free !== null && bundles.count > 0 && free < need) {
@@ -520,6 +534,63 @@ export async function collectDoctorReport(deps: DoctorDeps): Promise<DoctorRepor
}
}
}
+
+ // The publish lock (common/publish/stageLock.ts): free, held by a stage
+ // that is running, or left by one that is gone — judged by the lock's own
+ // rule (holderIsGone), never removed here.
+ if (builds) {
+ const lock = await import("../publish/stageLock");
+ const file = lock.publishLockPath(paths);
+ const st = statOrNull(file);
+ const clear = `clear it, when nothing is publishing: rm ${file}`;
+ if (!st) {
+ add(PB, "publish-lock", "ok", "free — no stage is publishing");
+ } else {
+ const holder = lock.parseLockHolder(readOrNull(file) ?? "");
+ const host = lock.lockHostId(env);
+ const nowMs = (deps.now ?? new Date()).getTime();
+ if (!holder) {
+ const torn = nowMs - st.mtime.getTime() > lock.LOCK_TORN_GRACE_MS;
+ add(PB, "publish-lock", torn ? "warn" : "info",
+ torn
+ ? `${file} does not parse, and has not for ${ago(st.mtime, new Date(nowMs)).replace(/ ago$/, "")} — a taker died writing it; the next stage takes it over, or ${clear}`
+ : `${file} is being written — a stage is taking the lock`);
+ } else if (holder.host !== host) {
+ add(PB, "publish-lock", "warn",
+ `held by ANOTHER host: ${lock.describeHolder(holder)} — never taken over from here (this host is "${host}"); if that host is gone, ${clear}`);
+ } else if (lock.holderIsGone(holder, { host })) {
+ add(PB, "publish-lock", "warn",
+ `stale: ${lock.describeHolder(holder)} — its process is gone; the next stage takes it over, or ${clear}`);
+ } else {
+ add(PB, "publish-lock", "info", `held: ${lock.describeHolder(holder)} — a stage is running`);
+ }
+ }
+ }
+
+ // The index stamp (common/publish/stamps.ts): how old, and which built
+ // targets came from an older one (they rebuild on their next build).
+ if (paths.exportIndexDir) {
+ const stamps = await import("../publish/stamps");
+ const idx = await stamps.readIndexStamp(paths);
+ const built = builds ? await builtStamps(paths) : [];
+ const now = deps.now ?? new Date();
+ if (!idx) {
+ const something = built.length > 0 || (await configuredSiteIds(paths)).length > 0;
+ add(PB, "index-stamp", something ? "warn" : "info",
+ `no index stamp at ${stamps.indexStampPath(paths)} — update the index first: archilyzer publish index`);
+ } else {
+ const at = new Date(idx.builtAt);
+ const head = `${idx.stampId}, built ${stamp(at)} (${ago(at, now)}), generation ${idx.generation}`;
+ const older = built.filter((b) => b.indexStampId !== idx.stampId).map((b) => b.target);
+ if (older.length > 0) {
+ add(PB, "index-stamp", "warn",
+ `${head}; built from an older stamp: ${older.join(", ")} — archilyzer publish build <id> rebuilds each`);
+ } else {
+ add(PB, "index-stamp", "ok",
+ `${head}${built.length > 0 ? `; ${built.length} bundle${built.length === 1 ? "" : "s"} built from it` : "; nothing built yet"}`);
+ }
+ }
+ }
}
// ── source publish ───────────────────────────────────────────────────────
@@ -983,9 +1054,12 @@ function statfsFree(dir: string): number | null {
}
}
-// Every `<builds>/<target>/out` bundle and their bytes, by stat (links not
-// followed). A bundle's own size is what a rebuild writes again beside it.
-async function bundleBytes(buildsDir: string): Promise<{ count: number; bytes: number }> {
+// Every `<builds>/<target>/out` bundle and its bytes: the `bytes` its
+// built.json recorded (common/publish/stamps.ts), else — a bundle no stamp
+// describes, from before the stages — a stat walk (links not followed). A
+// bundle's own size is what a rebuild writes again beside it.
+async function bundleBytes(paths: Paths): Promise<{ count: number; bytes: number }> {
+ const { readBuiltStamp } = await import("../publish/stamps");
let count = 0;
let bytes = 0;
const walk = async (d: string): Promise<void> => {
@@ -995,16 +1069,61 @@ async function bundleBytes(buildsDir: string): Promise<{ count: number; bytes: n
else if (ent.isFile()) bytes += statOrNull(p)?.size ?? 0;
}
};
- for (const ent of await readdir(buildsDir, { withFileTypes: true }).catch(() => [])) {
- if (!ent.isDirectory()) continue;
- const out = path.join(buildsDir, ent.name, "out");
+ for (const ent of await readdir(paths.exportBuildsDir, { withFileTypes: true }).catch(() => [])) {
+ if (!ent.isDirectory() || ent.name.startsWith(".")) continue;
+ const out = path.join(paths.exportBuildsDir, ent.name, "out");
if (!statOrNull(out)?.isDirectory()) continue;
count += 1;
- await walk(out);
+ const built = await readBuiltStamp(paths, ent.name);
+ if (built) bytes += built.bytes;
+ else await walk(out);
}
return { count, bytes };
}
+// Every built.json under the builds dir (sites, `_hub`, `_homepage`), by
+// target name. Read-only; an unreadable stamp is skipped.
+async function builtStamps(paths: Paths): Promise<{ target: string; indexStampId: string | null }[]> {
+ const { readBuiltStamp } = await import("../publish/stamps");
+ const out: { target: string; indexStampId: string | null }[] = [];
+ for (const ent of await readdir(paths.exportBuildsDir, { withFileTypes: true }).catch(() => [])) {
+ if (!ent.isDirectory() || ent.name.startsWith(".")) continue;
+ const built = await readBuiltStamp(paths, ent.name);
+ if (built) out.push({ target: built.target, indexStampId: built.indexStampId });
+ }
+ return out.sort((x, y) => x.target.localeCompare(y.target));
+}
+
+// The configured site ids (a sites/<id>/site.json), `_`-dirs excluded.
+async function configuredSiteIds(paths: Paths): Promise<string[]> {
+ if (!paths.sitesDir) return [];
+ const out: string[] = [];
+ for (const e of await readdir(paths.sitesDir, { withFileTypes: true }).catch(() => [])) {
+ if (e.isDirectory() && !e.name.startsWith("_") && existsSync(path.join(paths.sitesDir, e.name, "site.json"))) out.push(e.name);
+ }
+ return out.sort();
+}
+
+// The pinned wrangler's Node floor, from its package.json `engines.node`
+// (">=22.0.0"), when wrangler is installed in common/node_modules; else null.
+function wranglerNodeFloor(root: string): { version: string; range: string; min: [number, number, number] } | null {
+ const text = readOrNull(path.join(root, "common", "node_modules", "wrangler", "package.json"));
+ if (text === null) return null;
+ try {
+ const pkg = JSON.parse(text) as { version?: unknown; engines?: { node?: unknown } };
+ const range = typeof pkg.engines?.node === "string" ? pkg.engines.node : "";
+ const m = /(\d+)(?:\.(\d+))?(?:\.(\d+))?/.exec(range);
+ if (!m) return null;
+ return {
+ version: typeof pkg.version === "string" ? pkg.version : "?",
+ range,
+ min: [Number(m[1]), Number(m[2] ?? 0), Number(m[3] ?? 0)],
+ };
+ } catch {
+ return null;
+ }
+}
+
// The sites whose site.json names a Cloudflare Pages project — what "this
// machine is configured to deploy" means. Read-only; an unreadable file is
// skipped.
diff --git a/common/publish/stamps.ts b/common/publish/stamps.ts
@@ -139,8 +139,8 @@ export function deployedPath(paths: Pick<Paths, "exportBuildsDir">, target: stri
}
// The runtime image's build facts (Dockerfile build args → ENV): the stamps'
-// `commit` / `branch` where there is no .git to ask. Declared, with the two
-// names, in lib/envVars.ts; re-exported here for the stage bodies.
+// `commit` / `branch` where there is no checkout to ask. Declared once, beside
+// the two names, in lib/envVars.ts (release 18 S5); re-exported for the stages.
export { imageBuildFacts } from "../lib/envVars";
/** A fresh, sortable, unique stamp id. */
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -9,7 +9,7 @@
- **`build site`, `build all` and `deploy site` are aliases of the publish commands** and print what they run: `build site <id>` is `publish index` (skipped with `--nodata`) then `publish build <id> --force`; `build all` is `publish index` then `publish build all --runner auto` (containers when an engine answers, else one site at a time on the host); `deploy site <id>` is `publish deploy <id>`, which now ships the site's own bundle and refuses a site never built that way. `publish build all --runner docker` builds every stale site in containers on a Linux host and refuses with "the docker runner needs an engine on this host" where there is none.
- **Substitute your own yt-dlp in Docker.** Point `YTDLP_BIN` at a zipapp you built, or set `YTDLP_SOURCE_HOST_DIR` to a yt-dlp checkout and start with `docker-compose.ytdlp.yml`: the image runs it with its own python, and nothing is rebuilt. Every editor boot logs `yt-dlp: <path> <version> (image|override)` (`MISSING` when it does not run; the editor still starts), and `YTDLP_AUTO_UPDATE` updates the image's yt-dlp only, warning instead of touching yours.
- **The Docker image can publish.** It carries python, `pipx` and a pinned `git-filter-repo`, so the homepage's `/source` mirror builds in the container; `docker-compose.source.yml` mounts your repository read-only for it, and the scrub rules and denylist live in the config volume (`/data/config/archilyzer`). Cloudflare and R2 credentials come from `.env`. Run publish commands with `docker compose exec editor pnpm archilyzer …`, not `run --rm`. The `homepage` service serves a local deploy from the builds volume once there is one. RUNNING_IN_DOCKER.md has a Windows checklist.
-- **`archilyzer doctor` checks what a publish needs.** Which yt-dlp runs (the image's, the host's or an override, and whether it runs), whether the Cloudflare token and the R2 keys are set (never their values; R2 only when a bucket is configured), free space for the site bundles, the repository the source mirror reads, and the private config dir.
+- **`archilyzer doctor` checks what a publish needs.** Which yt-dlp runs (the image's, the host's or an override, and whether it runs), whether the Cloudflare token and the R2 keys are set (never their values; R2 only when a bucket is configured), free space for the site bundles, the publish lock (free, held by a running stage, or left by one that is gone — with the command to clear it; never cleared for you), the index stamp's age and which sites were built from an older one, the repository the source mirror reads, the private config dir, and whether this Node is new enough for the pinned wrangler (deploys need 22).
- **A cited moment at the very end of a recording prepares.** Prepare evidence media cuts a clip whose padding runs past the recording's end at the end (the recording's duration from its metadata), where it found no media for the padded span; a span that starts past the end is still refused. report-to-video keeps its strict rule.
- **Exporting a changed report records a new revision of it.** `reports export` (and **Export reports** on a site's Reports tab, and the end of a prepare) commits a revision to the report's own git history, `sites/<site>/reports/<id>/history-git/`, whenever its `report.json` changed since the last one: the `report.json`, its Markdown export and the checksums of every export file, with a message of `Revision N` and a summary of the change. A re-export of an unchanged report records nothing. The commits carry the site's name and a `noreply@<site>.invalid` address with dates in UTC, never your git name, email or time zone. The Reports tab shows each report's revision, its commit and the last change under **Exports**, and the site's next build publishes the history. Add `history-git/` to the corpus repository's `.gitignore`.
- **archive.org files come over BitTorrent when possible, else straight from archive.org — never through yt-dlp.** The chosen file of an archive.org import is fetched from the item's own torrent (`<identifier>_archive.torrent`, which lists archive.org as a web seed, so other peers take load off archive.org) with aria2c, only that file of the item, and seeded afterwards for 10 minutes or to a ratio of 1, whichever comes first; the log shows "torrent: <file> (n of m pieces, peers p, web seed yes)" and "seeding 10 min…". With no aria2c, a torrent that does not carry the file, or no progress for 5 minutes, it is downloaded directly from `archive.org/download/…` instead (resumable, backing off on 429/503), and the log says "fell back to direct download: <reason>". Every file is checked against archive.org's sha1/md5: a mismatch is downloaded once more directly, a second one fails the record. The record is written from the item's metadata: `metadata.info.json` with the file's page, the canonical id, the duration ffprobe measures and archive.org's playable copies of the file, the `archiveorg.json` provenance (a mirror's original title, date and uploader), and `audio.<fmt>` — an audio file already in the channel's format is used as is, anything else goes through the app's audio extraction, a video kept in the saved-video store when the channel keeps sources. An .avi/.mpeg/.flac/.wav original is fetched as archive.org's mp4 or mp3 of it. aria2c runs in its own process group: cancelling the job stops it and everything it started, and it stops itself if the editor exits. New settings block `archiveOrg` (`torrent`, `seedMinutes`, `seedRatio`, `stallMinutes`, `maxPeers`, `maxDownloadKiBps`, `maxUploadKiBps`), `ARIA2C_BIN`, an aria2c row in `archilyzer doctor`, and `aria2` in the runtime Docker images.
diff --git a/plans/release-18.md b/plans/release-18.md
@@ -964,7 +964,7 @@ merged, so the doctor's `wrangler`, `publish-lock` and `index-stamp` checks, the
| `fc793037` | RUNNING_IN_DOCKER.md names Node 22 in the image's contents; this table |
| `55d779a1` | **The publish lock's host identity** (S1's review: `os.hostname()` in a container is its id, new on every recreate, so a crashed holder's lock would look foreign forever; S1's `stageLock.ts` reads `ARCHILYZER_HOST_ID ?? os.hostname()`): `ARCHILYZER_HOST_ID: archilyzer-editor` on the **editor service's** `environment`, not `x-app-env` — site, homepage and umtool share the builds volume, and a container carrying the same id with its own pid namespace would judge the editor's live lock dead and take it (visible in `docker compose config` either way; checked: only the editor has it). envVars row, no TODO needed: the compose file names it, which the test accepts (`readBy` names `stageLock.ts`, S1's). RUNNING_IN_DOCKER.md: why the id is fixed, that `run --rm` would now carry it with other pids (one more reason for `exec`), and how to clear a foreign-host lock (`rm /data/builds/.export-builds/.publish.lock`, only when nothing is publishing) |
| `c238970a` | SETUP.md: Node 22 — Next needs ≥ 20.9, deploying runs the pinned wrangler (≥ 22) |
-| this one | this table |
+| `65d549e7` | this table |
Re-run after the fixes at `07bc2395`: tsc (all workspaces) clean; `doctor`, `buildImage`, `source` and
`envVars` tests **61/61** (`$T/s5-fix-tests.log`).
@@ -977,6 +977,44 @@ load in the runtime — `lmdb` opens, writes and reads (`process.versions.module
binding loads; `archilyzer doctor` reports `node v22.23.3` ok and every S5 check, exit 1 only for the
model the smoke skips. vulkan and cuda still unbuilt (above).
+**Second half** (S1 merged: `r18/integration` `0ce00f76`, a fast-forward of this branch; S2 not yet)
+
+| Commit | What |
+|---|---|
+| `0df61c8c` | `doctor:` **`publish/publish-lock`** over S1's `stageLock.ts` — free → ok; a live holder on this host → a note; the lock's own `holderIsGone` (dead pid, a different start time, a pid younger than the lock) → stale, with `rm <exportBuildsDir>/.publish.lock`; a lock that has not parsed past `LOCK_TORN_GRACE_MS` → torn, the same; another host's (`lockHostId(env)` differs) → named, never judged. Never cleared by the doctor. **`publish/index-stamp`** over S1's `stamps.ts` — id, age, generation; the built targets (sites, `_hub`, `_homepage`) whose `indexStampId` is older, as a warning with `publish build <id>`; no stamp → "update the index first: archilyzer publish index" (a warning once anything is built or configured). **`export-builds`** sums each bundle's `built.json` `bytes` (a bundle no stamp describes is still walked). **`workspace/node`** grades against wrangler's `engines.node` when `common/node_modules/wrangler/package.json` is there (below it: a warning — every deploy refuses), read the way the image's drift test reads it. `stamps.ts`'s local `imageBuildFacts` became a re-export of `lib/envVars`'s (one definition; S1's `stamps.test.ts` 6/6 unchanged). 3 new doctor tests (lock: 5 states; stamp + bytes; node vs the floor) |
+| this one | `plans:` this table and the smoke; the doctor's changelog bullet names the new checks |
+
+Gates at `0df61c8c`: tsc (all workspaces) clean; `doctor`, `buildImage`, `source`, `envVars`, `stamps`,
+`stageLock` tests **82 passed, 1 skipped** (the wrangler floor — not installed here).
+
+**Compose smoke, second half** (`--target runtime` rebuilt from `0df61c8c`, 363 s, 1.79 GB; `-p r18smoke`;
+the e2e `curated-tags-channel` fixture — 3 videos with VTTs — and one site `s5site` copied into
+`$T/s5-corpus2`; `docker-compose.source.yml` over a throwaway repo in `$T`; editor + `site`; `down -v` after,
+nothing left):
+- `exec editor pnpm archilyzer publish index` → exit 0, 10 s (index +3, stats built, signatures, the stamp).
+- `publish build s5site` → exit 0, 76 s: "bundle installed at /data/builds/.export-builds/s5site/out (copied
+ across filesystems)", 198 files, 5.3 MB. Again → "fresh — nothing to do", exit 0.
+- `publish deploy s5site --to local` → exit 0, 4 s, copied into `/data/builds/site`; the `site` service
+ serves it: `corpus.json` names `s5site` (1 channel, 3 videos), `/` 200.
+- `publish deploy s5site --preview smoke` with no token: **skipped — the credential preflight lands with
+ S2.** S1's deploy body still calls `build.ts`'s `deploySite` (unpinned `pnpm dlx wrangler`), which has no
+ preflight to refuse before wrangler; the bogus-token run is the third round's.
+- `source publish --check` over the throwaway repo (read-only, host-owned, `safe.directory`) with throwaway
+ scrub/denylist files put in the config volume by `docker compose cp` (mode 600): exit 0, 6 s — "check
+ passed — would publish main 60a126e08086 as 60a126e08086: 12 files … nothing written"; gitleaks skipped
+ with its WARNING and no history pages, as RUNNING_IN_DOCKER.md says. This is the review's open medium:
+ the container's mirror over the `:ro`, foreign-owned mount works.
+- `doctor` (exit 1 only for the model the smoke skips): `node v22.23.3` ok, `downloader/yt-dlp` ok (image),
+ `cloudflare-auth` note, `export-builds` "1 bundle, 5 MB" (from built.json), `publish-lock` "free",
+ `index-stamp` "…, generation 1; 1 bundle built from it", `filter-repo` ok, `source-repo` ok (main
+ 60a126e08086), `config-dir` "2 entries", `scrub rules` / `denylist` ok (1 each, mode 600 — counted).
+- `publish status` and `publish now` (RUNNING_IN_DOCKER.md names both) are S3's rows, not on this branch
+ yet: `archilyzer: unknown command "publish status"` here.
+
+**Third round** (after S2): the `wrangler` check (`wranglerBin` / `WRANGLER_MAJOR`), `cloudflare-auth`
+through `cloudflareCredentialProblem` + `wranglerOAuthConfigFiles`, the envVars `readBy` touch-ups and
+dedupe, the no-token and bogus-token deploy smoke.
+
## Rollout
(Steps 1–7 above; "### As it went" is written as the rollout runs.)
