commit 1957d1eac6190c482aa0114162f1ae98832b0a06
parent b2c696cba2f27ea1f874e195c08227ae00b5bc9c
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Wed, 30 Sep 2026 10:07:43 -0400
common: SG review fixes — every history log line is masked (L1); a render cache that cannot be made or written (EACCES, EROFS, ENOSPC on the dir, the lock or the key) is one line and a render without it, never a failed build (L2); the retry after stagit's -c misses a merge of older commits says so (L3); the post-pass drops the href of a link to a file main no longer has, or to a commit with no page (the parent past the cap), keeping the text and the #h target (L4); `source publish`'s usage names the history (L5); a test for the file-count drop, through a historyFileLimit seam (L6); a lock is stale when its pid is not running or it is over an hour old, and is replaced with one line (L8)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
5 files changed, 361 insertions(+), 49 deletions(-)
diff --git a/common/bin/archilyzer.ts b/common/bin/archilyzer.ts
@@ -153,7 +153,7 @@ export const COMMANDS: Command[] = [
{
path: ["source", "publish"],
usage:
- "[--force] [--check] [--keep-scratch] the scrubbed git mirror, raw tree and tarball into homepage/public, behind the denied-literal gate (--check: audit and count, write nothing)",
+ "[--force] [--check] [--keep-scratch] the scrubbed git mirror, raw tree, history pages (stagit, when installed) and tarball into homepage/public, behind the denied-literal gate (--check: audit and count, write nothing)",
flags: { force: "boolean", check: "boolean", "keep-scratch": "boolean" },
run: async ({ flags }) => {
const { publishSource } = await import("../publish/source");
diff --git a/common/publish/source.test.ts b/common/publish/source.test.ts
@@ -867,3 +867,50 @@ test("history: the render cache is the XDG cache's archilyzer/source-history —
const want = process.env.XDG_CACHE_HOME || path.join(os.homedir(), ".cache");
assert.equal(real, path.join(want, "archilyzer", "source-history"));
});
+
+test("history: an unusable render cache is one masked line and a render without it — the publish and its history go on (review L1, L2)", async () => {
+ const repo = sourceRepo();
+ // The cache's path carries a denied literal: the line naming it is masked,
+ // as every line of the step is.
+ const planted = "plantedcachedir";
+ const logs: string[] = [];
+ const log = path.join(dir("stagit-log"), "calls");
+ const root = path.join(dir("ro-cache"), planted);
+ mkdirSync(root, { recursive: true });
+ chmodSync(root, 0o555);
+ try {
+ const o = opts(repo, operatorFiles("", `${planted}\n`), logs, {
+ filterRepo: ["true"],
+ stagit: fakeStagit(log),
+ tokensFile: TOKENS_FILE,
+ historyCacheDir: path.join(root, "archilyzer", "source-history"),
+ });
+ assert.equal(await publishSource(o), 0, logs.join("\n"));
+ const text = logs.join("\n");
+ assert.match(text, /\[source\] history: the render cache .*\[REDACTED\].* is unusable \(EACCES\); rendering without it/);
+ assert.ok(!text.includes(planted), "never the literal");
+ assert.equal(stagitCalls(log).at(-1), `cache= limit= base=https://archilyzer.pages.dev/source/git/ repo=${MIRROR_DIR}`);
+ assert.equal(JSON.parse(readFileSync(path.join(o.publicDir!, "source", "manifest.json"), "utf8")).history.commits, 3);
+ } finally {
+ chmodSync(root, 0o755);
+ }
+});
+
+test("history: pages that would take the publish over the step's file limit are dropped with a WARNING, never the publish (review L6)", async () => {
+ const repo = sourceRepo();
+ const logs: string[] = [];
+ const log = path.join(dir("stagit-log"), "calls");
+ const o = opts(repo, operatorFiles("", ""), logs, {
+ filterRepo: ["true"],
+ stagit: fakeStagit(log),
+ tokensFile: TOKENS_FILE,
+ historyCacheDir: null,
+ // Any limit the rest already fills: the history is what is dropped.
+ historyFileLimit: 1,
+ });
+ assert.equal(await publishSource(o), 0, logs.join("\n"));
+ assert.match(logs.join("\n"), /\[source\] WARNING: 9 history files would make \d+ files to publish, over the step's limit of 1 \(Pages allows 20,000 per deployment\) — publishing without the history pages \(\/source\/git\/\)/);
+ assert.ok(!existsSync(path.join(o.publicDir!, "source", "git")));
+ assert.ok(existsSync(path.join(o.publicDir!, "source", MIRROR_DIR, "info", "refs")), "the rest is published");
+ assert.ok(!("history" in JSON.parse(readFileSync(path.join(o.publicDir!, "source", "manifest.json"), "utf8"))));
+});
diff --git a/common/publish/source.ts b/common/publish/source.ts
@@ -151,6 +151,9 @@ export type SourcePublishOpts = PublishOpts & {
historyCacheDir?: string | null;
// The history's cap. Default: SOURCE_HISTORY_MAX_COMMITS (the tests' seam).
maxHistoryCommits?: number;
+ // The file limit the history's last-resort drop checks. Default: MAX_FILES
+ // (the tests' seam; step 14 always applies MAX_FILES).
+ historyFileLimit?: number;
// The design tokens the history pages' style.css is written from. Default:
// <repo>/common/styles/tokens.css.
tokensFile?: string;
@@ -820,7 +823,7 @@ async function publish(
]);
if (badRoot) throw new SourceRefusal(badRoot);
// From here a refusal removes the history's render cache too.
- const historyCache = await historyCacheFor(opts, paths, publicDir, onLog);
+ const historyCache = await historyCacheFor(opts, paths, publicDir, (l) => onLog(maskLiterals(l, ctx.literals)));
progress.historyCache = historyCache;
await mkdir(scratchRoot, { recursive: true });
const scratch = await mkdtemp(path.join(scratchRoot, "archilyzer-source-"));
@@ -986,6 +989,8 @@ async function publish(
filterRepoId,
stage,
dest: stageHistoryDir,
+ // The raw tree's files, as tracked: links to any other become text.
+ treeFiles: new Set((await walkFiles(stageTree)).map((f) => f.rel)),
// Everything staged so far, and the manifest still to come.
stagedSoFar: (await walkFiles(stage)).length + 1,
});
@@ -1121,6 +1126,7 @@ async function stageHistoryPages(a: {
filterRepoId: string;
stage: string;
dest: string;
+ treeFiles: ReadonlySet<string>;
stagedSoFar: number;
}): Promise<SourceHistory | null> {
const { ctx } = a;
@@ -1177,16 +1183,20 @@ async function stageHistoryPages(a: {
baseUrl,
stylesheet,
themeScript: homepageThemeScript(),
+ treeFiles: a.treeFiles,
run: child,
- onLog: ctx.onLog,
+ // stagit's words (a failure's tail, a timeout's argv) can name the home
+ // dir: every line is masked, as every other child line of the step is.
+ onLog: (l) => ctx.onLog(maskLiterals(l, ctx.literals)),
});
} catch (err) {
if (err instanceof HistoryProblem) return without(`the history pages were not rendered: ${err.message}`);
throw err;
}
- if (a.stagedSoFar + r.files > MAX_FILES) {
+ const fileLimit = a.opts.historyFileLimit ?? MAX_FILES;
+ if (a.stagedSoFar + r.files > fileLimit) {
return without(
- `${r.files} history files would make ${a.stagedSoFar + r.files} files to publish, over the step's limit of ${MAX_FILES} (Pages allows 20,000 per deployment)`,
+ `${r.files} history files would make ${a.stagedSoFar + r.files} files to publish, over the step's limit of ${fileLimit} (Pages allows 20,000 per deployment)`,
);
}
if (r.largest.bytes > MAX_FILE_BYTES) {
diff --git a/common/publish/sourceHistory.test.ts b/common/publish/sourceHistory.test.ts
@@ -3,6 +3,7 @@ import assert from "node:assert/strict";
import { execFileSync, spawnSync } from "node:child_process";
import {
chmodSync,
+ utimesSync,
existsSync,
mkdirSync,
mkdtempSync,
@@ -18,6 +19,7 @@ import { buildThemeScript, HOMEPAGE_DEFAULT_BASE } from "../lib/themeConfig";
import { writeFakeStagit } from "./__fixtures__/fakeStagit";
import {
HISTORY_BACK_LINK,
+ HISTORY_LOCK_STALE_MS,
HISTORY_TOKENS,
HistoryProblem,
SITE_ICON_HREF,
@@ -115,6 +117,24 @@ test("the post-pass at the top level: files.html's links go to ../tree/<path>, a
assert.ok(!out.includes('href="file/'));
});
+test("the post-pass with what is published: a link to a file main no longer has, or to a commit with no page, becomes text (its id kept)", () => {
+ const sha = "a".repeat(40);
+ const gone = "b".repeat(40);
+ const page = `<html>\n<head>\n</head>\n<body>\n<a href="../commit/${sha}.html">${sha}</a> <b>parent</b> <a href="../commit/${gone}.html">${gone}</a>\n<b>diff --git a/<a id="h0" href="../file/old/name.txt.html">old/name.txt</a> b/<a href="../file/app/%5Bslug%5D/page.tsx.html">app/[slug]/page.tsx</a></b>\n<a href="../file/bad%zz.html">bad</a>\n</body>\n</html>\n`;
+ const published = {
+ tree: (p: string) => p === "app/[slug]/page.tsx",
+ commit: (c: string) => c === sha,
+ };
+ const out = rewriteHistoryPage(page, "t()", published);
+ assert.ok(out.includes(`<a href="../commit/${sha}.html">${sha}</a>`), "a published commit keeps its link");
+ assert.ok(out.includes(`<b>parent</b> <a>${gone}</a>`), "an unpublished commit is text");
+ assert.ok(out.includes(`a/<a id="h0">old/name.txt</a>`), "a file main no longer has is text, and keeps the diffstat's target");
+ assert.ok(out.includes(`b/<a href="../../tree/app/%5Bslug%5D/page.tsx">`), "a file in the tree keeps its (encoded) link");
+ assert.ok(out.includes(`<a>bad</a>`), "an encoding that does not decode is not linked");
+ // Without the set, nothing is dropped (the unit tests above).
+ assert.ok(rewriteHistoryPage(page, "t()").includes(`href="../../tree/old/name.txt"`));
+});
+
test("the theme script is the homepage's own: its base, its string", () => {
assert.equal(HOMEPAGE_DEFAULT_BASE, "dark");
assert.equal(homepageThemeScript(), buildThemeScript({ defaultBase: HOMEPAGE_DEFAULT_BASE }));
@@ -427,7 +447,7 @@ test("the cache: -c in the cache dir, incremental the next time, the key and the
o = renderOpts(gitDir, stagit, { cacheDir, cacheKey: "k2" });
r = await renderHistory(o);
assert.equal(r.cached, false);
- assert.match(o.logs.join("\n"), new RegExp(`the log lists ${oldest.slice(0, 12)}, whose page is not there from the cache; rendering every page again`));
+ assert.match(o.logs.join("\n"), new RegExp(`the cached render does not cover this head \\(the log lists ${oldest.slice(0, 12)}, whose page is not there\\) — stagit's -c stops at the last head it rendered, and a merge of older commits falls behind it; rendering every page again`));
assert.ok(existsSync(path.join(o.dest, "commit", `${oldest}.html`)));
// `fresh` (--force) renders every page, whatever the cache holds.
@@ -460,13 +480,106 @@ test("the cache: -c in the cache dir, incremental the next time, the key and the
assert.deepEqual(readdirSync(cacheDir), []);
});
+test("a stale lock: its pid not running, or over an hour old whoever runs its pid now, is replaced with one line", async () => {
+ const cacheDir = path.join(dir("stale"), "archilyzer", "source-history");
+ mkdirSync(path.join(cacheDir, "out"), { recursive: true });
+ const lock = path.join(cacheDir, "lock");
+ const logs: string[] = [];
+ const onLog = (l: string) => {
+ logs.push(l);
+ };
+ // Live and fresh: busy.
+ writeFileSync(lock, `${process.pid}\n`);
+ assert.equal(await holdHistoryCache(cacheDir, onLog), false);
+ assert.deepEqual(logs, []);
+ // Live, but two hours old: stale.
+ const old = new Date(Date.now() - 2 * HISTORY_LOCK_STALE_MS);
+ utimesSync(lock, old, old);
+ assert.equal(await holdHistoryCache(cacheDir, onLog), true);
+ assert.match(logs.join("\n"), new RegExp(`a stale lock on the render cache \\(pid ${process.pid}, 120 minutes old\\) was replaced; the cache is rendered afresh`));
+ assert.ok(!existsSync(path.join(cacheDir, "out")), "emptied");
+ assert.equal(readFileSync(lock, "utf8"), `${process.pid}\n`, "and held");
+ rmSync(lock);
+ // Not running: stale, whatever its age.
+ const dead = spawnSync("sh", ["-c", "echo $$"], { encoding: "utf8" }).stdout.trim();
+ writeFileSync(lock, `${dead}\n`);
+ logs.length = 0;
+ assert.equal(await holdHistoryCache(cacheDir, onLog), true);
+ assert.match(logs.join("\n"), new RegExp(`\\(pid ${dead}, not running\\) was replaced`));
+});
+
+test("a render cache that cannot be written (a read-only dir, or one that cannot be made) is one line and a render without it", async () => {
+ const { gitDir } = bareRepo(2);
+ const log = path.join(dir("log"), "calls");
+ const stagit = fakeStagit(log);
+ const root = dir("ro");
+ const readOnly = path.join(root, "cache");
+ mkdirSync(readOnly);
+ chmodSync(readOnly, 0o555);
+ const parent = path.join(root, "parent");
+ mkdirSync(parent);
+ chmodSync(parent, 0o555);
+ try {
+ for (const cacheDir of [readOnly, path.join(parent, "archilyzer", "source-history")]) {
+ const o = renderOpts(gitDir, stagit, { cacheDir });
+ const r = await renderHistory(o);
+ assert.equal(r.cached, false);
+ assert.equal(r.shown, 2);
+ assert.equal(calls(log).at(-1), `cache= limit= base=${o.baseUrl} repo=archilyzer.git`, "rendered without -c");
+ assert.match(o.logs.join("\n"), /the render cache .* is unusable \(EACCES\); rendering without it/);
+ assert.equal(readdirSync(path.join(o.dest, "commit")).length, 2);
+ }
+ } finally {
+ chmodSync(readOnly, 0o755);
+ chmodSync(parent, 0o755);
+ }
+});
+
+test("a --no-ff merge of commits older than the cached head: stagit's -c misses them, the count notices, and every page is rendered again", async () => {
+ const { gitDir, work } = bareRepo(2);
+ const log = path.join(dir("log"), "calls");
+ const stagit = fakeStagit(log);
+ const cacheDir = path.join(dir("merge-cache"), "archilyzer", "source-history");
+ // A branch whose commits are dated before the head the cache will name.
+ const base = gitIn(work, "rev-parse", "HEAD~1");
+ gitIn(work, "checkout", "-q", "-b", "side", base);
+ for (const i of [7, 8]) {
+ writeFileSync(path.join(work, `side-${i}.txt`), `${i}\n`);
+ gitIn(work, "add", "-A");
+ execFileSync("git", ["commit", "-q", "-m", `side ${i}`], {
+ cwd: work,
+ stdio: "pipe",
+ env: { ...process.env, GIT_COMMITTER_DATE: `2001-01-0${i - 6}T00:00:00Z`, GIT_AUTHOR_DATE: `2001-01-0${i - 6}T00:00:00Z` },
+ });
+ }
+ gitIn(work, "checkout", "-q", "main");
+ let o = renderOpts(gitDir, stagit, { cacheDir });
+ await renderHistory(o);
+ gitIn(work, "merge", "-q", "--no-ff", "-m", "merge side", "side");
+ gitIn(TMP, "--git-dir", gitDir, "fetch", "-q", work, "main:main");
+ o = renderOpts(gitDir, stagit, { cacheDir });
+ const r = await renderHistory(o);
+ assert.equal(r.shown, 5);
+ assert.equal(r.cached, false);
+ assert.match(o.logs.join("\n"), /the cached render does not cover this head \(stagit's log lists 3 commits, not 5 .*\) — stagit's -c stops at the last head it rendered, and a merge of older commits falls behind it; rendering every page again/);
+ assert.equal(readdirSync(path.join(o.dest, "commit")).length, 5);
+});
+
// ── the real stagit ─────────────────────────────────────────────────────────
test("the real stagit: a page per commit, every link into file/ now into the tree, every relative link resolves", async (t) => {
const stagit = resolveStagit(process.env.STAGIT_BIN ?? "stagit", process.env);
if (!stagit) return t.skip("stagit is not installed (STAGIT_BIN, PATH, ~/.local/bin)");
- const { gitDir } = bareRepo(3);
- const o = renderOpts(gitDir, stagit, { cacheDir: path.join(dir("real-root"), "archilyzer-source-history") });
+ const { gitDir, work } = bareRepo(1);
+ // A file that main no longer has: its diffs are published, its links not.
+ writeFileSync(path.join(work, "gone.txt"), "soon gone\n");
+ gitIn(work, "add", "-A");
+ gitIn(work, "commit", "-q", "-m", "add gone.txt");
+ gitIn(work, "rm", "-q", "gone.txt");
+ gitIn(work, "commit", "-q", "-m", "remove gone.txt");
+ gitIn(TMP, "--git-dir", gitDir, "fetch", "-q", work, "main:main");
+ const treeFiles = new Set(["README.md", "app/[slug]/page.tsx"]);
+ const o = renderOpts(gitDir, stagit, { cacheDir: path.join(dir("real-root"), "archilyzer", "source-history"), treeFiles });
const r = await renderHistory(o);
assert.equal(r.rendered, 3);
assert.equal(readdirSync(path.join(o.dest, "commit")).length, 3);
@@ -484,8 +597,8 @@ test("the real stagit: a page per commit, every link into file/ now into the tre
.normalize(path.posix.join(path.posix.dirname(`source/git/${page}`), target))
.replace(/\/$/, "");
if (resolved.startsWith("source/tree/")) {
- // Into the raw tree: a file of main.
- assert.ok(["source/tree/README.md", "source/tree/app/%5Bslug%5D/page.tsx"].includes(resolved), `${page}: ${target}`);
+ // Into the raw tree: only a file main has (`gone.txt` is text).
+ assert.ok(treeFiles.has(decodeURIComponent(resolved.slice("source/tree/".length))), `${page}: ${target}`);
} else if (resolved.startsWith("source/git/")) {
assert.ok(existsSync(path.join(path.dirname(path.dirname(o.dest)), resolved)), `${page}: ${target} → ${resolved}`);
} else {
@@ -493,6 +606,8 @@ test("the real stagit: a page per commit, every link into file/ now into the tre
}
}
}
+ const pages = readdirSync(path.join(o.dest, "commit")).map((f) => readFileSync(path.join(o.dest, "commit", f), "utf8"));
+ assert.ok(pages.some((h) => /<a id="h0">gone\.txt<\/a>/.test(h)), "gone.txt's diff header is text, its #h0 target kept");
const logHtml = readFileSync(path.join(o.dest, "log.html"), "utf8");
assert.match(logHtml, /<title>Log - archilyzer - Archilyzer<\/title>/);
assert.match(logHtml, /git clone <a href="https:\/\/archilyzer\.pages\.dev\/source\/archilyzer\.git">/);
@@ -501,7 +616,7 @@ test("the real stagit: a page per commit, every link into file/ now into the tre
// The cap, by stagit itself: -l keeps the NEWEST in the log (and says how
// many more), writes a page for every commit, and only the listed are
// published.
- const capped = renderOpts(gitDir, stagit, { maxCommits: 2 });
+ const capped = renderOpts(gitDir, stagit, { maxCommits: 2, treeFiles });
const rc = await renderHistory(capped);
const newest = gitIn(TMP, "--git-dir", gitDir, "rev-list", "--max-count=2", "main").split("\n");
assert.equal(rc.shown, 2);
@@ -509,4 +624,9 @@ test("the real stagit: a page per commit, every link into file/ now into the tre
assert.deepEqual(readdirSync(path.join(capped.dest, "commit")).sort(), newest.map((c) => `${c}.html`).sort());
assert.equal(readdirSync(path.join(capped.scratch, "history", "commit")).length, 3);
assert.match(readFileSync(path.join(capped.dest, "log.html"), "utf8"), /1 more commits remaining, fetch the repository/);
+ // The oldest published page's parent has no page: its link is text.
+ const oldestShown = readFileSync(path.join(capped.dest, "commit", `${newest[1]}.html`), "utf8");
+ const parent = gitIn(TMP, "--git-dir", gitDir, "rev-parse", `${newest[1]}^`);
+ assert.ok(oldestShown.includes(`<b>parent</b> <a>${parent}</a>`), "the parent past the cap is text");
+ assert.ok(!oldestShown.includes(`commit/${parent}.html`));
});
diff --git a/common/publish/sourceHistory.ts b/common/publish/sourceHistory.ts
@@ -44,8 +44,15 @@
// header text) and the last run finished (the key is removed before a render
// and written after it); its log lines only when the commit they end at is an
// ancestor of today's head. One publish holds it at a time (`lock`, the
-// holder's pid); a second renders without it. `--force` renders it afresh,
-// `--check` never touches it, and a refusal removes it.
+// holder's pid); a second renders without it, and a lock whose pid is not
+// running or that is over an hour old is stale and replaced. A cache that
+// cannot be written (EACCES, EROFS, ENOSPC) is one line and a render without
+// it. `--force` renders it afresh, `--check` never touches it, and a refusal
+// removes it.
+//
+// LINKS TO WHAT IS NOT PUBLISHED become text: a diff's file that main no
+// longer has, a commit with no page (past the cap). The raw tree's file list
+// decides (`treeFiles`).
import { createHash } from "node:crypto";
import { accessSync, constants, existsSync, statSync } from "node:fs";
@@ -54,7 +61,7 @@ import os from "node:os";
import path from "node:path";
import { PROJECT_NAME } from "../lib/project";
import { buildThemeScript, HOMEPAGE_DEFAULT_BASE } from "../lib/themeConfig";
-import { onPath } from "./sourceAudit";
+import { onPath, tildify } from "./sourceAudit";
/** How to install stagit, as the log line and the doctor say it. */
export const STAGIT_INSTALL =
@@ -237,6 +244,7 @@ body {
}
a { color: var(--brand); }
a:hover { color: var(--foreground); }
+a:not([href]) { color: inherit; text-decoration: none; }
p.archilyzer-source {
margin: 0 0 1rem;
font-family: system-ui, -apple-system, "Segoe UI", sans-serif;
@@ -285,15 +293,38 @@ pre a.d:hover { text-decoration: none; }
// ── the post-pass ───────────────────────────────────────────────────────────
/**
+ * What is published beside the pages, for the post-pass to link to only what
+ * is there: a path of the raw tree (decoded, as tracked), a commit with a page.
+ */
+export type PublishedSet = {
+ tree: (path: string) => boolean;
+ commit: (sha: string) => boolean;
+};
+
+// stagit's percent-encoding undone; null when it is not valid.
+function decodedPath(p: string): string | null {
+ try {
+ return decodeURIComponent(p);
+ } catch {
+ return null;
+ }
+}
+
+/**
* One stagit page, as published. Adds the theme script before `</head>` and
* HISTORY_BACK_LINK after `<body>`; points every `href="…file/<path>.html"`
* (the Files index, the header's README and LICENSE, a diff's file names) at
* the raw tree — `../tree/<path>` from the same depth, the path as stagit
* encoded it — and stagit's logo.png and favicon.png at the site's icon.
- * Nothing else changes. Page text cannot fake an `href="…"`: stagit encodes
- * every `"` it prints from the repository as `"`.
+ *
+ * With `published`, a link to what is not published loses its `href` and
+ * stays as text (an `<a>` with no `href`, its `id` kept — a diff header is the
+ * diffstat's `#h<n>` target): a file no longer in main (a diff of a deleted or
+ * renamed file), and a commit with no page (past the cap, the oldest page's
+ * parent). Nothing else changes. Page text cannot fake an `href="…"`: stagit
+ * encodes every `"` it prints from the repository as `"`.
*/
-export function rewriteHistoryPage(html: string, themeScript: string): string {
+export function rewriteHistoryPage(html: string, themeScript: string, published?: PublishedSet): string {
if (/<\/script/i.test(themeScript)) throw new Error("the theme script may not close its own element");
let out = html;
const head = out.indexOf("</head>");
@@ -303,10 +334,22 @@ export function rewriteHistoryPage(html: string, themeScript: string): string {
const at = body.index + body[0].length;
out = `${out.slice(0, at)}${HISTORY_BACK_LINK}\n${out.slice(at)}`;
}
- return out
- .replace(/href="((?:\.\.\/)*)file\/([^"]*)\.html"/g, (_m, up: string, p: string) => `href="${up}../tree/${p}"`)
+ out = out
+ .replace(/ href="((?:\.\.\/)*)file\/([^"]*)\.html"/g, (_m, up: string, p: string) => {
+ if (published) {
+ const tracked = decodedPath(p);
+ if (tracked === null || !published.tree(tracked)) return "";
+ }
+ return ` href="${up}../tree/${p}"`;
+ })
.replace(/src="(?:\.\.\/)*logo\.png"/g, `src="${SITE_ICON_HREF}"`)
.replace(/href="(?:\.\.\/)*favicon\.png"/g, `href="${SITE_ICON_HREF}"`);
+ if (published) {
+ out = out.replace(/ href="((?:\.\.\/)*)commit\/([0-9a-f]{40})\.html"/g, (m, _up: string, sha: string) =>
+ published.commit(sha) ? m : "",
+ );
+ }
+ return out;
}
/**
@@ -333,7 +376,13 @@ export function loggedCommits(logHtml: string): string[] {
export async function stageHistory(
work: string,
dest: string,
- o: { themeScript: string; stylesheet: string; commits: readonly string[] },
+ o: {
+ themeScript: string;
+ stylesheet: string;
+ commits: readonly string[];
+ // The raw tree's files (as tracked); absent, no tree link is dropped.
+ treeFiles?: ReadonlySet<string>;
+ },
): Promise<{ files: number; bytes: number; largest: { rel: string; bytes: number } }> {
if (/[^\x00-\x7f]/.test(o.themeScript)) throw new Error("the theme script must be ASCII");
const rels: string[] = [];
@@ -347,6 +396,11 @@ export async function stageHistory(
if (!existsSync(path.join(work, rel))) throw new HistoryProblem(`the log lists ${sha.slice(0, 12)}, whose page is not there`);
rels.push(rel);
}
+ const pages = new Set(o.commits);
+ const treeFiles = o.treeFiles;
+ const published: PublishedSet | undefined = treeFiles
+ ? { tree: (p) => treeFiles.has(p), commit: (sha) => pages.has(sha) }
+ : undefined;
await mkdir(path.join(dest, "commit"), { recursive: true });
let bytes = 0;
let largest = { rel: "", bytes: -1 };
@@ -358,7 +412,7 @@ export async function stageHistory(
const src = path.join(work, rel);
const dst = path.join(dest, rel);
if (rel.endsWith(".html")) {
- const text = rewriteHistoryPage((await readFile(src)).toString("latin1"), o.themeScript);
+ const text = rewriteHistoryPage((await readFile(src)).toString("latin1"), o.themeScript, published);
const buf = Buffer.from(text, "latin1");
await writeFile(dst, buf);
note(rel, buf.length);
@@ -409,6 +463,9 @@ export type RenderHistoryOpts = {
baseUrl: string;
stylesheet: string;
themeScript: string;
+ // The raw tree's files, as tracked: a page's link to a file not among them
+ // (deleted or renamed since) becomes text. Absent: every link is kept.
+ treeFiles?: ReadonlySet<string>;
// A child, run with the publish's environment and cancel signal.
run: HistoryRun;
onLog: (line: string) => void;
@@ -435,12 +492,24 @@ const pidAlive = (pid: number): boolean => {
};
/**
+ * A lock this old is stale whoever holds its pid now: a publish holds the
+ * cache for one render (stagit's timeout is 10 minutes), and a pid is reused.
+ */
+export const HISTORY_LOCK_STALE_MS = 60 * 60 * 1000;
+
+/**
* Hold the cache directory, or say it is busy. The directory is made on the
- * way (recursively). The lock names its holder's pid; a lock whose holder is
- * gone means a render was cut off, so nothing in the directory is trusted: it
- * is emptied and taken.
+ * way (recursively). The lock names its holder's pid. A lock is stale when
+ * that pid is not running OR the lock is older than HISTORY_LOCK_STALE_MS: a
+ * render was cut off, so nothing in the directory is trusted — it is emptied
+ * and taken, with one line. An I/O error (an unwritable or full cache dir)
+ * is thrown: renderHistory then renders without the cache.
*/
-export async function holdHistoryCache(dir: string): Promise<boolean> {
+export async function holdHistoryCache(
+ dir: string,
+ onLog: (line: string) => void = () => {},
+ now: () => number = Date.now,
+): Promise<boolean> {
await mkdir(dir, { recursive: true, mode: 0o700 });
const lock = path.join(dir, "lock");
for (let attempt = 0; attempt < 2; attempt++) {
@@ -450,7 +519,14 @@ export async function holdHistoryCache(dir: string): Promise<boolean> {
} catch (err) {
if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
const pid = Number((await readFile(lock, "utf8").catch(() => "")).trim());
- if (Number.isInteger(pid) && pid > 0 && pidAlive(pid)) return false;
+ const mtime = (await stat(lock).catch(() => null))?.mtimeMs ?? now();
+ const age = now() - mtime;
+ const running = Number.isInteger(pid) && pid > 0 && pidAlive(pid);
+ if (running && age < HISTORY_LOCK_STALE_MS) return false;
+ onLog(
+ `[source] history: a stale lock on the render cache (pid ${pid > 0 ? pid : "unknown"}, ` +
+ `${running ? `${Math.round(age / 60_000)} minutes old` : "not running"}) was replaced; the cache is rendered afresh`,
+ );
await emptyDir(dir);
}
}
@@ -501,42 +577,95 @@ export async function renderHistory(o: RenderHistoryOpts): Promise<RenderedHisto
await writeFile(path.join(o.gitDir, "description"), o.description);
await writeFile(path.join(o.gitDir, "url"), `${o.cloneUrl}\n`);
- const held = o.cacheDir ? await holdHistoryCache(o.cacheDir) : false;
- if (o.cacheDir && !held) o.onLog("[source] history: the render cache is in use by another publish; rendering without it");
+ const withoutCache = async (): Promise<RenderedHistory> => ({
+ ...(await renderOnce(o, path.join(o.scratch, "history"), null)),
+ cached: false,
+ });
+ const dir = o.cacheDir;
+ if (!dir) return withoutCache();
+ // A cache that cannot be used (unwritable, read-only, full) is one line and
+ // a render without it: never a failed build.
+ const unusable = (err: unknown) =>
+ o.onLog(`[source] history: the render cache ${tildify(dir)} is unusable (${ioCode(err)}); rendering without it`);
+ let held: boolean;
+ try {
+ held = await holdHistoryCache(dir, o.onLog);
+ } catch (err) {
+ if (!isIoError(err)) throw err;
+ unusable(err);
+ return withoutCache();
+ }
+ if (!held) {
+ o.onLog("[source] history: the render cache is in use by another publish; rendering without it");
+ return withoutCache();
+ }
try {
- if (!held || !o.cacheDir) {
- const work = path.join(o.scratch, "history");
- return { ...(await renderOnce(o, work, null)), cached: false };
- }
- const dir = o.cacheDir;
const keyFile = path.join(dir, "key.json");
const cacheFile = path.join(dir, "stagit.cache");
const work = path.join(dir, "out");
- let usable = !o.fresh && (await cacheUsable(o, keyFile, work));
- if (!usable) await emptyCache(dir);
- else await dropStaleLogCache(o, dir, cacheFile);
- // The key goes before the render and comes back after it: a render cut
- // off leaves none, and the next publish starts over.
- await rm(keyFile, { force: true });
+ let usable: boolean;
+ try {
+ usable = !o.fresh && (await cacheUsable(o, keyFile, work));
+ if (!usable) await emptyCache(dir);
+ else await dropStaleLogCache(o, dir, cacheFile);
+ // The key goes before the render and comes back after it: a render cut
+ // off leaves none, and the next publish starts over.
+ await rm(keyFile, { force: true });
+ } catch (err) {
+ if (!isIoError(err)) throw err;
+ unusable(err);
+ return await withoutCache();
+ }
try {
const r = await renderOnce(o, work, cacheFile);
- await writeFile(keyFile, JSON.stringify({ key: o.cacheKey }) + "\n");
+ await writeKey(o, keyFile);
return { ...r, cached: usable };
} catch (err) {
if (!(err instanceof HistoryProblem) || !usable) throw err;
- // A cache that looked sound and did not render: once more from nothing.
- o.onLog(`[source] history: ${err.message} from the cache; rendering every page again`);
- usable = false;
- await emptyCache(dir);
+ // Not a fault: stagit's -c walk is in commit-date order and stops at the
+ // head it rendered last, so the commits of a merge that are older than
+ // that head are left out, and the log comes up short. Once more, every
+ // page.
+ o.onLog(
+ `[source] history: the cached render does not cover this head (${err.message}) — stagit's -c stops at the last head it rendered, and a merge of older commits falls behind it; rendering every page again`,
+ );
+ try {
+ await emptyCache(dir);
+ } catch (ioErr) {
+ if (!isIoError(ioErr)) throw ioErr;
+ unusable(ioErr);
+ return await withoutCache();
+ }
const r = await renderOnce(o, work, cacheFile);
- await writeFile(keyFile, JSON.stringify({ key: o.cacheKey }) + "\n");
+ await writeKey(o, keyFile);
return { ...r, cached: false };
}
} catch (err) {
- if (held && o.cacheDir) await emptyCache(o.cacheDir);
+ await emptyCache(dir).catch(() => {});
throw err;
} finally {
- if (held && o.cacheDir) await releaseHistoryCache(o.cacheDir);
+ await releaseHistoryCache(dir).catch(() => {});
+ }
+}
+
+// An I/O error from the file system (it carries an errno code), as opposed to
+// the history's own problems, a cancel, or a bug.
+function isIoError(err: unknown): err is NodeJS.ErrnoException {
+ return err instanceof Error && !(err instanceof HistoryProblem) && typeof (err as NodeJS.ErrnoException).code === "string";
+}
+
+function ioCode(err: unknown): string {
+ return (err as NodeJS.ErrnoException)?.code ?? "an I/O error";
+}
+
+// The key, after a render that finished. A key that cannot be written is no
+// key: the next publish renders every page again.
+async function writeKey(o: RenderHistoryOpts, keyFile: string): Promise<void> {
+ try {
+ await writeFile(keyFile, JSON.stringify({ key: o.cacheKey }) + "\n");
+ } catch (err) {
+ if (!isIoError(err)) throw err;
+ o.onLog(`[source] history: the render cache's key could not be written (${ioCode(err)}); the next publish renders every page again`);
}
}
@@ -581,7 +710,12 @@ async function renderOnce(
work: string,
cacheFile: string | null,
): Promise<Omit<RenderedHistory, "cached">> {
- await mkdir(work, { recursive: true });
+ try {
+ await mkdir(work, { recursive: true });
+ } catch (err) {
+ if (!isIoError(err)) throw err;
+ throw new HistoryProblem(`the render directory cannot be made (${ioCode(err)})`);
+ }
const before = await countCommitPages(work);
const capped = o.commits > o.maxCommits;
const shown = Math.min(o.commits, o.maxCommits);
@@ -594,7 +728,7 @@ async function renderOnce(
const r = await o.run(o.stagit, args, { cwd: work, timeoutMs: 600_000 });
// The per-file pages are never published, and stagit writes them all again
// on every run: none is kept.
- await rm(path.join(work, "file"), { recursive: true, force: true });
+ await rm(path.join(work, "file"), { recursive: true, force: true }).catch(() => {});
if (r.code !== 0) {
const tail = r.out.filter((l) => l.trim()).slice(-2).join(" / ");
throw new HistoryProblem(`stagit exited ${r.code}${tail ? `: ${tail}` : ""}`);
@@ -610,6 +744,7 @@ async function renderOnce(
themeScript: o.themeScript,
stylesheet: o.stylesheet,
commits,
+ treeFiles: o.treeFiles,
});
const after = await countCommitPages(work);
return { ...staged, shown, rendered: cacheFile ? after - before : after };
