commit 6fcadfffea368687ee00a9ed64ed3eb6e01d8ad5
parent 6eafa357249f14f0a739c83fe1dc89b790fb4a50
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Wed, 30 Sep 2026 09:07:42 -0400
common: the source's history pages — `source publish` renders the scrubbed mirror with stagit into /source/git/ (log, a page per commit with its diff, refs, files, two Atom feeds) after the object audit and before the file audit; an allowlist of stagit's output, never its per-file pages (every file/ link now points into the raw tree); a post-pass adds the homepage's pre-paint theme script and one line back to /source/ to every page, never the feeds; style.css from tokens.css; the manifest's history block, stagit and the pages' digest in the skip key, the deploy check names history pages that are not the audited ones; a render cache under the scratch root (stagit -c), keyed, ancestry-checked, one holder at a time; without stagit, one line and no history, never a failed build; STAGIT_BIN; step version 4
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
9 files changed, 1618 insertions(+), 31 deletions(-)
diff --git a/ENVIRONMENT.md b/ENVIRONMENT.md
@@ -42,7 +42,8 @@ The one override surface for where things live and which binary runs. Every one
| `ARCHILYZER_CONFIG_DIR` | `~/.config/archilyzer` | The operator's private config dir, outside the repo: the two inputs of `archilyzer source publish` below. Never committed. | common/lib/paths.ts (getPaths) |
| `SOURCE_SCRUB_FILE` | `<ARCHILYZER_CONFIG_DIR>/source-scrub.txt` | git-filter-repo `lhs==>rhs` rules applied to file contents AND commit messages when the source mirror is generated (`<home dir>==>/home/user` is built in and runs first). Every rule's left side is also denied. See [PUBLISH.md](PUBLISH.md). | common/lib/paths.ts (getPaths) |
| `SOURCE_DENYLIST_FILE` | `<ARCHILYZER_CONFIG_DIR>/source-denylist.txt` | Literals the published source must never contain, one per line (`i:` = any case). One hit anywhere in the mirror, the tree or the tarball refuses the publish. | common/lib/paths.ts (getPaths) |
-| `ARCHILYZER_SOURCE_SCRATCH` | the OS temp dir | Where `source publish` makes its scratch clone and stage (removed afterwards unless `--keep-scratch`). | common/lib/paths.ts (getPaths) |
+| `ARCHILYZER_SOURCE_SCRATCH` | the OS temp dir | Where `source publish` makes its scratch clone and stage (removed afterwards unless `--keep-scratch`), and keeps the history pages' render cache (`archilyzer-source-history/`). | common/lib/paths.ts (getPaths) |
+| `STAGIT_BIN` | `stagit` on PATH, then `~/.local/bin/stagit` | stagit, which renders the source's history pages (`/source/git/`: the log and a page per commit with its diff). Optional: without it the source is published without them. See [PUBLISH.md](PUBLISH.md). | common/lib/paths.ts (getPaths) |
## Runtime
diff --git a/common/lib/envVars.ts b/common/lib/envVars.ts
@@ -83,7 +83,8 @@ const DECLARED: EnvVarDecl[] = [
paths("ARCHILYZER_CONFIG_DIR", "`~/.config/archilyzer`", "The operator's private config dir, outside the repo: the two inputs of `archilyzer source publish` below. Never committed."),
paths("SOURCE_SCRUB_FILE", "`<ARCHILYZER_CONFIG_DIR>/source-scrub.txt`", "git-filter-repo `lhs==>rhs` rules applied to file contents AND commit messages when the source mirror is generated (`<home dir>==>/home/user` is built in and runs first). Every rule's left side is also denied. See [PUBLISH.md](PUBLISH.md)."),
paths("SOURCE_DENYLIST_FILE", "`<ARCHILYZER_CONFIG_DIR>/source-denylist.txt`", "Literals the published source must never contain, one per line (`i:` = any case). One hit anywhere in the mirror, the tree or the tarball refuses the publish."),
- paths("ARCHILYZER_SOURCE_SCRATCH", "the OS temp dir", "Where `source publish` makes its scratch clone and stage (removed afterwards unless `--keep-scratch`)."),
+ paths("ARCHILYZER_SOURCE_SCRATCH", "the OS temp dir", "Where `source publish` makes its scratch clone and stage (removed afterwards unless `--keep-scratch`), and keeps the history pages' render cache (`archilyzer-source-history/`)."),
+ paths("STAGIT_BIN", "`stagit` on PATH, then `~/.local/bin/stagit`", "stagit, which renders the source's history pages (`/source/git/`: the log and a page per commit with its diff). Optional: without it the source is published without them. See [PUBLISH.md](PUBLISH.md)."),
// ── runtime ────────────────────────────────────────────────────────────
{ name: "WORKER_TOKEN", audience: "runtime", default: "unset (both surfaces off)", readBy: "common/lib/workerToken.ts, scripts/archilyzer-ops.mjs, mcp/src/fetchClip.ts", doc: "Bearer token for the remote-worker API and for `/api/ops/*` (`pnpm ops`, the MCP's `fetch_clip`). Set the same value on both ends." },
diff --git a/common/lib/paths.ts b/common/lib/paths.ts
@@ -168,6 +168,11 @@ export type Paths = {
sourceDenylistFile: string;
// Where `source publish` makes its scratch clone (removed afterwards).
sourceScratchDir: string;
+ // stagit, which renders the source's history pages (/source/git/). An
+ // operator-installed tool, never vendored: `stagit` on PATH, then
+ // ~/.local/bin/stagit (publish/sourceHistory.ts resolveStagit). Without it
+ // the publish goes on without the history pages.
+ stagitBin: string;
};
let cached: Paths | null = null;
@@ -296,6 +301,7 @@ export function getPaths(): Paths {
process.env.SOURCE_DENYLIST_FILE ??
path.join(configDir, "source-denylist.txt"),
sourceScratchDir: process.env.ARCHILYZER_SOURCE_SCRATCH ?? os.tmpdir(),
+ stagitBin: process.env.STAGIT_BIN ?? "stagit",
};
return cached;
}
diff --git a/common/lib/sourceManifest.test.ts b/common/lib/sourceManifest.test.ts
@@ -48,3 +48,37 @@ test("a wrong version, a bad id or sha, or any number the page reads that is not
for (const [what, make] of broken) assert.equal(parseSourceManifest(make(good())), null, what);
for (const junk of [null, 1, "x", [], {}]) assert.equal(parseSourceManifest(junk), null, JSON.stringify(junk));
});
+
+// Release 15 slice SG: the history block is optional — absent is a manifest
+// without History links — and all-or-nothing when present.
+const history = () => ({
+ href: "/source/git/log.html",
+ commits: 3,
+ head: "2".repeat(40),
+ files: 9,
+ bytes: 1234,
+ sha256: "4".repeat(64),
+ tool: "stagit (sha256 0123456789ab)",
+});
+
+test("a manifest with a history block parses; without one it is the same manifest", () => {
+ const withHistory = { ...good(), history: history() };
+ assert.deepEqual(parseSourceManifest(withHistory), withHistory);
+ assert.equal(parseSourceManifest(good())?.history, undefined);
+});
+
+test("a history block that is present but malformed makes the manifest untrusted", () => {
+ const broken: Array<[string, (h: ReturnType<typeof history>) => unknown]> = [
+ ["null", () => null],
+ ["commits a string", (h) => ({ ...h, commits: "3" })],
+ ["files negative", (h) => ({ ...h, files: -1 })],
+ ["bytes missing", (h) => ({ ...h, bytes: undefined })],
+ ["short head", (h) => ({ ...h, head: "abc" })],
+ ["sha256 not hex", (h) => ({ ...h, sha256: "z".repeat(64) })],
+ ["no href", (h) => ({ ...h, href: undefined })],
+ ["tool a number", (h) => ({ ...h, tool: 1 })],
+ ];
+ for (const [what, make] of broken) {
+ assert.equal(parseSourceManifest({ ...good(), history: make(history()) }), null, what);
+ }
+});
diff --git a/common/lib/sourceManifest.ts b/common/lib/sourceManifest.ts
@@ -25,6 +25,16 @@ export const CLONE_URL = `${PROJECT_URL}/source/${MIRROR_DIR}`;
/** The raw tree's generated index. */
export const TREE_HREF = "/source/tree/";
+/**
+ * The history pages' directory under /source/: stagit's rendering of the
+ * mirror (publish/sourceHistory.ts) — the log, a page per commit with its
+ * diff, the refs, the files index (into the raw tree) and two Atom feeds.
+ */
+export const HISTORY_DIR = "git";
+export const HISTORY_LOG_HREF = `/source/${HISTORY_DIR}/log.html`;
+export const HISTORY_REFS_HREF = `/source/${HISTORY_DIR}/refs.html`;
+export const HISTORY_ATOM_HREF = `/source/${HISTORY_DIR}/atom.xml`;
+
/** The manifest's public path. */
export const SOURCE_MANIFEST_HREF = "/source/manifest.json";
@@ -69,6 +79,27 @@ export type SourceManifest = {
};
// The tools that made it (a filter-repo upgrade may change mirrorHead).
tools: { git: string; filterRepo: string };
+ // The history pages, when stagit rendered them (absent: no stagit on the
+ // publishing machine, or a render that failed — the page then shows no
+ // History links).
+ history?: SourceHistory;
+};
+
+export type SourceHistory = {
+ // The log page (HISTORY_LOG_HREF).
+ href: string;
+ // Every commit of the mirror's main has a page; `head` is the one the log
+ // starts at (the manifest's mirrorHead).
+ commits: number;
+ head: string;
+ // Every file under /source/git/ (the pages, the two feeds, style.css): how
+ // many, their bytes, and one sha256 over them — each file's path, size and
+ // sha256, in sorted path order — which the deploy check recomputes over out/.
+ files: number;
+ bytes: number;
+ sha256: string;
+ // The renderer: "stagit (sha256 <first 12 of its binary's>)".
+ tool: string;
};
const HEX40 = /^[0-9a-f]{40}$/;
@@ -101,5 +132,15 @@ export function parseSourceManifest(value: unknown): SourceManifest | null {
}
if (!obj(m.tree) || !num(m.tree.files) || !num(m.tree.dirs) || !num(m.tree.bytes)) return null;
if (!obj(m.audit)) return null;
+ // Optional, and all-or-nothing: a history block that is present but
+ // malformed makes the whole manifest untrusted, like any other bad field.
+ if (m.history !== undefined) {
+ const h = m.history as Partial<SourceHistory> | null;
+ if (!obj(h)) return null;
+ if (!num(h.commits) || !num(h.files) || !num(h.bytes)) return null;
+ if (typeof h.head !== "string" || !HEX40.test(h.head)) return null;
+ if (typeof h.sha256 !== "string" || !HEX64.test(h.sha256)) return null;
+ if (typeof h.href !== "string" || typeof h.tool !== "string") return null;
+ }
return m as SourceManifest;
}
diff --git a/common/publish/source.test.ts b/common/publish/source.test.ts
@@ -18,6 +18,7 @@ import {
} from "node:fs";
import os from "node:os";
import path from "node:path";
+import { fileURLToPath } from "node:url";
import type { Paths } from "../lib/paths";
import { CLONE_URL, MIRROR_DIR, TARBALL_HREF, TREE_HREF } from "../lib/sourceManifest";
import {
@@ -33,12 +34,14 @@ import {
publishSource,
publishedSourceProblem,
gitleaksIdentity,
+ historyDigest,
resolveFilterRepo,
rulesHashOf,
scratchRootProblem,
sourceDigest,
type SourcePublishOpts,
} from "./source";
+import { HISTORY_BACK_LINK, historyCacheDir } from "./sourceHistory";
// Run with:
// pnpm --filter yt-dlp-transcript-common test
@@ -125,6 +128,8 @@ function opts(repo: string, files: { scrubFile: string; denylistFile: string },
publicDir: path.join(dir("site"), "public"),
scratchRoot: path.join(TMP, "scratch"),
gitleaks: null,
+ // No history pages unless a test gives a stagit (the machine may have one).
+ stagit: null,
homeDir: HOME,
onLog: (l) => logs.push(l),
now: () => new Date("2026-09-28T12:00:00.000Z"),
@@ -249,6 +254,8 @@ test("skip: an unchanged main with unchanged rules, tools and files does nothing
filterRepo: "false (given)",
gitleaks: "skipped",
contentDigest: await sourceDigest(pub),
+ stagit: "absent",
+ history: null,
};
writeFileSync(state, JSON.stringify({ ...key, ...tweak }));
};
@@ -268,6 +275,7 @@ test("skip: an unchanged main with unchanged rules, tools and files does nothing
["another gitleaks", { gitleaks: "gitleaks 8.0 (abc)" }],
["other files", { contentDigest: "0".repeat(64) }],
["other rules", { rulesHash: "0".repeat(64) }],
+ ["another stagit", { stagit: "stagit (sha256 0123456789ab)" }],
] as Array<[string, Record<string, string>]>) {
await plant(tweak);
assert.equal(await run(), 1, `${what} must not skip`);
@@ -299,7 +307,7 @@ test("the rules hash moves with the step's version (review R2-L3), and loadSourc
const files = operatorFiles("a==>b\n", "");
const rules = await loadSourceRules({ ...files, homeDir: "/" });
assert.equal(rules.rulesHash, rulesHashOf(["a==>b"], rules.literals, SOURCE_STEP_VERSION));
- assert.ok(SOURCE_STEP_VERSION >= 3);
+ assert.ok(SOURCE_STEP_VERSION >= 4, "release 15 slice SG: the history pages");
});
test("gitleaksIdentity: skipped, absent, or the version line with the binary's sha256 (review R2-L4)", async () => {
@@ -551,7 +559,7 @@ test("a denied literal no rule removes: refused, nothing written, the report nev
assert.ok(kept, report);
// The log tildifies with the real home dir, so a TMPDIR under it prints "~/…".
const keptDir = kept[1].replace(/^~(?=\/|$)/, os.homedir());
- assert.ok(existsSync(path.join(keptDir, "bare")));
+ assert.ok(existsSync(path.join(keptDir, MIRROR_DIR)), "the clone is named as the mirror is (stagit names it after its directory)");
assert.ok(!existsSync(path.join(keptDir, "replace.txt")));
rmSync(keptDir, { recursive: true, force: true });
});
@@ -603,3 +611,232 @@ test("--no-source's clear: the manifest, mirror, tree, tarball and skip key go;
assert.equal(readFileSync(path.join(elsewhere, "snapshot.json"), "utf8"), "the other checkout's");
assert.match(logs.join("\n"), /previously published source .* was removed/);
});
+
+// ── the history pages (release 15 slice SG) ─────────────────────────────────
+//
+// Over a fake stagit that writes what stagit writes (sourceHistory.test.ts
+// runs the real one), and with filter-repo replaced by `true` — no rewrite,
+// so these run on any machine; the history does not care what the rewrite did.
+
+const TOKENS_FILE = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "styles", "tokens.css");
+
+// A page per commit (kept if there is one, as stagit keeps it), the top-level
+// pages and feeds, a per-file page, and the cache file under -c. `extra` goes
+// into every commit page; `pad` bytes are added to the head's page.
+function fakeStagit(log: string, o: { exit?: number; extra?: string; pad?: number } = {}): string {
+ const file = path.join(dir("fake-stagit"), "stagit");
+ writeFileSync(
+ file,
+ `#!/bin/sh
+cache=""; base=""; repo=""
+while [ $# -gt 0 ]; do
+ case "$1" in
+ -c) cache="$2"; shift 2;;
+ -u) base="$2"; shift 2;;
+ *) repo="$1"; shift;;
+ esac
+done
+echo "cache=$cache base=$base repo=$(basename "$repo")" >> '${log}'
+${o.exit ? `echo 'stagit: something broke' >&2; exit ${o.exit}` : ""}
+mkdir -p commit file
+for c in $(git --git-dir "$repo" rev-list HEAD); do
+ [ -f "commit/$c.html" ] || printf '<html>\\n<head>\\n</head>\\n<body>\\n<a href="../file/README.md.html">README</a> ${o.extra ?? ""}\\n</body>\\n</html>\\n' > "commit/$c.html"
+done
+${o.pad ? `head -c ${o.pad} /dev/zero >> "commit/$(git --git-dir "$repo" rev-parse HEAD).html"` : ""}
+printf '<html>\\n<head>\\n</head>\\n<body>\\n<span class="desc">%s</span> %s <a href="file/README.md.html">README</a>\\n</body>\\n</html>\\n' "$(cat "$repo/description")" "$(cat "$repo/url")" > log.html
+for f in files refs; do printf '<html>\\n<head>\\n</head>\\n<body>\\n</body>\\n</html>\\n' > $f.html; done
+printf '<feed>%s</feed>\\n' "$base" > atom.xml
+printf '<feed/>\\n' > tags.xml
+printf 'x\\n' > file/README.md.html
+if [ -n "$cache" ]; then git --git-dir "$repo" rev-parse HEAD > "$cache"; fi
+exit 0
+`,
+ );
+ chmodSync(file, 0o755);
+ return file;
+}
+
+const stagitCalls = (log: string) => (existsSync(log) ? readFileSync(log, "utf8").trim().split("\n") : []);
+
+test("history: published with the source — the allowlist at /source/git/, the manifest's block, the skip key, incremental, and the deploy check names it", async () => {
+ const repo = sourceRepo();
+ const files = operatorFiles("", "");
+ const logs: string[] = [];
+ const log = path.join(dir("stagit-log"), "calls");
+ const scratchRoot = dir("scratch-root");
+ const o = opts(repo, files, logs, { filterRepo: ["true"], stagit: fakeStagit(log), tokensFile: TOKENS_FILE, scratchRoot });
+ const pub = o.publicDir!;
+ const site = path.dirname(pub);
+
+ assert.equal(await publishSource(o), 0, logs.join("\n"));
+ let text = logs.join("\n");
+ assert.match(text, /\[source\] history: stagit \(sha256 [0-9a-f]{12}\) — 3 commits, 3 pages rendered; 9 files, [\d.]+ MB, the largest git\/\S+ [\d.]+ MB/);
+ assert.match(text, /\[source\] published main .* tree \d+ dirs, history 3 commits in 9 files\)/);
+ assert.deepEqual(stagitCalls(log), [
+ `cache=${path.join(historyCacheDir(scratchRoot), "stagit.cache")} base=https://archilyzer.pages.dev/source/git/ repo=${MIRROR_DIR}`,
+ ]);
+
+ const manifest = JSON.parse(readFileSync(path.join(pub, "source", "manifest.json"), "utf8"));
+ assert.deepEqual(Object.keys(manifest.history).sort(), ["bytes", "commits", "files", "head", "href", "sha256", "tool"]);
+ assert.equal(manifest.history.href, "/source/git/log.html");
+ assert.equal(manifest.history.commits, 3);
+ assert.equal(manifest.history.head, manifest.mirrorHead);
+ assert.equal(manifest.history.files, 9);
+ assert.equal(manifest.history.sha256, await historyDigest(pub));
+ assert.match(manifest.history.tool, /^stagit \(sha256 [0-9a-f]{12}\)$/);
+ assert.ok(manifest.files >= 9, "the manifest's count includes the history");
+
+ const git = path.join(pub, "source", "git");
+ assert.deepEqual(readdirSync(git).sort(), ["atom.xml", "commit", "files.html", "log.html", "refs.html", "style.css", "tags.xml"]);
+ const logHtml = readFileSync(path.join(git, "log.html"), "utf8");
+ assert.ok(logHtml.includes(`<span class="desc">Archilyzer</span> ${CLONE_URL} `), "stagit's header: the name and the clone URL");
+ assert.ok(logHtml.includes(HISTORY_BACK_LINK));
+ assert.match(logHtml, /<script>\(function\(\)\{try\{var d=document\.documentElement/);
+ assert.match(logHtml, /href="\.\.\/tree\/README\.md"/);
+ assert.equal(readFileSync(path.join(git, "atom.xml"), "utf8"), "<feed>https://archilyzer.pages.dev/source/git/</feed>\n");
+ assert.match(readFileSync(path.join(git, "style.css"), "utf8"), /html\[data-base="dark"\] \{/);
+ // The description and url files stagit read are never published.
+ for (const f of ["description", "url"]) assert.ok(!existsSync(path.join(pub, "source", MIRROR_DIR, f)), f);
+
+ const stateFile = path.join(site, ".source-publish.json");
+ const state = JSON.parse(readFileSync(stateFile, "utf8"));
+ assert.equal(state.stagit, manifest.history.tool);
+ assert.deepEqual(state.history, { files: 9, digest: manifest.history.sha256 });
+
+ // Unchanged: skipped, stagit not run.
+ logs.length = 0;
+ assert.equal(await publishSource(o), 0);
+ assert.match(logs.join("\n"), /up to date at/);
+ assert.equal(stagitCalls(log).length, 1);
+
+ // The deploy check: this publish in out/ deploys; a history page edited,
+ // or the history missing, is named.
+ const out = path.join(dir("out"), "out");
+ mkdirSync(path.join(out, "source"), { recursive: true });
+ writeFileSync(path.join(out, "source", "index.html"), "<p>the page</p>");
+ cpSync(path.join(pub, "source"), path.join(out, "source"), { recursive: true });
+ cpSync(path.join(pub, "downloads"), path.join(out, "downloads"), { recursive: true });
+ const check = { ...files, homeDir: HOME, publicDir: pub, sourceRepo: path.join(repo, ".git"), gitleaks: null, stagit: null };
+ assert.equal(await publishedSourceProblem(o.paths!, out, check), null);
+ const outLog = path.join(out, "source", "git", "log.html");
+ const was = readFileSync(outLog);
+ writeFileSync(outLog, Buffer.concat([was, Buffer.from(" ")]));
+ assert.match((await publishedSourceProblem(o.paths!, out, check))!, /history pages \(\/source\/git\/\) are not the ones that were audited — run `archilyzer build homepage`/);
+ rmSync(path.join(out, "source", "git"), { recursive: true });
+ assert.match((await publishedSourceProblem(o.paths!, out, check))!, /history pages \(\/source\/git\/\) are not the ones that were audited/);
+ cpSync(path.join(pub, "source", "git"), path.join(out, "source", "git"), { recursive: true });
+ assert.equal(await publishedSourceProblem(o.paths!, out, check), null, "restored, it deploys");
+
+ // A new commit on main: only its page is rendered, the rest from the cache.
+ writeFileSync(path.join(repo, "later.txt"), "x\n");
+ gitIn(repo, "add", "-A");
+ gitIn(repo, "commit", "-q", "-m", "later");
+ logs.length = 0;
+ assert.equal(await publishSource(o), 0, logs.join("\n"));
+ assert.match(logs.join("\n"), /— 4 commits, 1 page rendered \(the rest from the cache\); 10 files/);
+ // --force renders every page again.
+ logs.length = 0;
+ assert.equal(await publishSource({ ...o, force: true }), 0, logs.join("\n"));
+ assert.match(logs.join("\n"), /— 4 commits, 4 pages rendered; 10 files/);
+ // --check writes nothing — the cache included — and still renders (and
+ // sweeps) the pages in its own scratch.
+ const cacheBefore = readFileSync(path.join(historyCacheDir(scratchRoot), "key.json"), "utf8");
+ logs.length = 0;
+ assert.equal(await publishSource({ ...o, check: true }), 0, logs.join("\n"));
+ assert.match(logs.join("\n"), /— 4 commits, 4 pages rendered;/);
+ assert.equal(stagitCalls(log).at(-1), `cache= base=https://archilyzer.pages.dev/source/git/ repo=${MIRROR_DIR}`);
+ assert.equal(readFileSync(path.join(historyCacheDir(scratchRoot), "key.json"), "utf8"), cacheBefore);
+});
+
+test("history: without stagit — one line with the install, nothing at /source/git/, no block in the manifest; installing it re-publishes", async () => {
+ const repo = sourceRepo();
+ const files = operatorFiles("", "");
+ const logs: string[] = [];
+ const o = opts(repo, files, logs, { filterRepo: ["true"], stagit: null, tokensFile: TOKENS_FILE, scratchRoot: dir("scratch-root") });
+ const pub = o.publicDir!;
+ assert.equal(await publishSource(o), 0, logs.join("\n"));
+ const said = logs.filter((l) => /stagit/.test(l));
+ assert.deepEqual(said, [
+ "[source] stagit not found (not on PATH, not in ~/.local/bin) — publishing without the history pages (/source/git/); install it once: git clone git://git.codemadness.org/stagit && make -C stagit && cp stagit/stagit ~/.local/bin/",
+ ]);
+ assert.match(logs.join("\n"), /\[source\] published main .*, no history\)/);
+ const manifest = JSON.parse(readFileSync(path.join(pub, "source", "manifest.json"), "utf8"));
+ assert.ok(!("history" in manifest));
+ assert.ok(!existsSync(path.join(pub, "source", "git")));
+ const state = JSON.parse(readFileSync(path.join(path.dirname(pub), ".source-publish.json"), "utf8"));
+ assert.equal(state.stagit, "absent");
+ assert.equal(state.history, null);
+
+ logs.length = 0;
+ assert.equal(await publishSource(o), 0);
+ assert.match(logs.join("\n"), /up to date at/, "no stagit, nothing changed: skipped");
+
+ // stagit installed since: not skipped, and the history is published.
+ logs.length = 0;
+ const log = path.join(dir("stagit-log"), "calls");
+ assert.equal(await publishSource({ ...o, stagit: fakeStagit(log) }), 0, logs.join("\n"));
+ assert.ok(!/up to date/.test(logs.join("\n")));
+ assert.equal(JSON.parse(readFileSync(path.join(pub, "source", "manifest.json"), "utf8")).history.commits, 3);
+ assert.ok(existsSync(path.join(pub, "source", "git", "log.html")));
+});
+
+test("history: a denied literal in a history page refuses, names the page and never the literal, and withdraws the publish and the render cache", async () => {
+ const repo = sourceRepo();
+ const planted = "plantedinhistory";
+ const files = operatorFiles("", "");
+ const logs: string[] = [];
+ const log = path.join(dir("stagit-log"), "calls");
+ const scratchRoot = dir("scratch-root");
+ // The literal is only ever in what stagit writes, never in the repository:
+ // what this proves is that the file gate reads source/git/**.
+ const o = opts(repo, files, logs, { filterRepo: ["true"], stagit: fakeStagit(log, { extra: `said ${planted} here` }), tokensFile: TOKENS_FILE, scratchRoot });
+ assert.equal(await publishSource(o), 0, logs.join("\n"));
+ assert.ok(existsSync(path.join(o.publicDir!, "source", "git", "log.html")));
+ assert.ok(existsSync(historyCacheDir(scratchRoot)));
+
+ // Deny it: the rules change, so every page is rendered again — and swept.
+ writeFileSync(files.denylistFile, `${planted}\n`);
+ logs.length = 0;
+ assert.equal(await publishSource(o), 1);
+ const report = logs.join("\n");
+ assert.match(report, /AUDIT REFUSED: 3 hits in \d+ objects \(3 commits\), \d+ staged files/);
+ assert.match(report, /denylist line 1 \(len 16\): 3 in files/);
+ assert.match(report, /\[source\] {3}file source\/git\/commit\/[0-9a-f]{40}\.html \(contents, byte \d+\): denylist line 1 \(len 16\)/);
+ assert.ok(!report.includes(planted), report);
+ assert.ok(!report.includes("said"), "no byte from beside the hit");
+ assert.match(report, /the previous publish was WITHDRAWN \(mirror, tree, history, tarball\)/);
+ assert.ok(!existsSync(path.join(o.publicDir!, "source")), "public/source, the history with it, is withdrawn");
+ assert.ok(!existsSync(historyCacheDir(scratchRoot)), "the render cache is removed too");
+});
+
+test("history: a stagit that fails, or a page over the host's limit, is a WARNING; the rest is published, and the next build tries again", async () => {
+ const repo = sourceRepo();
+ const files = operatorFiles("", "");
+ const logs: string[] = [];
+ const log = path.join(dir("stagit-log"), "calls");
+ const o = opts(repo, files, logs, { filterRepo: ["true"], stagit: fakeStagit(log, { exit: 3 }), tokensFile: TOKENS_FILE, scratchRoot: dir("scratch-root") });
+ const pub = o.publicDir!;
+ assert.equal(await publishSource(o), 0, logs.join("\n"));
+ assert.match(logs.join("\n"), /\[source\] WARNING: the history pages were not rendered: stagit exited 3: stagit: something broke — publishing without the history pages \(\/source\/git\/\)/);
+ assert.ok(existsSync(path.join(pub, "source", MIRROR_DIR, "info", "refs")), "the mirror is published");
+ assert.ok(!existsSync(path.join(pub, "source", "git")));
+ const state = JSON.parse(readFileSync(path.join(path.dirname(pub), ".source-publish.json"), "utf8"));
+ assert.match(state.stagit, /^stagit \(sha256/);
+ assert.equal(state.history, null);
+ logs.length = 0;
+ assert.equal(await publishSource(o), 0);
+ assert.ok(!/up to date/.test(logs.join("\n")), "a stagit present and no history: the next build tries again");
+ assert.equal(stagitCalls(log).length, 2);
+
+ // One page over the step's 24 MiB: the history is dropped, not the publish.
+ logs.length = 0;
+ assert.equal(await publishSource({ ...o, stagit: fakeStagit(log, { pad: MAX_FILE_BYTES }) }), 0, logs.join("\n"));
+ assert.match(logs.join("\n"), /\[source\] WARNING: git\/commit\/[0-9a-f]{40}\.html is 24\.0 MiB, over the step's limit of 24\.0 MiB \(Pages allows 25 MiB per file\) — publishing without the history pages/);
+ assert.ok(!existsSync(path.join(pub, "source", "git")));
+ assert.ok(!("history" in JSON.parse(readFileSync(path.join(pub, "source", "manifest.json"), "utf8"))));
+
+ // Unreadable design tokens: the same.
+ logs.length = 0;
+ assert.equal(await publishSource({ ...o, stagit: fakeStagit(log), tokensFile: path.join(dir("none"), "tokens.css") }), 0);
+ assert.match(logs.join("\n"), /\[source\] WARNING: the design tokens \(.*tokens\.css\) could not be read — publishing without the history pages/);
+});
diff --git a/common/publish/source.ts b/common/publish/source.ts
@@ -3,12 +3,15 @@
// The private repository is never rewritten. Every publish makes a FRESH bare
// clone of its `main`, rewrites that copy with git-filter-repo (the operator's
// scrub rules over file contents AND commit messages), repacks it for git's
-// dumb-HTTP protocol, and publishes three things under homepage/public:
+// dumb-HTTP protocol, and publishes these under homepage/public:
//
// source/archilyzer.git/ a clonable mirror: HEAD, refs, info/refs,
// objects/info/packs, the packs — static files only
// source/tree/ the tracked files of main, raw, with an
// index.html per directory (sourceTree.ts)
+// source/git/ the history: the log, a page per commit with its
+// diff, the refs, two Atom feeds — stagit's, when
+// it is installed (sourceHistory.ts)
// downloads/ archilyzer-source.tar.gz + snapshot.json, the
// tarball the Downloads page has always offered
// source/manifest.json written LAST: what was published, from what
@@ -38,16 +41,33 @@ import path from "node:path";
import { runChildIntoLog } from "../jobs/runChild";
import { copyPublicFile, ownDir, writePublicFile } from "../bin/_publicFile";
import { getPaths, type Paths } from "../lib/paths";
+import { PROJECT_NAME, PROJECT_URL } from "../lib/project";
import {
CLONE_URL,
+ HISTORY_DIR,
+ HISTORY_LOG_HREF,
MIRROR_DIR,
SOURCE_MANIFEST_VERSION,
TARBALL_HREF,
TREE_HREF,
parseSourceManifest,
+ type SourceHistory,
type SourceManifest,
} from "../lib/sourceManifest";
import {
+ HistoryProblem,
+ STAGIT_INSTALL,
+ dropHistoryCache,
+ historyCacheDir,
+ historyCacheKey,
+ historyStylesheet,
+ homepageThemeScript,
+ renderHistory,
+ resolveStagit,
+ stagitIdentity,
+ type HistoryRun,
+} from "./sourceHistory";
+import {
SourceRefusal,
auditBare,
auditFiles,
@@ -82,8 +102,11 @@ export const SOURCE_BRANCH = "main";
* left sides refused, `404.html` refused, no context bytes.
* 3 — the re-review: masked refusal messages, a digest of every published
* file and the gitleaks identity in the key.
+ * 4 — release 15 slice SG: the history pages (source/git/, stagit) are
+ * staged, swept by the file gate and bound by the digest; stagit's
+ * identity is in the key.
*/
-export const SOURCE_STEP_VERSION = 3;
+export const SOURCE_STEP_VERSION = 4;
/** What `pipx run` fetches when `git filter-repo` is not installed. */
export const FILTER_REPO_PIPX_SPEC = "git-filter-repo==2.47.0";
@@ -120,6 +143,12 @@ export type SourcePublishOpts = PublishOpts & {
filterRepo?: string[] | null;
// The gitleaks binary; null skips the secret scan (tests). Default "gitleaks".
gitleaks?: string | null;
+ // The stagit binary; null publishes without the history pages, as a
+ // machine without stagit does. Default: resolveStagit(paths.stagitBin).
+ stagit?: string | null;
+ // The design tokens the history pages' style.css is written from. Default:
+ // <repo>/common/styles/tokens.css.
+ tokensFile?: string;
// Where the scratch dir is made. Default: paths.sourceScratchDir.
scratchRoot?: string;
// The environment the children run in (PATH decides which tools). Default:
@@ -433,18 +462,46 @@ type PublishState = {
// sourceDigest() over every published file: an out/ (or public/) holding
// another publish's mirror or tree beside this manifest does not match.
contentDigest: string;
+ // stagitIdentity(): "absent", or which stagit rendered the history. An
+ // install (or an upgrade) re-publishes.
+ stagit: string;
+ // The history pages as audited (historyDigest), or null when none were
+ // published. A null beside a stagit that is present (a render that failed,
+ // pages over the limits) never skips: the next build tries again.
+ history: { files: number; digest: string } | null;
};
/**
* One digest over every file a publish puts on the site, under `root` laid
* out as public/ and out/ both are: `source/archilyzer.git/**`,
- * `source/tree/**`, `source/manifest.json`, `downloads/archilyzer-source.tar.gz`
- * and `downloads/snapshot.json`. Each file contributes its relative path, size
- * and sha256 (streamed), in sorted path order; a symlink or a missing piece
- * contributes a marker, so it can only differ. The /source PAGE and anything
- * else Next writes beside them are not part of it.
+ * `source/tree/**`, `source/git/**`, `source/manifest.json`,
+ * `downloads/archilyzer-source.tar.gz` and `downloads/snapshot.json`. Each
+ * file contributes its relative path, size and sha256 (streamed), in sorted
+ * path order; a symlink or a missing piece contributes a marker, so it can
+ * only differ. The /source PAGE and anything else Next writes beside them are
+ * not part of it.
*/
export async function sourceDigest(root: string): Promise<string> {
+ return digestOf(root, [
+ `source/${MIRROR_DIR}`,
+ "source/tree",
+ `source/${HISTORY_DIR}`,
+ "source/manifest.json",
+ `downloads/${TARBALL_NAME}`,
+ "downloads/snapshot.json",
+ ]);
+}
+
+/**
+ * The same digest over the history pages alone (`source/git/**`): the
+ * manifest's `history.sha256`, and what the deploy check recomputes over
+ * out/ to name a history that is not the audited one.
+ */
+export async function historyDigest(root: string): Promise<string> {
+ return digestOf(root, [`source/${HISTORY_DIR}`]);
+}
+
+async function digestOf(root: string, tops: readonly string[]): Promise<string> {
const entries: string[] = [];
const walk = async (rel: string): Promise<void> => {
const abs = path.join(root, rel);
@@ -459,15 +516,7 @@ export async function sourceDigest(root: string): Promise<string> {
entries.push(`${rel}\0${st.size}\0${await sha256File(abs)}`);
}
};
- for (const top of [
- `source/${MIRROR_DIR}`,
- "source/tree",
- "source/manifest.json",
- `downloads/${TARBALL_NAME}`,
- "downloads/snapshot.json",
- ]) {
- await walk(top);
- }
+ for (const top of tops) await walk(top);
entries.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
const h = createHash("sha256");
for (const e of entries) h.update(`${e}\n`);
@@ -512,10 +561,10 @@ export const NO_REPOSITORY =
/**
* Remove a publish from `publicDir`: the manifest FIRST (the page never
- * describes a half-removed tree), the skip key, the mirror and the tree, the
- * tarball and snapshot.json. Link-safe like the install: a linked
- * `source/` or `downloads/` (a worktree's, into the primary) goes as a link,
- * its target untouched. True when there was something to remove.
+ * describes a half-removed tree), the skip key, the mirror, the tree and the
+ * history pages, the tarball and snapshot.json. Link-safe like the install: a
+ * linked `source/` or `downloads/` (a worktree's, into the primary) goes as a
+ * link, its target untouched. True when there was something to remove.
*/
export async function removePublishedSource(publicDir: string): Promise<boolean> {
const pubSource = path.join(publicDir, "source");
@@ -523,6 +572,7 @@ export async function removePublishedSource(publicDir: string): Promise<boolean>
const had =
existsSync(path.join(pubSource, "manifest.json")) ||
existsSync(path.join(pubSource, MIRROR_DIR)) ||
+ existsSync(path.join(pubSource, HISTORY_DIR)) ||
existsSync(path.join(pubDownloads, TARBALL_NAME));
await rm(path.join(pubSource, "manifest.json"), { force: true });
await rm(statePath(publicDir), { force: true });
@@ -614,14 +664,16 @@ export async function publishSource(opts: SourcePublishOpts = {}): Promise<numbe
const signal = opts.signal ?? new AbortController().signal;
const ctx: Ctx = { onLog, signal, env: cleanGitEnv(opts.env ?? process.env), literals: [] };
const publicDir = sourcePublicDir(paths, opts.publicDir);
- const progress = { rulesLoaded: false };
+ const progress = { rulesLoaded: false, scratchRoot: null as string | null };
const withdraw = async () => {
if (!progress.rulesLoaded || opts.check) return;
if (await removePublishedSource(publicDir)) {
onLog(
- "[source] the previous publish was WITHDRAWN (mirror, tree, tarball): it was audited under rules that may not be today's. /source shows its empty state until a publish passes.",
+ "[source] the previous publish was WITHDRAWN (mirror, tree, history, tarball): it was audited under rules that may not be today's. /source shows its empty state until a publish passes.",
);
}
+ // The history's render cache was rendered under those rules too.
+ if (progress.scratchRoot) await dropHistoryCache(historyCacheDir(progress.scratchRoot));
};
let code: number;
try {
@@ -649,7 +701,7 @@ async function publish(
paths: Paths,
ctx: Ctx,
publicDir: string,
- progress: { rulesLoaded: boolean },
+ progress: { rulesLoaded: boolean; scratchRoot: string | null },
): Promise<number> {
const { onLog } = ctx;
const started = Date.now();
@@ -684,9 +736,17 @@ async function publish(
const filterRepoId = `${filterRepo.label} ${filterRepo.version}`;
const gitleaksBin = opts.gitleaks === undefined ? "gitleaks" : opts.gitleaks;
const gitleaksId = await gitleaksIdentity(gitleaksBin, ctx.env);
+ // stagit is optional: absent, the source is published without its history.
+ const stagit =
+ opts.stagit === undefined
+ ? resolveStagit(paths.stagitBin ?? "stagit", ctx.env)
+ : opts.stagit && existsSync(opts.stagit)
+ ? opts.stagit
+ : null;
+ const stagitId = await stagitIdentity(stagit);
// 4. Nothing changed: skip. The key is main, the rules (with the step's
- // version), both tools, and the published files themselves.
+ // version), the tools, and the published files themselves.
if (!opts.force && !opts.check) {
const manifest = await readPublishedManifest(publicDir);
const state = (await readJson(statePath(publicDir))) as Partial<PublishState> | null;
@@ -696,6 +756,8 @@ async function publish(
state.rulesHash === rules.rulesHash &&
state.filterRepo === filterRepoId &&
state.gitleaks === gitleaksId &&
+ state.stagit === stagitId &&
+ (stagitId === "absent" ? state.history === null : !!state.history) &&
state.mirrorHead === manifest.mirrorHead &&
existsSync(path.join(pubSource, MIRROR_DIR, "info", "refs")) &&
existsSync(path.join(pubDownloads, TARBALL_NAME)) &&
@@ -720,11 +782,14 @@ async function publish(
["the public dir", publicDir],
]);
if (badRoot) throw new SourceRefusal(badRoot);
+ progress.scratchRoot = scratchRoot;
await mkdir(scratchRoot, { recursive: true });
const scratch = await mkdtemp(path.join(scratchRoot, "archilyzer-source-"));
const replace = path.join(scratch, "replace.txt");
try {
- const bare = path.join(scratch, "bare");
+ // Named as the published mirror is: stagit names the repository after its
+ // directory ("archilyzer", the `.git` dropped).
+ const bare = path.join(scratch, MIRROR_DIR);
// 5. A fresh bare clone of main alone. --no-local: through upload-pack, not
// a copy of the object dir (which would carry every loose leftover).
@@ -852,6 +917,41 @@ async function publish(
}
const mirrorFiles = await walkFiles(stageMirror);
+ // 12b. The history pages (stagit), rendered from the audited mirror into
+ // the stage, so the file gate (13) reads every page. Without stagit — or
+ // when the render fails, or would break the host's limits — one line says
+ // so and the rest is published without them: never a failed build.
+ const stageHistoryDir = path.join(stageSource, HISTORY_DIR);
+ let history: SourceHistory | null = null;
+ if (!stagit) {
+ const why = (paths.stagitBin ?? "stagit").includes("/")
+ ? "STAGIT_BIN names no executable"
+ : "not on PATH, not in ~/.local/bin";
+ onLog(
+ `[source] stagit not found (${why}) — publishing without the history pages (/source/${HISTORY_DIR}/); install it once: ${STAGIT_INSTALL}`,
+ );
+ } else {
+ const commits = Number(lastLine((await g(["rev-list", "--count", `refs/heads/${SOURCE_BRANCH}`])).out));
+ history = await stageHistoryPages({
+ ctx,
+ opts,
+ paths,
+ stagit,
+ stagitId,
+ bare,
+ mirrorHead,
+ commits,
+ scratch,
+ scratchRoot,
+ rulesHash: rules.rulesHash,
+ filterRepoId,
+ stage,
+ dest: stageHistoryDir,
+ // Everything staged so far, and the manifest still to come.
+ stagedSoFar: (await walkFiles(stage)).length + 1,
+ });
+ }
+
// The manifest, staged with the rest so the file sweep reads it too. It
// carries no rules hash and no literal count: `plans/` publishes which
// literals the plan put in the files, so either would say whether the
@@ -881,6 +981,7 @@ async function publish(
gitleaks: audit.gitleaks === "clean" ? "clean" : "skipped",
},
tools: { git: gitVersion, filterRepo: filterRepoId },
+ ...(history ? { history } : {}),
};
const manifestText = JSON.stringify(manifest, null, 2) + "\n";
await writeFile(path.join(stageSource, "manifest.json"), manifestText);
@@ -904,7 +1005,9 @@ async function publish(
const totalBytes = all.reduce((n, f) => n + f.bytes, 0);
const summary =
`main ${sourceCommit.slice(0, 12)} as ${mirrorHead.slice(0, 12)}: ${all.length} files, ${mb(totalBytes)} MB ` +
- `(mirror ${packs} pack${packs === 1 ? "" : "s"}, tree ${tree.dirs} dirs), tarball ${mb(tarBytes)} MB sha256 ${tarSha.slice(0, 12)}`;
+ `(mirror ${packs} pack${packs === 1 ? "" : "s"}, tree ${tree.dirs} dirs, ` +
+ `${history ? `history ${history.commits} commits in ${history.files} files` : "no history"}), ` +
+ `tarball ${mb(tarBytes)} MB sha256 ${tarSha.slice(0, 12)}`;
// 15. --check writes nothing.
if (opts.check) {
@@ -919,8 +1022,10 @@ async function publish(
await rm(path.join(pubSource, "manifest.json"), { force: true });
await rm(path.join(pubSource, MIRROR_DIR), { recursive: true, force: true });
await rm(path.join(pubSource, "tree"), { recursive: true, force: true });
+ await rm(path.join(pubSource, HISTORY_DIR), { recursive: true, force: true });
await cp(stageMirror, path.join(pubSource, MIRROR_DIR), { recursive: true });
await cp(stageTree, path.join(pubSource, "tree"), { recursive: true });
+ if (history) await cp(stageHistoryDir, path.join(pubSource, HISTORY_DIR), { recursive: true });
await ownDir(pubDownloads);
await copyPublicFile(tarball, path.join(pubDownloads, TARBALL_NAME));
await writePublicFile(path.join(pubDownloads, "snapshot.json"), snapshotText);
@@ -931,6 +1036,8 @@ async function publish(
filterRepo: filterRepoId,
gitleaks: gitleaksId,
contentDigest,
+ stagit: stagitId,
+ history: history ? { files: history.files, digest: history.sha256 } : null,
};
await writeFile(statePath(publicDir), JSON.stringify(state, null, 2) + "\n");
await writePublicFile(path.join(pubSource, "manifest.json"), manifestText);
@@ -954,6 +1061,118 @@ function elapsed(started: number): string {
}
/**
+ * Step 12b: render the history pages into `a.dest` (stage/source/git) and say
+ * what they are, or say why not and leave no `dest` — a WARNING, never a
+ * refusal: the tokens unreadable, stagit failing, or pages that would break
+ * the host's limits (the step's, which step 14 applies to the whole publish).
+ * A cancel still cancels.
+ */
+async function stageHistoryPages(a: {
+ ctx: Ctx;
+ opts: SourcePublishOpts;
+ paths: Paths;
+ stagit: string;
+ stagitId: string;
+ bare: string;
+ mirrorHead: string;
+ commits: number;
+ scratch: string;
+ scratchRoot: string;
+ rulesHash: string;
+ filterRepoId: string;
+ stage: string;
+ dest: string;
+ stagedSoFar: number;
+}): Promise<SourceHistory | null> {
+ const { ctx } = a;
+ const started = Date.now();
+ const where = `/source/${HISTORY_DIR}/`;
+ const without = async (why: string): Promise<null> => {
+ await rm(a.dest, { recursive: true, force: true });
+ ctx.onLog(`[source] WARNING: ${maskLiterals(why, ctx.literals)} — publishing without the history pages (${where})`);
+ return null;
+ };
+ const tokensFile =
+ a.opts.tokensFile ??
+ path.join(/* turbopackIgnore: true */ a.paths.monorepoRoot, "common", "styles", "tokens.css");
+ let stylesheet: string;
+ try {
+ stylesheet = historyStylesheet(await readFile(tokensFile, "utf8"));
+ } catch (err) {
+ if (err instanceof HistoryProblem) return without(err.message);
+ return without(`the design tokens (${tildify(tokensFile)}) could not be read`);
+ }
+ // stagit and git through the step's own runner (the environment, the
+ // cancel, the timeout); a failure or a timeout is the history's problem.
+ const child: HistoryRun = async (command, args, o) => {
+ try {
+ return await run(ctx, command, args, { ...o, allowFail: true });
+ } catch (err) {
+ if (err instanceof SourceRefusal) throw new HistoryProblem(err.message);
+ throw err;
+ }
+ };
+ const baseUrl = `${PROJECT_URL}${where}`;
+ let r;
+ try {
+ r = await renderHistory({
+ stagit: a.stagit,
+ gitDir: a.bare,
+ head: a.mirrorHead,
+ commits: a.commits,
+ dest: a.dest,
+ scratch: a.scratch,
+ // `--check` writes nothing outside its own scratch.
+ cacheDir: a.opts.check ? null : historyCacheDir(a.scratchRoot),
+ cacheKey: historyCacheKey({
+ rulesHash: a.rulesHash,
+ filterRepo: a.filterRepoId,
+ stagit: a.stagitId,
+ description: PROJECT_NAME,
+ cloneUrl: CLONE_URL,
+ baseUrl,
+ }),
+ fresh: !!a.opts.force,
+ description: PROJECT_NAME,
+ cloneUrl: CLONE_URL,
+ baseUrl,
+ stylesheet,
+ themeScript: homepageThemeScript(),
+ run: child,
+ onLog: ctx.onLog,
+ });
+ } catch (err) {
+ if (err instanceof HistoryProblem) return without(`the history pages were not rendered: ${err.message}`);
+ throw err;
+ }
+ if (a.stagedSoFar + r.files > MAX_FILES) {
+ return without(
+ `${r.files} history files would make ${a.stagedSoFar + r.files} files to publish, over the step's limit of ${MAX_FILES} (Pages allows 20,000 per deployment)`,
+ );
+ }
+ if (r.largest.bytes > MAX_FILE_BYTES) {
+ return without(
+ `${HISTORY_DIR}/${r.largest.rel} is ${mb(r.largest.bytes)} MiB, over the step's limit of ${mb(MAX_FILE_BYTES)} MiB (Pages allows 25 MiB per file)`,
+ );
+ }
+ const sha256 = await historyDigest(a.stage);
+ ctx.onLog(
+ `[source] history: ${a.stagitId} — ${a.commits} commits, ${r.rendered} page${r.rendered === 1 ? "" : "s"} rendered` +
+ `${r.cached ? " (the rest from the cache)" : ""}; ${r.files} files, ${mb(r.bytes)} MB, ` +
+ `the largest ${HISTORY_DIR}/${r.largest.rel} ${mb(r.largest.bytes)} MB (${elapsed(started)})`,
+ );
+ return {
+ href: HISTORY_LOG_HREF,
+ commits: a.commits,
+ head: a.mirrorHead,
+ files: r.files,
+ bytes: r.bytes,
+ sha256,
+ tool: a.stagitId,
+ };
+}
+
+/**
* `build homepage --no-source`: remove what an earlier publish left, because
* it was audited against the rules of ITS day. The pages then show their
* empty states.
@@ -966,7 +1185,7 @@ export async function clearPublishedSource(
const had = await removePublishedSource(sourcePublicDir(paths, opts.publicDir));
onLog(
had
- ? "[notice] --no-source: the previously published source (mirror, tree, tarball) was removed — this build ships none.\n"
+ ? "[notice] --no-source: the previously published source (mirror, tree, history, tarball) was removed — this build ships none.\n"
: "[notice] --no-source: no source published in this build.\n",
);
}
@@ -1009,6 +1228,7 @@ export async function publishedSourceProblem(
path.join(outSource, "manifest.json"),
path.join(outSource, MIRROR_DIR),
path.join(outSource, "tree"),
+ path.join(outSource, HISTORY_DIR),
outTarball,
];
// A `--no-source` build: the page's empty state and nothing else.
@@ -1061,6 +1281,13 @@ export async function publishedSourceProblem(
if (state.gitleaks !== gitleaks) {
return `homepage/out's source was scanned by another gitleaks (${state.gitleaks}; this machine has ${gitleaks}) — ${rebuild}`;
}
+ // The history pages: exactly the audited ones, or none when none were
+ // published. The digest below binds them too; this names them.
+ const historyNow = existsSync(path.join(outSource, HISTORY_DIR)) ? await historyDigest(outDir) : null;
+ const historyThen = state.history?.digest ?? null;
+ if (historyNow !== historyThen || (manifest.history?.sha256 ?? null) !== historyThen) {
+ return `homepage/out's history pages (/source/${HISTORY_DIR}/) are not the ones that were audited — ${rebuild}`;
+ }
// Last, and the one that binds every byte: the mirror, the tree, the
// manifest and both downloads, against the digest of what was audited.
if ((await sourceDigest(outDir)) !== state.contentDigest) {
diff --git a/common/publish/sourceHistory.test.ts b/common/publish/sourceHistory.test.ts
@@ -0,0 +1,467 @@
+import { test, after } from "node:test";
+import assert from "node:assert/strict";
+import { execFileSync, spawnSync } from "node:child_process";
+import {
+ chmodSync,
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ rmSync,
+ writeFileSync,
+} from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { buildThemeScript, HOMEPAGE_DEFAULT_BASE } from "../lib/themeConfig";
+import {
+ HISTORY_BACK_LINK,
+ HISTORY_TOKENS,
+ HistoryProblem,
+ SITE_ICON_HREF,
+ historyStylesheet,
+ holdHistoryCache,
+ homepageThemeScript,
+ renderHistory,
+ resolveStagit,
+ rewriteHistoryPage,
+ stageHistory,
+ stagitIdentity,
+ tokenBlocks,
+ type HistoryRun,
+ type RenderHistoryOpts,
+} from "./sourceHistory";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common test
+//
+// The history pages (sourceHistory.ts): the post-pass, the stylesheet from
+// the tokens, the binary's lookup, and the render with its cache — over a
+// fake stagit that writes what stagit writes, and over the real one when it
+// is installed (that test SKIPS without it; a gate says whether it ran).
+for (const key of ["GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_PREFIX"]) {
+ delete process.env[key];
+}
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const TOKENS = readFileSync(path.join(HERE, "..", "styles", "tokens.css"), "utf8");
+const TMP = mkdtempSync(path.join(os.tmpdir(), "source-history-"));
+after(() => rmSync(TMP, { recursive: true, force: true }));
+
+let n = 0;
+function dir(name: string): string {
+ const d = path.join(TMP, `${name}-${n++}`);
+ mkdirSync(d, { recursive: true });
+ return d;
+}
+
+// ── the post-pass ───────────────────────────────────────────────────────────
+
+// stagit's own markup, as it writes a commit page two levels down (relpath
+// "../"): the header, the diffstat's anchors, a diff header linking both
+// sides into file/, and a message that quotes an href as text.
+const COMMIT_PAGE = `<!DOCTYPE html>
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+<title>a subject - archilyzer - Archilyzer</title>
+<link rel="icon" type="image/png" href="../favicon.png" />
+<link rel="alternate" type="application/atom+xml" title="archilyzer.git Atom Feed" href="../atom.xml" />
+<link rel="stylesheet" type="text/css" href="../style.css" />
+</head>
+<body>
+<table><tr><td><a href="../../"><img src="../logo.png" alt="" width="32" height="32" /></a></td><td><h1>archilyzer</h1></td></tr><tr><td></td><td>
+<a href="../log.html">Log</a> | <a href="../files.html">Files</a> | <a href="../refs.html">Refs</a> | <a href="../file/README.md.html">README</a> | <a href="../file/LICENSE.html">LICENSE</a></td></tr></table>
+<hr/>
+<div id="content">
+<pre><b>commit</b> <a href="../commit/${"a".repeat(40)}.html">${"a".repeat(40)}</a>
+a message that says href="file/x.html" as text
+<b>diff --git a/<a id="h0" href="../file/app/%5Bslug%5D/page.tsx.html">app/[slug]/page.tsx</a> b/<a href="../file/app/%5Bslug%5D/page.tsx.html">app/[slug]/page.tsx</a></b>
+<a href="#h0-0-0" id="h0-0-0" class="i">+added
+</a></pre>
+</div>
+</body>
+</html>
+`;
+
+test("the post-pass: the theme script before </head>, the back link after <body>, file/ links to the raw tree, the logo and favicon to the site's icon — nothing else", () => {
+ const script = "var x=1;";
+ const out = rewriteHistoryPage(COMMIT_PAGE, script);
+ const expected = COMMIT_PAGE
+ .replace("</head>", `<script>${script}</script>\n</head>`)
+ .replace("<body>\n", `<body>\n${HISTORY_BACK_LINK}\n`)
+ .replace('href="../favicon.png"', `href="${SITE_ICON_HREF}"`)
+ .replace('src="../logo.png"', `src="${SITE_ICON_HREF}"`)
+ .replace('href="../file/README.md.html"', 'href="../../tree/README.md"')
+ .replace('href="../file/LICENSE.html"', 'href="../../tree/LICENSE"')
+ .replaceAll('href="../file/app/%5Bslug%5D/page.tsx.html"', 'href="../../tree/app/%5Bslug%5D/page.tsx"');
+ assert.equal(out, expected);
+ // Text that quotes an href is encoded by stagit, and left alone.
+ assert.match(out, /says href="file\/x\.html" as text/);
+ // The back link is the one line added to the body, and it is ASCII.
+ assert.equal(HISTORY_BACK_LINK, '<p class="archilyzer-source"><a href="/source/">Archilyzer · Source</a></p>');
+ assert.ok(!/[^\x00-\x7f]/.test(HISTORY_BACK_LINK));
+});
+
+test("the post-pass at the top level: files.html's links go to ../tree/<path>, as stagit encoded them", () => {
+ const files = `<html>\n<head>\n</head>\n<body>\n<tr><td>-rw-r--r--</td><td><a href="file/fonts/Archivo%5Bwdth%2Cwght%5D.ttf.html">fonts/Archivo[wdth,wght].ttf</a></td></tr>\n<a href="file/a.html.html">a.html</a>\n</body>\n</html>\n`;
+ const out = rewriteHistoryPage(files, "s()");
+ assert.match(out, /<a href="\.\.\/tree\/fonts\/Archivo%5Bwdth%2Cwght%5D\.ttf">/);
+ assert.match(out, /<a href="\.\.\/tree\/a\.html">a\.html<\/a>/, "a file that is itself .html keeps its name");
+ assert.ok(!out.includes('href="file/'));
+});
+
+test("the theme script is the homepage's own: its base, its string", () => {
+ assert.equal(HOMEPAGE_DEFAULT_BASE, "dark");
+ assert.equal(homepageThemeScript(), buildThemeScript({ defaultBase: HOMEPAGE_DEFAULT_BASE }));
+ assert.ok(!/[^\x00-\x7f]/.test(homepageThemeScript()), "ASCII, for the byte-for-byte rewrite");
+ // It must not be able to end its own element.
+ assert.throws(() => rewriteHistoryPage("<head></head>", "a</script><b>"));
+ // The homepage's layout passes the same constant to ThemeScript.
+ const layout = readFileSync(path.join(HERE, "..", "..", "homepage", "app", "layout.tsx"), "utf8");
+ assert.match(layout, /<ThemeScript defaultBase=\{HOMEPAGE_DEFAULT_BASE\} \/>/);
+});
+
+// ── the stylesheet ──────────────────────────────────────────────────────────
+
+test("style.css: the homepage's two grounds from tokens.css, by prefers-color-scheme and by data-base", () => {
+ const css = historyStylesheet(TOKENS);
+ const { light, dark } = tokenBlocks(TOKENS);
+ const block = (selector: RegExp) => {
+ const m = selector.exec(css);
+ assert.ok(m, String(selector));
+ return m[1];
+ };
+ const lightBlock = block(/:root,\nhtml\[data-base="light"\] \{\n([\s\S]*?)\n\}/);
+ const mediaBlock = block(/@media \(prefers-color-scheme: dark\) \{\n {2}:root:not\(\[data-base\]\) \{\n([\s\S]*?)\n {2}\}/);
+ const darkBlock = block(/\nhtml\[data-base="dark"\] \{\n([\s\S]*?)\n\}/);
+ for (const name of [...HISTORY_TOKENS, "color-scheme", "--swatch-signal"]) {
+ assert.ok(lightBlock.includes(` ${name}: ${light.get(name)};`), `light ${name}`);
+ assert.ok(darkBlock.includes(` ${name}: ${dark.get(name)};`), `dark ${name}`);
+ assert.ok(mediaBlock.includes(` ${name}: ${dark.get(name)};`), `no-JS dark ${name}`);
+ }
+ // Only what the rules read (and what that reaches through var()).
+ assert.ok(!lightBlock.includes("--chart-1"));
+ // Diff lines: insertions in --success, deletions in --destructive.
+ assert.match(css, /pre a\.i \{ color: var\(--success\); \}/);
+ assert.match(css, /pre a\.d \{ color: var\(--destructive\); \}/);
+ assert.match(css, /p\.archilyzer-source a \{/);
+});
+
+test("style.css follows the tokens: a changed value is the new value; a missing block or token is a HistoryProblem", () => {
+ const moved = TOKENS.replace(/(html\[data-base="dark"\] \{[\s\S]*?--background: )#[0-9a-f]+;/, "$1#010203;");
+ assert.notEqual(moved, TOKENS);
+ assert.match(historyStylesheet(moved), /html\[data-base="dark"\] \{[\s\S]*?--background: #010203;/);
+ assert.throws(() => historyStylesheet(":root { --background: #fff; }"), HistoryProblem);
+ const noInfo = TOKENS.replace(/--info: #[0-9a-f]+;/g, "");
+ assert.throws(() => historyStylesheet(noInfo), (e) => e instanceof HistoryProblem && /no --info/.test(e.message));
+});
+
+// ── the binary ──────────────────────────────────────────────────────────────
+
+function exe(file: string, body = "#!/bin/sh\nexit 0\n"): string {
+ mkdirSync(path.dirname(file), { recursive: true });
+ writeFileSync(file, body);
+ chmodSync(file, 0o755);
+ return file;
+}
+
+test("resolveStagit: STAGIT_BIN as a path, else stagit on PATH, else ~/.local/bin/stagit, else null", async () => {
+ const home = dir("home");
+ const bin = dir("bin");
+ const empty = dir("empty");
+ assert.equal(resolveStagit("stagit", { PATH: empty }, home), null);
+ const local = exe(path.join(home, ".local", "bin", "stagit"));
+ assert.equal(resolveStagit("stagit", { PATH: empty }, home), local);
+ const onPathBin = exe(path.join(bin, "stagit"));
+ assert.equal(resolveStagit("stagit", { PATH: `${empty}:${bin}` }, home), onPathBin, "PATH first");
+ const given = exe(path.join(dir("given"), "my-stagit"));
+ assert.equal(resolveStagit(given, { PATH: bin }, home), given);
+ assert.equal(resolveStagit(path.join(empty, "nope"), { PATH: bin }, home), null, "a path that is not there is nothing, not a PATH lookup");
+ writeFileSync(path.join(empty, "not-exec"), "x");
+ assert.equal(resolveStagit(path.join(empty, "not-exec"), { PATH: bin }, home), null);
+
+ assert.equal(await stagitIdentity(null), "absent");
+ const a = await stagitIdentity(given);
+ assert.match(a, /^stagit \(sha256 [0-9a-f]{12}\)$/);
+ exe(given, "#!/bin/sh\nexit 1\n");
+ assert.notEqual(await stagitIdentity(given), a, "another binary, another identity");
+ assert.ok(!a.includes(given), "never the path");
+});
+
+// ── the render, over a fake stagit ──────────────────────────────────────────
+
+function gitIn(cwd: string, ...args: string[]): string {
+ return execFileSync("git", args, { cwd, stdio: "pipe" }).toString().trim();
+}
+
+// A bare repository named archilyzer.git with `count` commits on main.
+function bareRepo(count: number): { gitDir: string; work: string } {
+ const work = dir("work");
+ gitIn(work, "init", "-q", "-b", "main");
+ gitIn(work, "config", "user.name", "history test");
+ gitIn(work, "config", "user.email", "history@example.invalid");
+ gitIn(work, "config", "commit.gpgsign", "false");
+ for (let i = 0; i < count; i++) addCommit(work, i);
+ const gitDir = path.join(dir("bare"), "archilyzer.git");
+ gitIn(TMP, "clone", "-q", "--bare", work, gitDir);
+ return { gitDir, work };
+}
+
+function addCommit(work: string, i: number): void {
+ writeFileSync(path.join(work, "README.md"), `hello ${i}\n`);
+ mkdirSync(path.join(work, "app", "[slug]"), { recursive: true });
+ writeFileSync(path.join(work, "app", "[slug]", "page.tsx"), `export default ${i};\n`);
+ gitIn(work, "add", "-A");
+ gitIn(work, "commit", "-q", "-m", `commit ${i}`);
+}
+
+// What stagit writes, in the directory it runs in: a page per commit (kept
+// when there already is one, as stagit keeps it), the log, the files index,
+// the refs, the feeds, and a per-file page. `-c` makes it write the cache
+// file (the head on its first line). Every call is logged to `log`.
+function fakeStagit(log: string, o: { exit?: number; extra?: string } = {}): string {
+ return exe(
+ path.join(dir("fake"), "stagit"),
+ `#!/bin/sh
+cache=""; base=""; repo=""
+while [ $# -gt 0 ]; do
+ case "$1" in
+ -c) cache="$2"; shift 2;;
+ -u) base="$2"; shift 2;;
+ *) repo="$1"; shift;;
+ esac
+done
+echo "cache=$cache base=$base repo=$(basename "$repo")" >> '${log}'
+${o.exit ? `echo 'stagit: something broke' >&2; exit ${o.exit}` : ""}
+mkdir -p commit file/app
+for c in $(git --git-dir "$repo" rev-list HEAD); do
+ if [ ! -f "commit/$c.html" ]; then
+ printf '<!DOCTYPE html>\\n<html>\\n<head>\\n<link rel="icon" type="image/png" href="../favicon.png" />\\n</head>\\n<body>\\n<a href="../file/README.md.html">README</a> ${o.extra ?? ""}\\n</body>\\n</html>\\n' > "commit/$c.html"
+ fi
+done
+printf '<!DOCTYPE html>\\n<html>\\n<head>\\n</head>\\n<body>\\n<span class="desc">%s</span> %s <a href="file/README.md.html">README</a> <img src="logo.png" />\\n</body>\\n</html>\\n' "$(cat "$repo/description")" "$(cat "$repo/url")" > log.html
+printf '<html>\\n<head>\\n</head>\\n<body>\\n<a href="file/app/%%5Bslug%%5D/page.tsx.html">x</a>\\n</body>\\n</html>\\n' > files.html
+printf 'refs\\n' > refs.html
+printf '<feed>%s file/README.md.html</feed>\\n' "$base" > atom.xml
+printf '<feed/>\\n' > tags.xml
+printf 'x\\n' > file/README.md.html
+touch cache.XXXXleftover
+if [ -n "$cache" ]; then git --git-dir "$repo" rev-parse HEAD > "$cache"; fi
+exit 0
+`,
+ );
+}
+
+const run: HistoryRun = async (command, args, o) => {
+ const p = spawnSync(command, args, { cwd: o.cwd, encoding: "utf8" });
+ return { code: p.status ?? 1, out: `${p.stdout}${p.stderr}`.split("\n") };
+};
+
+function renderOpts(gitDir: string, stagit: string, extra: Partial<RenderHistoryOpts> = {}): RenderHistoryOpts & { logs: string[] } {
+ const logs: string[] = [];
+ const head = gitIn(TMP, "--git-dir", gitDir, "rev-parse", "main");
+ const commits = Number(gitIn(TMP, "--git-dir", gitDir, "rev-list", "--count", "main"));
+ return {
+ stagit,
+ gitDir,
+ head,
+ commits,
+ dest: path.join(dir("stage"), "source", "git"),
+ scratch: dir("scratch"),
+ cacheDir: null,
+ cacheKey: "k1",
+ fresh: false,
+ description: "Archilyzer",
+ cloneUrl: "https://archilyzer.pages.dev/source/archilyzer.git",
+ baseUrl: "https://archilyzer.pages.dev/source/git/",
+ stylesheet: "/* css */\n",
+ themeScript: "t()",
+ run,
+ onLog: (l) => logs.push(l),
+ logs,
+ ...extra,
+ };
+}
+
+const calls = (log: string) => (existsSync(log) ? readFileSync(log, "utf8").trim().split("\n") : []);
+
+test("render: the allowlist of stagit's output, each page through the post-pass, the feeds as they are, style.css; never file/", async () => {
+ const { gitDir } = bareRepo(3);
+ const log = path.join(dir("log"), "calls");
+ const o = renderOpts(gitDir, fakeStagit(log));
+ const r = await renderHistory(o);
+ assert.deepEqual(calls(log), [`cache= base=${o.baseUrl} repo=archilyzer.git`]);
+ assert.equal(r.files, 3 + 5 + 1);
+ assert.equal(r.rendered, 3);
+ assert.equal(r.cached, false);
+ assert.deepEqual(readdirSync(o.dest).sort(), ["atom.xml", "commit", "files.html", "log.html", "refs.html", "style.css", "tags.xml"]);
+ assert.equal(readdirSync(path.join(o.dest, "commit")).length, 3);
+ const logHtml = readFileSync(path.join(o.dest, "log.html"), "utf8");
+ // stagit's header reads the description and the clone URL from the repo.
+ assert.match(logHtml, /<span class="desc">Archilyzer<\/span> https:\/\/archilyzer\.pages\.dev\/source\/archilyzer\.git /);
+ assert.ok(logHtml.includes(`<body>\n${HISTORY_BACK_LINK}\n`));
+ assert.ok(logHtml.includes("<script>t()</script>\n</head>"));
+ assert.match(logHtml, /href="\.\.\/tree\/README\.md"/);
+ assert.match(logHtml, new RegExp(`src="${SITE_ICON_HREF}"`));
+ assert.match(readFileSync(path.join(o.dest, "files.html"), "utf8"), /href="\.\.\/tree\/app\/%5Bslug%5D\/page\.tsx"/);
+ // The feeds are not pages: not one byte changes.
+ assert.equal(readFileSync(path.join(o.dest, "atom.xml"), "utf8"), `<feed>${o.baseUrl} file/README.md.html</feed>\n`);
+ assert.equal(readFileSync(path.join(o.dest, "style.css"), "utf8"), "/* css */\n");
+ // No cache: the render ran in the publish's own scratch.
+ assert.ok(existsSync(path.join(o.scratch, "history", "log.html")));
+ assert.ok(!existsSync(path.join(o.scratch, "history", "file")), "the per-file pages are dropped where they were rendered");
+});
+
+test("render: the page bytes are kept (a diff of a file that is not UTF-8), only the ASCII injections added", async () => {
+ const work = dir("bytes");
+ mkdirSync(path.join(work, "commit"));
+ const odd = Buffer.concat([Buffer.from("<html>\n<head>\n</head>\n<body>\n"), Buffer.from([0xff, 0xfe, 0x80]), Buffer.from("\n</body>\n</html>\n")]);
+ writeFileSync(path.join(work, "log.html"), odd);
+ for (const f of ["files.html", "refs.html"]) writeFileSync(path.join(work, f), "<html>\n<head>\n</head>\n<body>\n</body>\n</html>\n");
+ for (const f of ["atom.xml", "tags.xml"]) writeFileSync(path.join(work, f), Buffer.from([0x3c, 0xff, 0x3e]));
+ const dest = path.join(dir("dest"), "git");
+ await stageHistory(work, dest, { themeScript: "t()", stylesheet: "" });
+ const out = readFileSync(path.join(dest, "log.html"));
+ assert.ok(out.includes(Buffer.from([0xff, 0xfe, 0x80])), "the odd bytes survive");
+ assert.ok(out.includes(Buffer.from(HISTORY_BACK_LINK)));
+ assert.deepEqual(readFileSync(path.join(dest, "atom.xml")), Buffer.from([0x3c, 0xff, 0x3e]));
+ await assert.rejects(stageHistory(work, dest, { themeScript: "é", stylesheet: "" }), /ASCII/);
+});
+
+test("render: a failing stagit, or one that leaves the wrong pages, is a HistoryProblem", async () => {
+ const { gitDir } = bareRepo(2);
+ const log = path.join(dir("log"), "calls");
+ await assert.rejects(
+ renderHistory(renderOpts(gitDir, fakeStagit(log, { exit: 3 }))),
+ (e) => e instanceof HistoryProblem && /stagit exited 3: stagit: something broke/.test(e.message),
+ );
+ await assert.rejects(
+ renderHistory(renderOpts(gitDir, fakeStagit(log), { commits: 5 })),
+ (e) => e instanceof HistoryProblem && /stagit left 2 commit pages for 5 commits/.test(e.message),
+ );
+});
+
+test("the cache: -c under the cache dir, incremental the next time, the key and the ancestry checked, busy or cut-off locks handled", async () => {
+ const { gitDir, work } = bareRepo(2);
+ const log = path.join(dir("log"), "calls");
+ const stagit = fakeStagit(log);
+ const cacheDir = path.join(dir("scratch-root"), "archilyzer-source-history");
+ const cacheFile = path.join(cacheDir, "stagit.cache");
+ const cacheArgs = (o: RenderHistoryOpts) => `cache=${cacheFile} base=${o.baseUrl} repo=archilyzer.git`;
+
+ // First: everything rendered, the cache kept (key, cache file, output) and
+ // unlocked; no per-file pages and no stagit leftovers are kept.
+ let o = renderOpts(gitDir, stagit, { cacheDir });
+ let r = await renderHistory(o);
+ assert.equal(r.cached, false);
+ assert.equal(r.rendered, 2);
+ assert.equal(calls(log).at(-1), cacheArgs(o));
+ assert.deepEqual(readdirSync(cacheDir).sort(), ["key.json", "out", "stagit.cache"]);
+ assert.ok(!existsSync(path.join(cacheDir, "out", "file")));
+ assert.ok(!existsSync(path.join(o.dest, "cache.XXXXleftover")), "only the allowlist is published");
+
+ // A new commit: the cache is used, and only the new page is rendered.
+ addCommit(work, 2);
+ gitIn(TMP, "--git-dir", gitDir, "fetch", "-q", work, "main:main");
+ o = renderOpts(gitDir, stagit, { cacheDir });
+ r = await renderHistory(o);
+ assert.equal(r.cached, true);
+ assert.equal(r.rendered, 1);
+ assert.equal(r.files, 3 + 5 + 1);
+
+ // Another key (other rules, another stagit…): every page rendered again,
+ // and a page the cache held that is not in history is gone.
+ const junk = path.join(cacheDir, "out", "commit", `${"f".repeat(40)}.html`);
+ writeFileSync(junk, "junk");
+ o = renderOpts(gitDir, stagit, { cacheDir, cacheKey: "k2" });
+ r = await renderHistory(o);
+ assert.equal(r.cached, false);
+ assert.equal(r.rendered, 3);
+ assert.ok(!existsSync(junk));
+ assert.ok(!existsSync(path.join(o.dest, "commit", `${"f".repeat(40)}.html`)));
+
+ // A cache whose commit is not an ancestor of the head (a rewritten main).
+ writeFileSync(cacheFile, `${"e".repeat(40)}\n`);
+ o = renderOpts(gitDir, stagit, { cacheDir, cacheKey: "k2" });
+ r = await renderHistory(o);
+ assert.equal(r.cached, false);
+
+ // A cache that looks sound but holds a page history does not: the render
+ // notices (pages ≠ commits) and renders every page again.
+ writeFileSync(junk, "junk");
+ o = renderOpts(gitDir, stagit, { cacheDir, cacheKey: "k2" });
+ r = await renderHistory(o);
+ assert.equal(r.cached, false);
+ assert.match(o.logs.join("\n"), /stagit left 4 commit pages for 3 commits from the cache; rendering every page again/);
+ assert.ok(!existsSync(junk));
+
+ // `fresh` (--force) renders every page, whatever the cache holds.
+ o = renderOpts(gitDir, stagit, { cacheDir, cacheKey: "k2", fresh: true });
+ r = await renderHistory(o);
+ assert.equal(r.cached, false);
+ assert.equal(r.rendered, 3);
+
+ // Busy: another live publish holds it (this process's pid stands in).
+ writeFileSync(path.join(cacheDir, "lock"), `${process.pid}\n`);
+ o = renderOpts(gitDir, stagit, { cacheDir, cacheKey: "k2" });
+ r = await renderHistory(o);
+ assert.equal(r.cached, false);
+ assert.equal(calls(log).at(-1), `cache= base=${o.baseUrl} repo=archilyzer.git`, "rendered without -c");
+ assert.match(o.logs.join("\n"), /the render cache is in use by another publish; rendering without it/);
+ assert.ok(existsSync(path.join(cacheDir, "lock")), "the other holder's lock is left alone");
+
+ // Cut off: the holder is gone, so nothing in the cache is trusted.
+ const dead = spawnSync("sh", ["-c", "echo $$"], { encoding: "utf8" }).stdout.trim();
+ writeFileSync(path.join(cacheDir, "lock"), `${dead}\n`);
+ writeFileSync(junk, "junk");
+ assert.equal(await holdHistoryCache(cacheDir), true);
+ assert.ok(!existsSync(junk), "emptied");
+ assert.ok(!existsSync(path.join(cacheDir, "key.json")));
+ rmSync(path.join(cacheDir, "lock"));
+
+ // A render that fails empties the cache and releases it.
+ o = renderOpts(gitDir, fakeStagit(log, { exit: 2 }), { cacheDir, cacheKey: "k2" });
+ await assert.rejects(renderHistory(o), HistoryProblem);
+ assert.deepEqual(readdirSync(cacheDir), []);
+});
+
+// ── the real stagit ─────────────────────────────────────────────────────────
+
+test("the real stagit: a page per commit, every link into file/ now into the tree, every relative link resolves", async (t) => {
+ const stagit = resolveStagit(process.env.STAGIT_BIN ?? "stagit", process.env);
+ if (!stagit) return t.skip("stagit is not installed (STAGIT_BIN, PATH, ~/.local/bin)");
+ const { gitDir } = bareRepo(3);
+ const o = renderOpts(gitDir, stagit, { cacheDir: path.join(dir("real-root"), "archilyzer-source-history") });
+ const r = await renderHistory(o);
+ assert.equal(r.rendered, 3);
+ assert.equal(readdirSync(path.join(o.dest, "commit")).length, 3);
+ assert.ok(!existsSync(path.join(o.dest, "file")));
+ for (const page of ["log.html", "files.html", "refs.html", ...readdirSync(path.join(o.dest, "commit")).map((f) => `commit/${f}`)]) {
+ const html = readFileSync(path.join(o.dest, page), "utf8");
+ assert.ok(html.includes(`<body>\n${HISTORY_BACK_LINK}\n`), page);
+ assert.ok(html.includes("<script>t()</script>\n</head>"), page);
+ assert.ok(!/href="(?:\.\.\/)*file\//.test(html), `${page} links into file/`);
+ assert.ok(!/(?:logo|favicon)\.png"/.test(html), `${page} names stagit's images`);
+ for (const m of html.matchAll(/(?:href|src)="([^"#:]+)"/g)) {
+ const target = m[1];
+ if (target.startsWith("/")) continue; // the site's own: /source/, the icon
+ const resolved = path.posix
+ .normalize(path.posix.join(path.posix.dirname(`source/git/${page}`), target))
+ .replace(/\/$/, "");
+ if (resolved.startsWith("source/tree/")) {
+ // Into the raw tree: a file of main.
+ assert.ok(["source/tree/README.md", "source/tree/app/%5Bslug%5D/page.tsx"].includes(resolved), `${page}: ${target}`);
+ } else if (resolved.startsWith("source/git/")) {
+ assert.ok(existsSync(path.join(path.dirname(path.dirname(o.dest)), resolved)), `${page}: ${target} → ${resolved}`);
+ } else {
+ assert.equal(resolved, "source", `${page}: ${target}`); // the logo's link, to /source/
+ }
+ }
+ }
+ const logHtml = readFileSync(path.join(o.dest, "log.html"), "utf8");
+ assert.match(logHtml, /<title>Log - archilyzer - Archilyzer<\/title>/);
+ assert.match(logHtml, /git clone <a href="https:\/\/archilyzer\.pages\.dev\/source\/archilyzer\.git">/);
+ assert.match(readFileSync(path.join(o.dest, "atom.xml"), "utf8"), /href="https:\/\/archilyzer\.pages\.dev\/source\/git\/commit\/[0-9a-f]{40}\.html"/);
+});
diff --git a/common/publish/sourceHistory.ts b/common/publish/sourceHistory.ts
@@ -0,0 +1,573 @@
+// The source's history pages: stagit's rendering of the scrubbed mirror,
+// published at /source/git/ by `archilyzer source publish` (source.ts), which
+// runs this after the mirror is built and its objects audited, and before the
+// staged files are audited — so every page goes through the same gate as the
+// mirror, the tree and the tarball.
+//
+// stagit (codemadness.org, C over libgit2) is an OPERATOR-INSTALLED tool, like
+// git-filter-repo: never vendored, never committed. It is found as STAGIT_BIN,
+// else `stagit` on PATH, else ~/.local/bin/stagit (resolveStagit). WITHOUT IT
+// THE PUBLISH GOES ON: one log line says so and how to install it, and the
+// source is published without /source/git/ (the /source/ page then shows no
+// History links). A render that fails, or pages over the host's limits, are
+// the same: a WARNING, and no history — never a failed build.
+//
+// WHAT IS PUBLISHED, from an allowlist of what stagit writes:
+// log.html, files.html, refs.html, atom.xml, tags.xml, commit/<sha>.html
+// and style.css, written here from common/styles/tokens.css. NOT stagit's
+// per-file pages (file/…): browsing is the raw tree at /source/tree/, so
+// every link into file/ is rewritten to it and file/ is never staged.
+//
+// THE POST-PASS (rewriteHistoryPage) touches every .html page and never the
+// two feeds. It adds exactly two things — the homepage's pre-paint theme
+// script (lib/themeConfig.ts buildThemeScript, the string the homepage's
+// ThemeScript emits), and one line at the top linking back to /source/ — and
+// points two kinds of stagit link somewhere that exists: `…file/<path>.html`
+// at the raw tree, and its logo.png / favicon.png at the site's own icon.
+// stagit's header (the name, the description, the clone line, Log | Files |
+// Refs) stays as stagit writes it.
+//
+// THE CACHE. stagit's `-c <cachefile>` renders incrementally: it walks from
+// HEAD to the commit the cache names and keeps every commit page already in
+// its output directory. So the cache is a DIRECTORY kept between publishes —
+// `<ARCHILYZER_SOURCE_SCRATCH>/archilyzer-source-history/` (outside the
+// checkout and the public dir; the scratch root's own check) holding the
+// cache file, stagit's output and a key. It is trusted only when the key
+// matches (the scrub rules and step, filter-repo, stagit, the header text),
+// the commit it names is an ancestor of today's head, and the last run
+// finished (the key is removed before a render and written after it). One
+// publish holds it at a time (`lock`, the holder's pid); a second renders
+// without it. `--force` renders it afresh, `--check` never touches it, and a
+// refusal removes it.
+
+import { createHash } from "node:crypto";
+import { accessSync, constants, existsSync, statSync } from "node:fs";
+import { cp, mkdir, readdir, readFile, realpath, rm, stat, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { PROJECT_NAME } from "../lib/project";
+import { buildThemeScript, HOMEPAGE_DEFAULT_BASE } from "../lib/themeConfig";
+import { onPath } from "./sourceAudit";
+
+/** How to install stagit, as the log line and the doctor say it. */
+export const STAGIT_INSTALL =
+ "git clone git://git.codemadness.org/stagit && make -C stagit && cp stagit/stagit ~/.local/bin/";
+
+/** The cache directory's name under the scratch root. */
+export const HISTORY_CACHE_NAME = "archilyzer-source-history";
+
+/** The site's own 32 px icon, which stagit's logo.png and favicon.png become. */
+export const SITE_ICON_HREF = "/icons/icon-32.png";
+
+/**
+ * The one line put at the top of every page. ASCII only (`·`): the
+ * pages are rewritten byte for byte (latin1 in, latin1 out), so anything
+ * injected must be the same bytes in every encoding.
+ */
+export const HISTORY_BACK_LINK = `<p class="archilyzer-source"><a href="/source/">${PROJECT_NAME} · Source</a></p>`;
+
+/** The pre-paint script the homepage emits (homepage/app/layout.tsx). */
+export function homepageThemeScript(): string {
+ return buildThemeScript({ defaultBase: HOMEPAGE_DEFAULT_BASE });
+}
+
+// What stagit writes that is published: the top-level pages and feeds, and
+// one page per commit.
+const TOP_FILES = ["log.html", "files.html", "refs.html", "atom.xml", "tags.xml"] as const;
+const COMMIT_PAGE = /^[0-9a-f]{40}\.html$/;
+
+/** Something about the history went wrong: a WARNING, and no history. */
+export class HistoryProblem extends Error {
+ constructor(message: string) {
+ super(message);
+ this.name = "HistoryProblem";
+ }
+}
+
+// ── the binary ──────────────────────────────────────────────────────────────
+
+function isExecutableFile(p: string): boolean {
+ try {
+ accessSync(p, constants.X_OK);
+ return statSync(p).isFile();
+ } catch {
+ return false;
+ }
+}
+
+/**
+ * The stagit binary to run, or null. `bin` is paths.stagitBin: STAGIT_BIN,
+ * else "stagit". A name with a slash is that file, or nothing. A bare name is
+ * looked up on PATH, then in ~/.local/bin (the editor's process may not have
+ * it on its PATH, where a hand-built tool is usually put).
+ */
+export function resolveStagit(bin: string, env: NodeJS.ProcessEnv, homeDir: string = os.homedir()): string | null {
+ if (bin.includes("/")) return isExecutableFile(bin) ? bin : null;
+ const found = onPath(bin, env.PATH);
+ if (found) return found;
+ const local = path.join(homeDir, ".local", "bin", bin);
+ return isExecutableFile(local) ? local : null;
+}
+
+/**
+ * Which stagit renders: "absent", else the first 12 hex of its binary's
+ * sha256 (stagit has no version flag). Part of the publish's skip key and of
+ * the cache's, and the manifest's `history.tool`. Never the path: the
+ * manifest is published, and a path under the home dir is a denied literal.
+ */
+export async function stagitIdentity(found: string | null): Promise<string> {
+ if (!found) return "absent";
+ const bytes = await readFile(await realpath(found));
+ return `stagit (sha256 ${createHash("sha256").update(bytes).digest("hex").slice(0, 12)})`;
+}
+
+// ── the stylesheet ──────────────────────────────────────────────────────────
+
+// The tokens the stylesheet reads, on each base. `--brand` is Signal on both
+// (the homepage's accent: the base blocks default to it), and every token a
+// value names through var() comes along.
+export const HISTORY_TOKENS = [
+ "--background",
+ "--surface",
+ "--foreground",
+ "--muted-foreground",
+ "--faint",
+ "--border-strong",
+ "--brand",
+ "--brand-soft",
+ "--info",
+ "--success",
+ "--destructive",
+] as const;
+
+type Decls = Map<string, string>;
+
+function declarations(body: string): Decls {
+ const out: Decls = new Map();
+ for (const part of body.split(";")) {
+ const i = part.indexOf(":");
+ if (i === -1) continue;
+ const name = part.slice(0, i).trim();
+ if (name.startsWith("--") || name === "color-scheme") out.set(name, part.slice(i + 1).trim());
+ }
+ return out;
+}
+
+/**
+ * The light and the dark base blocks of tokens.css, as declarations: the rule
+ * whose selector list holds `html[data-base="light"]`, and the one holding
+ * `html[data-base="dark"]`. A file without both is an error.
+ */
+export function tokenBlocks(tokensCss: string): { light: Decls; dark: Decls } {
+ const text = tokensCss.replace(/\/\*[\s\S]*?\*\//g, "");
+ let light: Decls | null = null;
+ let dark: Decls | null = null;
+ for (const m of text.matchAll(/([^{}]+)\{([^{}]*)\}/g)) {
+ const selectors = m[1].split(",").map((s) => s.trim());
+ if (selectors.includes('html[data-base="light"]')) light = declarations(m[2]);
+ else if (selectors.includes('html[data-base="dark"]')) dark = declarations(m[2]);
+ }
+ if (!light || !dark) throw new HistoryProblem("tokens.css has no light or no dark base block");
+ return { light, dark };
+}
+
+// `names` and every custom property their values reach through var(), in
+// the block's own order, as ` name: value;` lines.
+function closure(block: Decls, names: readonly string[], base: string): string {
+ const want = new Set<string>(["color-scheme"]);
+ const visit = (name: string) => {
+ if (want.has(name)) return;
+ const value = block.get(name);
+ if (value === undefined) throw new HistoryProblem(`tokens.css's ${base} block has no ${name}`);
+ want.add(name);
+ for (const ref of value.matchAll(/var\(\s*(--[\w-]+)/g)) visit(ref[1]);
+ };
+ for (const n of names) visit(n);
+ return [...block].filter(([k]) => want.has(k)).map(([k, v]) => ` ${k}: ${v};`).join("\n");
+}
+
+/**
+ * style.css for the history pages, from common/styles/tokens.css: the
+ * homepage's two grounds. Without the theme script (no JS) the pages follow
+ * `prefers-color-scheme`; with it, `html[data-base]` is the visitor's stored
+ * choice, or the homepage's default. The rules are stagit's own stylesheet,
+ * recoloured: links in the accent, diff insertions in --success and deletions
+ * in --destructive (each line also keeps its + or − sign).
+ */
+export function historyStylesheet(tokensCss: string): string {
+ const { light, dark } = tokenBlocks(tokensCss);
+ const lightVars = closure(light, HISTORY_TOKENS, "light");
+ const darkVars = closure(dark, HISTORY_TOKENS, "dark");
+ const indent = (s: string) => s.split("\n").map((l) => ` ${l}`).join("\n");
+ return `/* The source's history pages (stagit), styled by \`archilyzer source publish\`
+ from common/styles/tokens.css — the homepage's two grounds. Generated. */
+:root,
+html[data-base="light"] {
+${lightVars}
+}
+@media (prefers-color-scheme: dark) {
+ :root:not([data-base]) {
+${indent(darkVars)}
+ }
+}
+html[data-base="dark"] {
+${darkVars}
+}
+
+html { background: var(--background); }
+body {
+ margin: 0;
+ padding: 1rem;
+ background: var(--background);
+ color: var(--foreground);
+ font-family: ui-monospace, "IBM Plex Mono", SFMono-Regular, Menlo, Consolas, "Liberation Mono", monospace;
+ font-size: 0.875rem;
+ line-height: 1.5;
+}
+a { color: var(--brand); }
+a:hover { color: var(--foreground); }
+p.archilyzer-source {
+ margin: 0 0 1rem;
+ font-family: system-ui, -apple-system, "Segoe UI", sans-serif;
+ font-size: 0.8125rem;
+}
+p.archilyzer-source a { color: var(--muted-foreground); text-decoration: none; }
+p.archilyzer-source a:hover { color: var(--foreground); text-decoration: underline; }
+h1, h2, h3, h4, h5, h6 { font-size: 1em; margin: 0; }
+img, h1, h2 { vertical-align: middle; }
+img { border: 0; }
+a:target { background-color: var(--brand-soft); }
+a.d, a.h, a.i, a.line { text-decoration: none; }
+#blob a { color: var(--faint); }
+#blob a:hover { color: var(--brand); text-decoration: none; }
+table thead td { font-weight: bold; }
+table td { padding: 0 0.4em; }
+#content { overflow-x: auto; }
+#content table td { vertical-align: top; white-space: nowrap; }
+#branches tr:hover td,
+#tags tr:hover td,
+#index tr:hover td,
+#log tr:hover td,
+#files tr:hover td { background-color: var(--surface); }
+#index tr td:nth-child(2),
+#tags tr td:nth-child(3),
+#branches tr td:nth-child(3),
+#log tr td:nth-child(2) { white-space: normal; }
+td.num { text-align: right; }
+.desc { color: var(--muted-foreground); }
+hr { border: 0; border-top: 1px solid var(--border-strong); height: 1px; }
+pre { font-family: inherit; }
+pre a.h { color: var(--info); }
+.A,
+span.i,
+pre a.i { color: var(--success); }
+.D,
+span.d,
+pre a.d { color: var(--destructive); }
+pre a.h:hover,
+pre a.i:hover,
+pre a.d:hover { text-decoration: none; }
+`;
+}
+
+// ── the post-pass ───────────────────────────────────────────────────────────
+
+/**
+ * One stagit page, as published. Adds the theme script before `</head>` and
+ * HISTORY_BACK_LINK after `<body>`; points every `href="…file/<path>.html"`
+ * (the Files index, the header's README and LICENSE, a diff's file names) at
+ * the raw tree — `../tree/<path>` from the same depth, the path as stagit
+ * encoded it — and stagit's logo.png and favicon.png at the site's icon.
+ * Nothing else changes. Page text cannot fake an `href="…"`: stagit encodes
+ * every `"` it prints from the repository as `"`.
+ */
+export function rewriteHistoryPage(html: string, themeScript: string): string {
+ if (/<\/script/i.test(themeScript)) throw new Error("the theme script may not close its own element");
+ let out = html;
+ const head = out.indexOf("</head>");
+ if (head !== -1) out = `${out.slice(0, head)}<script>${themeScript}</script>\n${out.slice(head)}`;
+ const body = /<body[^>]*>\n?/.exec(out);
+ if (body) {
+ const at = body.index + body[0].length;
+ out = `${out.slice(0, at)}${HISTORY_BACK_LINK}\n${out.slice(at)}`;
+ }
+ return out
+ .replace(/href="((?:\.\.\/)*)file\/([^"]*)\.html"/g, (_m, up: string, p: string) => `href="${up}../tree/${p}"`)
+ .replace(/src="(?:\.\.\/)*logo\.png"/g, `src="${SITE_ICON_HREF}"`)
+ .replace(/href="(?:\.\.\/)*favicon\.png"/g, `href="${SITE_ICON_HREF}"`);
+}
+
+/**
+ * Copy the allowlist of stagit's output from `work` into `dest`, each page
+ * through the post-pass (byte for byte otherwise: read and written as latin1,
+ * so a diff of a file that is not UTF-8 keeps its bytes), then style.css.
+ * The feeds are copied as they are.
+ */
+export async function stageHistory(
+ work: string,
+ dest: string,
+ o: { themeScript: string; stylesheet: string },
+): Promise<{ files: number; bytes: number; largest: { rel: string; bytes: number } }> {
+ if (/[^\x00-\x7f]/.test(o.themeScript)) throw new Error("the theme script must be ASCII");
+ const rels: string[] = [];
+ for (const f of TOP_FILES) {
+ if (!existsSync(path.join(work, f))) throw new HistoryProblem(`stagit wrote no ${f}`);
+ rels.push(f);
+ }
+ const commitDir = path.join(work, "commit");
+ for (const f of existsSync(commitDir) ? await readdir(commitDir) : []) {
+ if (COMMIT_PAGE.test(f)) rels.push(`commit/${f}`);
+ }
+ await mkdir(path.join(dest, "commit"), { recursive: true });
+ let bytes = 0;
+ let largest = { rel: "", bytes: -1 };
+ const note = (rel: string, n: number) => {
+ bytes += n;
+ if (n > largest.bytes) largest = { rel, bytes: n };
+ };
+ for (const rel of rels) {
+ const src = path.join(work, rel);
+ const dst = path.join(dest, rel);
+ if (rel.endsWith(".html")) {
+ const text = rewriteHistoryPage((await readFile(src)).toString("latin1"), o.themeScript);
+ const buf = Buffer.from(text, "latin1");
+ await writeFile(dst, buf);
+ note(rel, buf.length);
+ } else {
+ await cp(src, dst);
+ note(rel, (await stat(dst)).size);
+ }
+ }
+ await writeFile(path.join(dest, "style.css"), o.stylesheet);
+ note("style.css", Buffer.byteLength(o.stylesheet));
+ return { files: rels.length + 1, bytes, largest };
+}
+
+// ── the render, with its cache ──────────────────────────────────────────────
+
+export type HistoryRun = (
+ command: string,
+ args: string[],
+ o: { cwd: string; timeoutMs: number },
+) => Promise<{ code: number; out: string[] }>;
+
+export type RenderHistoryOpts = {
+ stagit: string;
+ // The scrubbed bare clone — a directory named archilyzer.git (stagit names
+ // the repository after it) — at `head`, with `commits` commits.
+ gitDir: string;
+ head: string;
+ commits: number;
+ // Where the published copy goes (stage/source/git).
+ dest: string;
+ // The publish's own scratch dir: the render's directory when there is no
+ // cache to use.
+ scratch: string;
+ // The kept cache directory, or null (a `--check`).
+ cacheDir: string | null;
+ // Changes whenever pages rendered before must not be kept.
+ cacheKey: string;
+ // `--force`: render every page again.
+ fresh: boolean;
+ // stagit's header: the description line and the clone URL.
+ description: string;
+ cloneUrl: string;
+ // The Atom feeds' absolute base (`<site>/source/git/`).
+ baseUrl: string;
+ stylesheet: string;
+ themeScript: string;
+ // A child, run with the publish's environment and cancel signal.
+ run: HistoryRun;
+ onLog: (line: string) => void;
+};
+
+export type RenderedHistory = {
+ files: number;
+ bytes: number;
+ largest: { rel: string; bytes: number };
+ // Pages stagit wrote this run (all of them without a usable cache).
+ rendered: number;
+ cached: boolean;
+};
+
+const pidAlive = (pid: number): boolean => {
+ try {
+ process.kill(pid, 0);
+ return true;
+ } catch (err) {
+ return (err as NodeJS.ErrnoException).code === "EPERM";
+ }
+};
+
+/**
+ * Hold the cache directory, or say it is busy. The lock names its holder's
+ * pid; a lock whose holder is gone means a render was cut off, so nothing in
+ * the directory is trusted: it is emptied and taken.
+ */
+export async function holdHistoryCache(dir: string): Promise<boolean> {
+ await mkdir(dir, { recursive: true, mode: 0o700 });
+ const lock = path.join(dir, "lock");
+ for (let attempt = 0; attempt < 2; attempt++) {
+ try {
+ await writeFile(lock, `${process.pid}\n`, { flag: "wx" });
+ return true;
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
+ const pid = Number((await readFile(lock, "utf8").catch(() => "")).trim());
+ if (Number.isInteger(pid) && pid > 0 && pidAlive(pid)) return false;
+ await emptyDir(dir);
+ }
+ }
+ return false;
+}
+
+async function emptyDir(dir: string): Promise<void> {
+ for (const f of await readdir(dir).catch(() => [] as string[])) {
+ await rm(path.join(dir, f), { recursive: true, force: true });
+ }
+}
+
+async function releaseHistoryCache(dir: string): Promise<void> {
+ await rm(path.join(dir, "lock"), { force: true });
+}
+
+/**
+ * Remove the cache (a refusal: it was rendered under rules that may not be
+ * today's). Left alone while another publish holds it.
+ */
+export async function dropHistoryCache(dir: string): Promise<void> {
+ if (!existsSync(dir)) return;
+ if (await holdHistoryCache(dir)) await rm(dir, { recursive: true, force: true });
+}
+
+async function countCommitPages(work: string): Promise<number> {
+ const names = await readdir(path.join(work, "commit")).catch(() => [] as string[]);
+ return names.filter((f) => COMMIT_PAGE.test(f)).length;
+}
+
+/**
+ * Render the history into `o.dest`. Throws HistoryProblem when stagit fails
+ * or its output is not what it should be; the caller publishes without the
+ * history then.
+ */
+export async function renderHistory(o: RenderHistoryOpts): Promise<RenderedHistory> {
+ // stagit's header reads both from the repository directory. Neither is
+ // published as a file (the mirror is staged from an allowlist). No newline
+ // after the description: stagit keeps it, in the <title> and the header.
+ await writeFile(path.join(o.gitDir, "description"), o.description);
+ await writeFile(path.join(o.gitDir, "url"), `${o.cloneUrl}\n`);
+
+ const held = o.cacheDir ? await holdHistoryCache(o.cacheDir) : false;
+ if (o.cacheDir && !held) o.onLog("[source] history: the render cache is in use by another publish; rendering without it");
+ try {
+ if (!held || !o.cacheDir) {
+ const work = path.join(o.scratch, "history");
+ return { ...(await renderOnce(o, work, null)), cached: false };
+ }
+ const dir = o.cacheDir;
+ const keyFile = path.join(dir, "key.json");
+ const cacheFile = path.join(dir, "stagit.cache");
+ const work = path.join(dir, "out");
+ let usable = !o.fresh && (await cacheUsable(o, dir, keyFile, cacheFile, work));
+ if (!usable) await emptyCache(dir);
+ // The key goes before the render and comes back after it: a render cut
+ // off leaves none, and the next publish starts over.
+ await rm(keyFile, { force: true });
+ try {
+ const r = await renderOnce(o, work, cacheFile);
+ await writeFile(keyFile, JSON.stringify({ key: o.cacheKey }) + "\n");
+ return { ...r, cached: usable };
+ } catch (err) {
+ if (!(err instanceof HistoryProblem) || !usable) throw err;
+ // A cache that looked sound and did not render: once more from nothing.
+ o.onLog(`[source] history: ${err.message} from the cache; rendering every page again`);
+ usable = false;
+ await emptyCache(dir);
+ const r = await renderOnce(o, work, cacheFile);
+ await writeFile(keyFile, JSON.stringify({ key: o.cacheKey }) + "\n");
+ return { ...r, cached: false };
+ }
+ } catch (err) {
+ if (held && o.cacheDir) await emptyCache(o.cacheDir);
+ throw err;
+ } finally {
+ if (held && o.cacheDir) await releaseHistoryCache(o.cacheDir);
+ }
+}
+
+// Everything in the cache but its lock.
+async function emptyCache(dir: string): Promise<void> {
+ for (const f of await readdir(dir).catch(() => [] as string[])) {
+ if (f !== "lock") await rm(path.join(dir, f), { recursive: true, force: true });
+ }
+}
+
+async function cacheUsable(
+ o: RenderHistoryOpts,
+ dir: string,
+ keyFile: string,
+ cacheFile: string,
+ work: string,
+): Promise<boolean> {
+ let key: unknown = null;
+ try {
+ key = (JSON.parse(await readFile(keyFile, "utf8")) as { key?: unknown }).key;
+ } catch {
+ return false;
+ }
+ if (key !== o.cacheKey || !existsSync(path.join(work, "log.html"))) return false;
+ const last = (await readFile(cacheFile, "utf8").catch(() => "")).split("\n")[0].trim();
+ if (!/^[0-9a-f]{40}$/.test(last)) return false;
+ // History that was rewritten since (a force-push to the private main)
+ // leaves pages of commits that are no longer in it.
+ const r = await o.run("git", ["--git-dir", o.gitDir, "merge-base", "--is-ancestor", last, o.head], {
+ cwd: dir,
+ timeoutMs: 30_000,
+ });
+ return r.code === 0;
+}
+
+async function renderOnce(
+ o: RenderHistoryOpts,
+ work: string,
+ cacheFile: string | null,
+): Promise<Omit<RenderedHistory, "cached">> {
+ await mkdir(work, { recursive: true });
+ const before = await countCommitPages(work);
+ const args = [...(cacheFile ? ["-c", cacheFile] : []), "-u", o.baseUrl, o.gitDir];
+ const r = await o.run(o.stagit, args, { cwd: work, timeoutMs: 600_000 });
+ // The per-file pages are never published, and stagit writes them all again
+ // on every run: none is kept.
+ await rm(path.join(work, "file"), { recursive: true, force: true });
+ if (r.code !== 0) {
+ const tail = r.out.filter((l) => l.trim()).slice(-2).join(" / ");
+ throw new HistoryProblem(`stagit exited ${r.code}${tail ? `: ${tail}` : ""}`);
+ }
+ const pages = await countCommitPages(work);
+ if (pages !== o.commits) {
+ throw new HistoryProblem(`stagit left ${pages} commit pages for ${o.commits} commits`);
+ }
+ const staged = await stageHistory(work, o.dest, { themeScript: o.themeScript, stylesheet: o.stylesheet });
+ return { ...staged, rendered: cacheFile ? pages - before : pages };
+}
+
+/** The cache directory under a scratch root. */
+export function historyCacheDir(scratchRoot: string): string {
+ return path.join(scratchRoot, HISTORY_CACHE_NAME);
+}
+
+/**
+ * The cache's key: whatever, changed, makes a page rendered before wrong — the
+ * rules (with the step's version) and filter-repo that made the ids, the
+ * stagit that wrote the pages, and the header text every page carries.
+ */
+export function historyCacheKey(parts: {
+ rulesHash: string;
+ filterRepo: string;
+ stagit: string;
+ description: string;
+ cloneUrl: string;
+ baseUrl: string;
+}): string {
+ return createHash("sha256").update(JSON.stringify(parts)).digest("hex");
+}
