commit f9c8fce26e3c6fefbdc5ca32af77738de91f9b2d
parent 8cf542e1f31bf51986ff517c69ede358d06fa370
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Wed, 30 Sep 2026 09:34:05 -0400
common: the history is capped by commit count — past SOURCE_HISTORY_MAX_COMMITS (10,000) stagit runs with -l, whose log keeps the newest and says how many more; stagit refuses -c with -l and still writes a page for every commit, so the pages published are exactly the log's commits (loggedCommits), under the cap too; the manifest's history gains `total` (the latest N of M); the 15,000-file drop stays as the last resort. The render cache moves to ${XDG_CACHE_HOME:-~/.cache}/archilyzer/source-history (getPaths().sourceHistoryCacheDir, XDG_CACHE_HOME declared), made with a recursive mkdir, never inside the checkout or the public dir (historyCacheFor renders without it there, and says so); its pages are kept by key, its log lines only when they end at an ancestor. The fake stagit is one fixture (__fixtures__/fakeStagit.ts) with -c and -l as stagit.c has them
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
10 files changed, 446 insertions(+), 173 deletions(-)
diff --git a/ENVIRONMENT.md b/ENVIRONMENT.md
@@ -42,7 +42,8 @@ The one override surface for where things live and which binary runs. Every one
| `ARCHILYZER_CONFIG_DIR` | `~/.config/archilyzer` | The operator's private config dir, outside the repo: the two inputs of `archilyzer source publish` below. Never committed. | common/lib/paths.ts (getPaths) |
| `SOURCE_SCRUB_FILE` | `<ARCHILYZER_CONFIG_DIR>/source-scrub.txt` | git-filter-repo `lhs==>rhs` rules applied to file contents AND commit messages when the source mirror is generated (`<home dir>==>/home/user` is built in and runs first). Every rule's left side is also denied. See [PUBLISH.md](PUBLISH.md). | common/lib/paths.ts (getPaths) |
| `SOURCE_DENYLIST_FILE` | `<ARCHILYZER_CONFIG_DIR>/source-denylist.txt` | Literals the published source must never contain, one per line (`i:` = any case). One hit anywhere in the mirror, the tree or the tarball refuses the publish. | common/lib/paths.ts (getPaths) |
-| `ARCHILYZER_SOURCE_SCRATCH` | the OS temp dir | Where `source publish` makes its scratch clone and stage (removed afterwards unless `--keep-scratch`), and keeps the history pages' render cache (`archilyzer-source-history/`). | common/lib/paths.ts (getPaths) |
+| `ARCHILYZER_SOURCE_SCRATCH` | the OS temp dir | Where `source publish` makes its scratch clone and stage (removed afterwards unless `--keep-scratch`). | common/lib/paths.ts (getPaths) |
+| `XDG_CACHE_HOME` | `~/.cache` | The cache root: `source publish` keeps the history pages' render cache in `<it>/archilyzer/source-history/` (about 140 MB; never inside the checkout). | common/lib/paths.ts (getPaths) |
| `STAGIT_BIN` | `stagit` on PATH, then `~/.local/bin/stagit` | stagit, which renders the source's history pages (`/source/git/`: the log and a page per commit with its diff). Optional: without it the source is published without them. See [PUBLISH.md](PUBLISH.md). | common/lib/paths.ts (getPaths) |
## Runtime
diff --git a/common/lib/envVars.ts b/common/lib/envVars.ts
@@ -83,7 +83,8 @@ const DECLARED: EnvVarDecl[] = [
paths("ARCHILYZER_CONFIG_DIR", "`~/.config/archilyzer`", "The operator's private config dir, outside the repo: the two inputs of `archilyzer source publish` below. Never committed."),
paths("SOURCE_SCRUB_FILE", "`<ARCHILYZER_CONFIG_DIR>/source-scrub.txt`", "git-filter-repo `lhs==>rhs` rules applied to file contents AND commit messages when the source mirror is generated (`<home dir>==>/home/user` is built in and runs first). Every rule's left side is also denied. See [PUBLISH.md](PUBLISH.md)."),
paths("SOURCE_DENYLIST_FILE", "`<ARCHILYZER_CONFIG_DIR>/source-denylist.txt`", "Literals the published source must never contain, one per line (`i:` = any case). One hit anywhere in the mirror, the tree or the tarball refuses the publish."),
- paths("ARCHILYZER_SOURCE_SCRATCH", "the OS temp dir", "Where `source publish` makes its scratch clone and stage (removed afterwards unless `--keep-scratch`), and keeps the history pages' render cache (`archilyzer-source-history/`)."),
+ paths("ARCHILYZER_SOURCE_SCRATCH", "the OS temp dir", "Where `source publish` makes its scratch clone and stage (removed afterwards unless `--keep-scratch`)."),
+ paths("XDG_CACHE_HOME", "`~/.cache`", "The cache root: `source publish` keeps the history pages' render cache in `<it>/archilyzer/source-history/` (about 140 MB; never inside the checkout)."),
paths("STAGIT_BIN", "`stagit` on PATH, then `~/.local/bin/stagit`", "stagit, which renders the source's history pages (`/source/git/`: the log and a page per commit with its diff). Optional: without it the source is published without them. See [PUBLISH.md](PUBLISH.md)."),
// ── runtime ────────────────────────────────────────────────────────────
diff --git a/common/lib/paths.ts b/common/lib/paths.ts
@@ -168,6 +168,11 @@ export type Paths = {
sourceDenylistFile: string;
// Where `source publish` makes its scratch clone (removed afterwards).
sourceScratchDir: string;
+ // The history pages' render cache (publish/sourceHistory.ts): the XDG
+ // cache dir's archilyzer/source-history — ~/.cache unless XDG_CACHE_HOME
+ // says otherwise (an empty one is unset, as the XDG spec has it). Never
+ // inside the checkout or the public dir: the step renders without it there.
+ sourceHistoryCacheDir: string;
// stagit, which renders the source's history pages (/source/git/). An
// operator-installed tool, never vendored: `stagit` on PATH, then
// ~/.local/bin/stagit (publish/sourceHistory.ts resolveStagit). Without it
@@ -301,6 +306,11 @@ export function getPaths(): Paths {
process.env.SOURCE_DENYLIST_FILE ??
path.join(configDir, "source-denylist.txt"),
sourceScratchDir: process.env.ARCHILYZER_SOURCE_SCRATCH ?? os.tmpdir(),
+ sourceHistoryCacheDir: path.join(
+ process.env.XDG_CACHE_HOME || path.join(os.homedir(), ".cache"),
+ "archilyzer",
+ "source-history",
+ ),
stagitBin: process.env.STAGIT_BIN ?? "stagit",
};
return cached;
diff --git a/common/lib/sourceManifest.test.ts b/common/lib/sourceManifest.test.ts
@@ -54,6 +54,7 @@ test("a wrong version, a bad id or sha, or any number the page reads that is not
const history = () => ({
href: "/source/git/log.html",
commits: 3,
+ total: 3,
head: "2".repeat(40),
files: 9,
bytes: 1234,
@@ -64,6 +65,8 @@ const history = () => ({
test("a manifest with a history block parses; without one it is the same manifest", () => {
const withHistory = { ...good(), history: history() };
assert.deepEqual(parseSourceManifest(withHistory), withHistory);
+ const capped = { ...good(), history: { ...history(), total: 12_345 } };
+ assert.deepEqual(parseSourceManifest(capped), capped, "the latest 3 of 12,345");
assert.equal(parseSourceManifest(good())?.history, undefined);
});
@@ -71,6 +74,8 @@ test("a history block that is present but malformed makes the manifest untrusted
const broken: Array<[string, (h: ReturnType<typeof history>) => unknown]> = [
["null", () => null],
["commits a string", (h) => ({ ...h, commits: "3" })],
+ ["no total", (h) => ({ ...h, total: undefined })],
+ ["fewer in all than with a page", (h) => ({ ...h, total: 2 })],
["files negative", (h) => ({ ...h, files: -1 })],
["bytes missing", (h) => ({ ...h, bytes: undefined })],
["short head", (h) => ({ ...h, head: "abc" })],
diff --git a/common/lib/sourceManifest.ts b/common/lib/sourceManifest.ts
@@ -88,9 +88,12 @@ export type SourceManifest = {
export type SourceHistory = {
// The log page (HISTORY_LOG_HREF).
href: string;
- // Every commit of the mirror's main has a page; `head` is the one the log
- // starts at (the manifest's mirrorHead).
+ // How many commits have a page (the newest ones: the log lists exactly
+ // these), of `total`, every commit of the mirror's main. Equal until main
+ // passes the cap (publish/sourceHistory.ts SOURCE_HISTORY_MAX_COMMITS).
+ // `head` is the one the log starts at (the manifest's mirrorHead).
commits: number;
+ total: number;
head: string;
// Every file under /source/git/ (the pages, the two feeds, style.css): how
// many, their bytes, and one sha256 over them — each file's path, size and
@@ -137,7 +140,8 @@ export function parseSourceManifest(value: unknown): SourceManifest | null {
if (m.history !== undefined) {
const h = m.history as Partial<SourceHistory> | null;
if (!obj(h)) return null;
- if (!num(h.commits) || !num(h.files) || !num(h.bytes)) return null;
+ if (!num(h.commits) || !num(h.total) || !num(h.files) || !num(h.bytes)) return null;
+ if ((h.total as number) < (h.commits as number)) return null;
if (typeof h.head !== "string" || !HEX40.test(h.head)) return null;
if (typeof h.sha256 !== "string" || !HEX64.test(h.sha256)) return null;
if (typeof h.href !== "string" || typeof h.tool !== "string") return null;
diff --git a/common/publish/__fixtures__/fakeStagit.ts b/common/publish/__fixtures__/fakeStagit.ts
@@ -0,0 +1,82 @@
+import { chmodSync, mkdirSync, writeFileSync } from "node:fs";
+import path from "node:path";
+
+// A stand-in for stagit, for the source step's tests (sourceHistory.test.ts,
+// source.test.ts; the real one is tested where it is installed). It writes
+// what stagit writes, where stagit writes it (its CWD), the way stagit.c does:
+// - `-c <file>`: walks from HEAD down to the commit the file names, writes a
+// log line and a page for each NEW commit, appends the file's old lines,
+// and rewrites the file (the head, then every line);
+// - `-l <n>`: a log line for the newest n and "<m> more commits remaining,
+// fetch the repository"; a page for EVERY commit that has none (-l does
+// not cap the pages);
+// - neither: a line and a page for every commit;
+// - `-c` with `-l`: stagit's usage error;
+// - and every run: files.html, refs.html, the two feeds (atom.xml carries
+// the -u base), a per-file page, a leftover in the CWD. A commit page is
+// never rewritten once it exists.
+// Each call is appended to `log` as `cache=… limit=… base=… repo=<dir name>`.
+// `exit` fails at once (with a stderr line); `extra` goes into every commit
+// page; `pad` bytes are added to the head's page.
+export function writeFakeStagit(
+ file: string,
+ log: string,
+ o: { exit?: number; extra?: string; pad?: number } = {},
+): string {
+ mkdirSync(path.dirname(file), { recursive: true });
+ writeFileSync(
+ file,
+ `#!/bin/sh
+cache=""; base=""; repo=""; limit=""
+while [ $# -gt 0 ]; do
+ case "$1" in
+ -c) cache="$2"; shift 2;;
+ -l) limit="$2"; shift 2;;
+ -u) base="$2"; shift 2;;
+ *) repo="$1"; shift;;
+ esac
+done
+echo "cache=$cache limit=$limit base=$base repo=$(basename "$repo")" >> '${log}'
+${o.exit ? `echo 'stagit: something broke' >&2; exit ${o.exit}` : ""}
+if [ -n "$cache" ] && [ -n "$limit" ]; then echo 'usage: stagit [-c cachefile | -l commits] [-u baseurl] repodir' >&2; exit 1; fi
+mkdir -p commit file
+tip=$(git --git-dir "$repo" rev-parse HEAD)
+last=""
+if [ -n "$cache" ] && [ -f "$cache" ]; then last=$(head -n 1 "$cache"); fi
+: > lines.tmp
+n=0; rem=0
+for c in $(git --git-dir "$repo" rev-list HEAD); do
+ [ "$c" = "$last" ] && break
+ if [ -n "$limit" ] && [ "$n" -ge "$limit" ]; then
+ rem=$((rem + 1))
+ [ -f "commit/$c.html" ] && continue
+ else
+ n=$((n + 1))
+ echo "<tr><td><a href=\\"commit/$c.html\\">$c</a></td></tr>" >> lines.tmp
+ fi
+ [ -f "commit/$c.html" ] || printf '<html>\\n<head>\\n</head>\\n<body>\\n<a href="../file/README.md.html">README</a> ${o.extra ?? ""}\\n</body>\\n</html>\\n' > "commit/$c.html"
+done
+if [ -n "$cache" ]; then
+ if [ -n "$last" ]; then tail -n +2 "$cache" >> lines.tmp; fi
+ { echo "$tip"; cat lines.tmp; } > "$cache"
+fi
+if [ "$rem" -gt 0 ]; then echo "<tr><td></td><td colspan=\\"5\\">$rem more commits remaining, fetch the repository</td></tr>" >> lines.tmp; fi
+{
+ printf '<html>\\n<head>\\n</head>\\n<body>\\n<span class="desc">%s</span> %s <a href="file/README.md.html">README</a> <img src="logo.png" />\\n<table id="log">\\n' "$(cat "$repo/description")" "$(cat "$repo/url")"
+ cat lines.tmp
+ printf '</table>\\n</body>\\n</html>\\n'
+} > log.html
+rm -f lines.tmp
+printf '<html>\\n<head>\\n</head>\\n<body>\\n<a href="file/app/%%5Bslug%%5D/page.tsx.html">x</a>\\n</body>\\n</html>\\n' > files.html
+printf '<html>\\n<head>\\n</head>\\n<body>\\n</body>\\n</html>\\n' > refs.html
+printf '<feed>%s file/README.md.html</feed>\\n' "$base" > atom.xml
+printf '<feed/>\\n' > tags.xml
+printf 'x\\n' > file/README.md.html
+touch cache.XXXXleftover
+${o.pad ? `head -c ${o.pad} /dev/zero >> "commit/$tip.html"` : ""}
+exit 0
+`,
+ );
+ chmodSync(file, 0o755);
+ return file;
+}
diff --git a/common/publish/source.test.ts b/common/publish/source.test.ts
@@ -34,6 +34,7 @@ import {
publishSource,
publishedSourceProblem,
gitleaksIdentity,
+ historyCacheFor,
historyDigest,
resolveFilterRepo,
rulesHashOf,
@@ -41,7 +42,8 @@ import {
sourceDigest,
type SourcePublishOpts,
} from "./source";
-import { HISTORY_BACK_LINK, historyCacheDir } from "./sourceHistory";
+import { writeFakeStagit } from "./__fixtures__/fakeStagit";
+import { HISTORY_BACK_LINK } from "./sourceHistory";
// Run with:
// pnpm --filter yt-dlp-transcript-common test
@@ -620,42 +622,15 @@ test("--no-source's clear: the manifest, mirror, tree, tarball and skip key go;
const TOKENS_FILE = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "styles", "tokens.css");
-// A page per commit (kept if there is one, as stagit keeps it), the top-level
-// pages and feeds, a per-file page, and the cache file under -c. `extra` goes
-// into every commit page; `pad` bytes are added to the head's page.
+// The fake (__fixtures__/fakeStagit.ts) writes what stagit writes, with -c
+// and -l as stagit.c has them.
function fakeStagit(log: string, o: { exit?: number; extra?: string; pad?: number } = {}): string {
- const file = path.join(dir("fake-stagit"), "stagit");
- writeFileSync(
- file,
- `#!/bin/sh
-cache=""; base=""; repo=""
-while [ $# -gt 0 ]; do
- case "$1" in
- -c) cache="$2"; shift 2;;
- -u) base="$2"; shift 2;;
- *) repo="$1"; shift;;
- esac
-done
-echo "cache=$cache base=$base repo=$(basename "$repo")" >> '${log}'
-${o.exit ? `echo 'stagit: something broke' >&2; exit ${o.exit}` : ""}
-mkdir -p commit file
-for c in $(git --git-dir "$repo" rev-list HEAD); do
- [ -f "commit/$c.html" ] || printf '<html>\\n<head>\\n</head>\\n<body>\\n<a href="../file/README.md.html">README</a> ${o.extra ?? ""}\\n</body>\\n</html>\\n' > "commit/$c.html"
-done
-${o.pad ? `head -c ${o.pad} /dev/zero >> "commit/$(git --git-dir "$repo" rev-parse HEAD).html"` : ""}
-printf '<html>\\n<head>\\n</head>\\n<body>\\n<span class="desc">%s</span> %s <a href="file/README.md.html">README</a>\\n</body>\\n</html>\\n' "$(cat "$repo/description")" "$(cat "$repo/url")" > log.html
-for f in files refs; do printf '<html>\\n<head>\\n</head>\\n<body>\\n</body>\\n</html>\\n' > $f.html; done
-printf '<feed>%s</feed>\\n' "$base" > atom.xml
-printf '<feed/>\\n' > tags.xml
-printf 'x\\n' > file/README.md.html
-if [ -n "$cache" ]; then git --git-dir "$repo" rev-parse HEAD > "$cache"; fi
-exit 0
-`,
- );
- chmodSync(file, 0o755);
- return file;
+ return writeFakeStagit(path.join(dir("fake-stagit"), "stagit"), log, o);
}
+// A render cache of its own, where XDG_CACHE_HOME would put it.
+const cacheDirIn = (root: string) => path.join(root, "archilyzer", "source-history");
+
const stagitCalls = (log: string) => (existsSync(log) ? readFileSync(log, "utf8").trim().split("\n") : []);
test("history: published with the source — the allowlist at /source/git/, the manifest's block, the skip key, incremental, and the deploy check names it", async () => {
@@ -663,8 +638,8 @@ test("history: published with the source — the allowlist at /source/git/, the
const files = operatorFiles("", "");
const logs: string[] = [];
const log = path.join(dir("stagit-log"), "calls");
- const scratchRoot = dir("scratch-root");
- const o = opts(repo, files, logs, { filterRepo: ["true"], stagit: fakeStagit(log), tokensFile: TOKENS_FILE, scratchRoot });
+ const cacheDir = cacheDirIn(dir("cache-home"));
+ const o = opts(repo, files, logs, { filterRepo: ["true"], stagit: fakeStagit(log), tokensFile: TOKENS_FILE, historyCacheDir: cacheDir });
const pub = o.publicDir!;
const site = path.dirname(pub);
@@ -673,13 +648,14 @@ test("history: published with the source — the allowlist at /source/git/, the
assert.match(text, /\[source\] history: stagit \(sha256 [0-9a-f]{12}\) — 3 commits, 3 pages rendered; 9 files, [\d.]+ MB, the largest git\/\S+ [\d.]+ MB/);
assert.match(text, /\[source\] published main .* tree \d+ dirs, history 3 commits in 9 files\)/);
assert.deepEqual(stagitCalls(log), [
- `cache=${path.join(historyCacheDir(scratchRoot), "stagit.cache")} base=https://archilyzer.pages.dev/source/git/ repo=${MIRROR_DIR}`,
+ `cache=${path.join(cacheDir, "stagit.cache")} limit= base=https://archilyzer.pages.dev/source/git/ repo=${MIRROR_DIR}`,
]);
const manifest = JSON.parse(readFileSync(path.join(pub, "source", "manifest.json"), "utf8"));
- assert.deepEqual(Object.keys(manifest.history).sort(), ["bytes", "commits", "files", "head", "href", "sha256", "tool"]);
+ assert.deepEqual(Object.keys(manifest.history).sort(), ["bytes", "commits", "files", "head", "href", "sha256", "tool", "total"]);
assert.equal(manifest.history.href, "/source/git/log.html");
assert.equal(manifest.history.commits, 3);
+ assert.equal(manifest.history.total, 3);
assert.equal(manifest.history.head, manifest.mirrorHead);
assert.equal(manifest.history.files, 9);
assert.equal(manifest.history.sha256, await historyDigest(pub));
@@ -693,7 +669,8 @@ test("history: published with the source — the allowlist at /source/git/, the
assert.ok(logHtml.includes(HISTORY_BACK_LINK));
assert.match(logHtml, /<script>\(function\(\)\{try\{var d=document\.documentElement/);
assert.match(logHtml, /href="\.\.\/tree\/README\.md"/);
- assert.equal(readFileSync(path.join(git, "atom.xml"), "utf8"), "<feed>https://archilyzer.pages.dev/source/git/</feed>\n");
+ // The feed is copied as it is: not even its file/ text is rewritten.
+ assert.equal(readFileSync(path.join(git, "atom.xml"), "utf8"), "<feed>https://archilyzer.pages.dev/source/git/ file/README.md.html</feed>\n");
assert.match(readFileSync(path.join(git, "style.css"), "utf8"), /html\[data-base="dark"\] \{/);
// The description and url files stagit read are never published.
for (const f of ["description", "url"]) assert.ok(!existsSync(path.join(pub, "source", MIRROR_DIR, f)), f);
@@ -740,12 +717,12 @@ test("history: published with the source — the allowlist at /source/git/, the
assert.match(logs.join("\n"), /— 4 commits, 4 pages rendered; 10 files/);
// --check writes nothing — the cache included — and still renders (and
// sweeps) the pages in its own scratch.
- const cacheBefore = readFileSync(path.join(historyCacheDir(scratchRoot), "key.json"), "utf8");
+ const cacheBefore = readFileSync(path.join(cacheDir, "key.json"), "utf8");
logs.length = 0;
assert.equal(await publishSource({ ...o, check: true }), 0, logs.join("\n"));
assert.match(logs.join("\n"), /— 4 commits, 4 pages rendered;/);
- assert.equal(stagitCalls(log).at(-1), `cache= base=https://archilyzer.pages.dev/source/git/ repo=${MIRROR_DIR}`);
- assert.equal(readFileSync(path.join(historyCacheDir(scratchRoot), "key.json"), "utf8"), cacheBefore);
+ assert.equal(stagitCalls(log).at(-1), `cache= limit= base=https://archilyzer.pages.dev/source/git/ repo=${MIRROR_DIR}`);
+ assert.equal(readFileSync(path.join(cacheDir, "key.json"), "utf8"), cacheBefore);
});
test("history: without stagit — one line with the install, nothing at /source/git/, no block in the manifest; installing it re-publishes", async () => {
@@ -786,13 +763,13 @@ test("history: a denied literal in a history page refuses, names the page and ne
const files = operatorFiles("", "");
const logs: string[] = [];
const log = path.join(dir("stagit-log"), "calls");
- const scratchRoot = dir("scratch-root");
+ const cacheDir = cacheDirIn(dir("cache-home"));
// The literal is only ever in what stagit writes, never in the repository:
// what this proves is that the file gate reads source/git/**.
- const o = opts(repo, files, logs, { filterRepo: ["true"], stagit: fakeStagit(log, { extra: `said ${planted} here` }), tokensFile: TOKENS_FILE, scratchRoot });
+ const o = opts(repo, files, logs, { filterRepo: ["true"], stagit: fakeStagit(log, { extra: `said ${planted} here` }), tokensFile: TOKENS_FILE, historyCacheDir: cacheDir });
assert.equal(await publishSource(o), 0, logs.join("\n"));
assert.ok(existsSync(path.join(o.publicDir!, "source", "git", "log.html")));
- assert.ok(existsSync(historyCacheDir(scratchRoot)));
+ assert.ok(existsSync(cacheDir));
// Deny it: the rules change, so every page is rendered again — and swept.
writeFileSync(files.denylistFile, `${planted}\n`);
@@ -806,7 +783,7 @@ test("history: a denied literal in a history page refuses, names the page and ne
assert.ok(!report.includes("said"), "no byte from beside the hit");
assert.match(report, /the previous publish was WITHDRAWN \(mirror, tree, history, tarball\)/);
assert.ok(!existsSync(path.join(o.publicDir!, "source")), "public/source, the history with it, is withdrawn");
- assert.ok(!existsSync(historyCacheDir(scratchRoot)), "the render cache is removed too");
+ assert.ok(!existsSync(cacheDir), "the render cache is removed too");
});
test("history: a stagit that fails, or a page over the host's limit, is a WARNING; the rest is published, and the next build tries again", async () => {
@@ -840,3 +817,53 @@ test("history: a stagit that fails, or a page over the host's limit, is a WARNIN
assert.equal(await publishSource({ ...o, stagit: fakeStagit(log), tokensFile: path.join(dir("none"), "tokens.css") }), 0);
assert.match(logs.join("\n"), /\[source\] WARNING: the design tokens \(.*tokens\.css\) could not be read — publishing without the history pages/);
});
+
+test("history: past the cap, the latest N of M — the manifest's commits and total, the log line, the summary; the pages of the newest only", async () => {
+ const repo = sourceRepo();
+ const logs: string[] = [];
+ const log = path.join(dir("stagit-log"), "calls");
+ const o = opts(repo, operatorFiles("", ""), logs, {
+ filterRepo: ["true"],
+ stagit: fakeStagit(log),
+ tokensFile: TOKENS_FILE,
+ historyCacheDir: cacheDirIn(dir("cache-home")),
+ maxHistoryCommits: 2,
+ });
+ assert.equal(await publishSource(o), 0, logs.join("\n"));
+ const text = logs.join("\n");
+ assert.equal(stagitCalls(log).at(-1), `cache= limit=2 base=https://archilyzer.pages.dev/source/git/ repo=${MIRROR_DIR}`, "-l, and no -c");
+ assert.match(text, /\[source\] history: stagit \(sha256 [0-9a-f]{12}\) — the latest 2 of 3 commits, 3 pages rendered; 8 files/);
+ assert.match(text, /history the latest 2 of 3 commits in 8 files\)/);
+ const manifest = JSON.parse(readFileSync(path.join(o.publicDir!, "source", "manifest.json"), "utf8"));
+ assert.equal(manifest.history.commits, 2);
+ assert.equal(manifest.history.total, 3);
+ const newest = gitIn(repo, "rev-list", "--max-count=2", "main").split("\n");
+ const published = readdirSync(path.join(o.publicDir!, "source", "git", "commit")).sort();
+ assert.equal(published.length, 2);
+ // filter-repo is `true` here, so the mirror's ids are main's.
+ assert.deepEqual(published, newest.map((c) => `${c}.html`).sort());
+});
+
+test("history: the render cache is the XDG cache's archilyzer/source-history — none for --check, none inside the checkout or the public dir", async () => {
+ const home = dir("xdg");
+ const checkout = dir("chk");
+ const pub = path.join(dir("site"), "public");
+ const paths = { monorepoRoot: checkout, sourceHistoryCacheDir: cacheDirIn(home) } as Paths;
+ const logs: string[] = [];
+ const onLog = (l: string) => logs.push(l);
+ assert.equal(await historyCacheFor({}, paths, pub, onLog), cacheDirIn(home));
+ assert.equal(await historyCacheFor({ check: true }, paths, pub, onLog), null);
+ assert.equal(await historyCacheFor({ historyCacheDir: null }, paths, pub, onLog), null);
+ assert.equal(await historyCacheFor({ historyCacheDir: path.join(home, "other") }, paths, pub, onLog), path.join(home, "other"));
+ assert.deepEqual(logs, []);
+ assert.equal(await historyCacheFor({ historyCacheDir: path.join(checkout, ".cache", "h") }, paths, pub, onLog), null);
+ assert.match(logs.at(-1)!, /the render cache .* \(XDG_CACHE_HOME\) is inside the checkout; rendering without it/);
+ assert.equal(await historyCacheFor({ historyCacheDir: path.join(pub, "h") }, paths, pub, onLog), null);
+ assert.match(logs.at(-1)!, /is inside the public dir/);
+ // getPaths: XDG_CACHE_HOME, else ~/.cache (an empty one is unset).
+ const { getPaths } = await import("../lib/paths");
+ const real = getPaths().sourceHistoryCacheDir;
+ assert.ok(real.endsWith(path.join("archilyzer", "source-history")), real);
+ const want = process.env.XDG_CACHE_HOME || path.join(os.homedir(), ".cache");
+ assert.equal(real, path.join(want, "archilyzer", "source-history"));
+});
diff --git a/common/publish/source.ts b/common/publish/source.ts
@@ -58,7 +58,7 @@ import {
HistoryProblem,
STAGIT_INSTALL,
dropHistoryCache,
- historyCacheDir,
+ SOURCE_HISTORY_MAX_COMMITS,
historyCacheKey,
historyStylesheet,
homepageThemeScript,
@@ -146,6 +146,11 @@ export type SourcePublishOpts = PublishOpts & {
// The stagit binary; null publishes without the history pages, as a
// machine without stagit does. Default: resolveStagit(paths.stagitBin).
stagit?: string | null;
+ // The history's render cache; null renders without one. Default:
+ // paths.sourceHistoryCacheDir (~/.cache/archilyzer/source-history).
+ historyCacheDir?: string | null;
+ // The history's cap. Default: SOURCE_HISTORY_MAX_COMMITS (the tests' seam).
+ maxHistoryCommits?: number;
// The design tokens the history pages' style.css is written from. Default:
// <repo>/common/styles/tokens.css.
tokensFile?: string;
@@ -633,15 +638,47 @@ export async function scratchRootProblem(
scratchRoot: string,
places: ReadonlyArray<[string, string]>,
): Promise<string | null> {
- const at = await landsAt(scratchRoot);
+ const what = await insideOf(scratchRoot, places);
+ return what
+ ? `the scratch root ${tildify(scratchRoot)} (ARCHILYZER_SOURCE_SCRATCH) is inside ${what}, where a kept scratch dir could be committed or published — point it outside`
+ : null;
+}
+
+// Which of `places` ([name, dir]) `p` lands inside, through symlinks, or null.
+async function insideOf(p: string, places: ReadonlyArray<[string, string]>): Promise<string | null> {
+ const at = await landsAt(p);
for (const [what, dir] of places) {
- if (within(at, await landsAt(dir))) {
- return `the scratch root ${tildify(scratchRoot)} (ARCHILYZER_SOURCE_SCRATCH) is inside ${what}, where a kept scratch dir could be committed or published — point it outside`;
- }
+ if (within(at, await landsAt(dir))) return what;
}
return null;
}
+/**
+ * The history's render cache for this publish, or null: none for `--check`
+ * (it writes nothing outside its scratch), none when asked for none, and none
+ * — with a line saying so — when it would land inside the checkout or the
+ * public dir, where it could be committed or published.
+ */
+export async function historyCacheFor(
+ opts: Pick<SourcePublishOpts, "check" | "historyCacheDir">,
+ paths: Paths,
+ publicDir: string,
+ onLog: (line: string) => void,
+): Promise<string | null> {
+ if (opts.check) return null;
+ const dir = opts.historyCacheDir === undefined ? (paths.sourceHistoryCacheDir ?? null) : opts.historyCacheDir;
+ if (!dir) return null;
+ const what = await insideOf(dir, [
+ ["the checkout", paths.monorepoRoot],
+ ["the public dir", publicDir],
+ ]);
+ if (what) {
+ onLog(`[source] history: the render cache ${tildify(dir)} (XDG_CACHE_HOME) is inside ${what}; rendering without it`);
+ return null;
+ }
+ return dir;
+}
+
// ── the step ────────────────────────────────────────────────────────────────
/**
@@ -664,7 +701,7 @@ export async function publishSource(opts: SourcePublishOpts = {}): Promise<numbe
const signal = opts.signal ?? new AbortController().signal;
const ctx: Ctx = { onLog, signal, env: cleanGitEnv(opts.env ?? process.env), literals: [] };
const publicDir = sourcePublicDir(paths, opts.publicDir);
- const progress = { rulesLoaded: false, scratchRoot: null as string | null };
+ const progress = { rulesLoaded: false, historyCache: null as string | null };
const withdraw = async () => {
if (!progress.rulesLoaded || opts.check) return;
if (await removePublishedSource(publicDir)) {
@@ -673,7 +710,7 @@ export async function publishSource(opts: SourcePublishOpts = {}): Promise<numbe
);
}
// The history's render cache was rendered under those rules too.
- if (progress.scratchRoot) await dropHistoryCache(historyCacheDir(progress.scratchRoot));
+ if (progress.historyCache) await dropHistoryCache(progress.historyCache);
};
let code: number;
try {
@@ -701,7 +738,7 @@ async function publish(
paths: Paths,
ctx: Ctx,
publicDir: string,
- progress: { rulesLoaded: boolean; scratchRoot: string | null },
+ progress: { rulesLoaded: boolean; historyCache: string | null },
): Promise<number> {
const { onLog } = ctx;
const started = Date.now();
@@ -782,7 +819,9 @@ async function publish(
["the public dir", publicDir],
]);
if (badRoot) throw new SourceRefusal(badRoot);
- progress.scratchRoot = scratchRoot;
+ // From here a refusal removes the history's render cache too.
+ const historyCache = await historyCacheFor(opts, paths, publicDir, onLog);
+ progress.historyCache = historyCache;
await mkdir(scratchRoot, { recursive: true });
const scratch = await mkdtemp(path.join(scratchRoot, "archilyzer-source-"));
const replace = path.join(scratch, "replace.txt");
@@ -942,7 +981,7 @@ async function publish(
mirrorHead,
commits,
scratch,
- scratchRoot,
+ historyCache,
rulesHash: rules.rulesHash,
filterRepoId,
stage,
@@ -1006,7 +1045,7 @@ async function publish(
const summary =
`main ${sourceCommit.slice(0, 12)} as ${mirrorHead.slice(0, 12)}: ${all.length} files, ${mb(totalBytes)} MB ` +
`(mirror ${packs} pack${packs === 1 ? "" : "s"}, tree ${tree.dirs} dirs, ` +
- `${history ? `history ${history.commits} commits in ${history.files} files` : "no history"}), ` +
+ `${history ? `history ${history.commits < history.total ? `the latest ${history.commits} of ${history.total}` : history.commits} commits in ${history.files} files` : "no history"}), ` +
`tarball ${mb(tarBytes)} MB sha256 ${tarSha.slice(0, 12)}`;
// 15. --check writes nothing.
@@ -1077,7 +1116,7 @@ async function stageHistoryPages(a: {
mirrorHead: string;
commits: number;
scratch: string;
- scratchRoot: string;
+ historyCache: string | null;
rulesHash: string;
filterRepoId: string;
stage: string;
@@ -1120,10 +1159,10 @@ async function stageHistoryPages(a: {
gitDir: a.bare,
head: a.mirrorHead,
commits: a.commits,
+ maxCommits: a.opts.maxHistoryCommits ?? SOURCE_HISTORY_MAX_COMMITS,
dest: a.dest,
scratch: a.scratch,
- // `--check` writes nothing outside its own scratch.
- cacheDir: a.opts.check ? null : historyCacheDir(a.scratchRoot),
+ cacheDir: a.historyCache,
cacheKey: historyCacheKey({
rulesHash: a.rulesHash,
filterRepo: a.filterRepoId,
@@ -1157,13 +1196,14 @@ async function stageHistoryPages(a: {
}
const sha256 = await historyDigest(a.stage);
ctx.onLog(
- `[source] history: ${a.stagitId} — ${a.commits} commits, ${r.rendered} page${r.rendered === 1 ? "" : "s"} rendered` +
+ `[source] history: ${a.stagitId} — ${r.shown < a.commits ? `the latest ${r.shown} of ${a.commits}` : a.commits} commits, ${r.rendered} page${r.rendered === 1 ? "" : "s"} rendered` +
`${r.cached ? " (the rest from the cache)" : ""}; ${r.files} files, ${mb(r.bytes)} MB, ` +
`the largest ${HISTORY_DIR}/${r.largest.rel} ${mb(r.largest.bytes)} MB (${elapsed(started)})`,
);
return {
href: HISTORY_LOG_HREF,
- commits: a.commits,
+ commits: r.shown,
+ total: a.commits,
head: a.mirrorHead,
files: r.files,
bytes: r.bytes,
diff --git a/common/publish/sourceHistory.test.ts b/common/publish/sourceHistory.test.ts
@@ -15,11 +15,14 @@ import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { buildThemeScript, HOMEPAGE_DEFAULT_BASE } from "../lib/themeConfig";
+import { writeFakeStagit } from "./__fixtures__/fakeStagit";
import {
HISTORY_BACK_LINK,
HISTORY_TOKENS,
HistoryProblem,
SITE_ICON_HREF,
+ SOURCE_HISTORY_MAX_COMMITS,
+ loggedCommits,
historyStylesheet,
holdHistoryCache,
homepageThemeScript,
@@ -217,41 +220,10 @@ function addCommit(work: string, i: number): void {
gitIn(work, "commit", "-q", "-m", `commit ${i}`);
}
-// What stagit writes, in the directory it runs in: a page per commit (kept
-// when there already is one, as stagit keeps it), the log, the files index,
-// the refs, the feeds, and a per-file page. `-c` makes it write the cache
-// file (the head on its first line). Every call is logged to `log`.
-function fakeStagit(log: string, o: { exit?: number; extra?: string } = {}): string {
- return exe(
- path.join(dir("fake"), "stagit"),
- `#!/bin/sh
-cache=""; base=""; repo=""
-while [ $# -gt 0 ]; do
- case "$1" in
- -c) cache="$2"; shift 2;;
- -u) base="$2"; shift 2;;
- *) repo="$1"; shift;;
- esac
-done
-echo "cache=$cache base=$base repo=$(basename "$repo")" >> '${log}'
-${o.exit ? `echo 'stagit: something broke' >&2; exit ${o.exit}` : ""}
-mkdir -p commit file/app
-for c in $(git --git-dir "$repo" rev-list HEAD); do
- if [ ! -f "commit/$c.html" ]; then
- printf '<!DOCTYPE html>\\n<html>\\n<head>\\n<link rel="icon" type="image/png" href="../favicon.png" />\\n</head>\\n<body>\\n<a href="../file/README.md.html">README</a> ${o.extra ?? ""}\\n</body>\\n</html>\\n' > "commit/$c.html"
- fi
-done
-printf '<!DOCTYPE html>\\n<html>\\n<head>\\n</head>\\n<body>\\n<span class="desc">%s</span> %s <a href="file/README.md.html">README</a> <img src="logo.png" />\\n</body>\\n</html>\\n' "$(cat "$repo/description")" "$(cat "$repo/url")" > log.html
-printf '<html>\\n<head>\\n</head>\\n<body>\\n<a href="file/app/%%5Bslug%%5D/page.tsx.html">x</a>\\n</body>\\n</html>\\n' > files.html
-printf 'refs\\n' > refs.html
-printf '<feed>%s file/README.md.html</feed>\\n' "$base" > atom.xml
-printf '<feed/>\\n' > tags.xml
-printf 'x\\n' > file/README.md.html
-touch cache.XXXXleftover
-if [ -n "$cache" ]; then git --git-dir "$repo" rev-parse HEAD > "$cache"; fi
-exit 0
-`,
- );
+// The fake (__fixtures__/fakeStagit.ts) writes what stagit writes, with -c
+// and -l as stagit.c has them; every call is logged to `log`.
+function fakeStagit(log: string, o: { exit?: number; extra?: string; pad?: number } = {}): string {
+ return writeFakeStagit(path.join(dir("fake"), "stagit"), log, o);
}
const run: HistoryRun = async (command, args, o) => {
@@ -268,6 +240,7 @@ function renderOpts(gitDir: string, stagit: string, extra: Partial<RenderHistory
gitDir,
head,
commits,
+ maxCommits: SOURCE_HISTORY_MAX_COMMITS,
dest: path.join(dir("stage"), "source", "git"),
scratch: dir("scratch"),
cacheDir: null,
@@ -292,9 +265,10 @@ test("render: the allowlist of stagit's output, each page through the post-pass,
const log = path.join(dir("log"), "calls");
const o = renderOpts(gitDir, fakeStagit(log));
const r = await renderHistory(o);
- assert.deepEqual(calls(log), [`cache= base=${o.baseUrl} repo=archilyzer.git`]);
+ assert.deepEqual(calls(log), [`cache= limit= base=${o.baseUrl} repo=archilyzer.git`]);
assert.equal(r.files, 3 + 5 + 1);
assert.equal(r.rendered, 3);
+ assert.equal(r.shown, 3);
assert.equal(r.cached, false);
assert.deepEqual(readdirSync(o.dest).sort(), ["atom.xml", "commit", "files.html", "log.html", "refs.html", "style.css", "tags.xml"]);
assert.equal(readdirSync(path.join(o.dest, "commit")).length, 3);
@@ -322,12 +296,17 @@ test("render: the page bytes are kept (a diff of a file that is not UTF-8), only
for (const f of ["files.html", "refs.html"]) writeFileSync(path.join(work, f), "<html>\n<head>\n</head>\n<body>\n</body>\n</html>\n");
for (const f of ["atom.xml", "tags.xml"]) writeFileSync(path.join(work, f), Buffer.from([0x3c, 0xff, 0x3e]));
const dest = path.join(dir("dest"), "git");
- await stageHistory(work, dest, { themeScript: "t()", stylesheet: "" });
+ await stageHistory(work, dest, { themeScript: "t()", stylesheet: "", commits: [] });
const out = readFileSync(path.join(dest, "log.html"));
assert.ok(out.includes(Buffer.from([0xff, 0xfe, 0x80])), "the odd bytes survive");
assert.ok(out.includes(Buffer.from(HISTORY_BACK_LINK)));
assert.deepEqual(readFileSync(path.join(dest, "atom.xml")), Buffer.from([0x3c, 0xff, 0x3e]));
- await assert.rejects(stageHistory(work, dest, { themeScript: "é", stylesheet: "" }), /ASCII/);
+ await assert.rejects(stageHistory(work, dest, { themeScript: "é", stylesheet: "", commits: [] }), /ASCII/);
+ // A commit the log lists must have its page.
+ await assert.rejects(
+ stageHistory(work, dest, { themeScript: "t()", stylesheet: "", commits: ["a".repeat(40)] }),
+ (e) => e instanceof HistoryProblem && /the log lists aaaaaaaaaaaa, whose page is not there/.test(e.message),
+ );
});
test("render: a failing stagit, or one that leaves the wrong pages, is a HistoryProblem", async () => {
@@ -339,20 +318,62 @@ test("render: a failing stagit, or one that leaves the wrong pages, is a History
);
await assert.rejects(
renderHistory(renderOpts(gitDir, fakeStagit(log), { commits: 5 })),
- (e) => e instanceof HistoryProblem && /stagit left 2 commit pages for 5 commits/.test(e.message),
+ (e) => e instanceof HistoryProblem && /stagit's log lists 2 commits, not 5 \(5 in all, at most 10000\)/.test(e.message),
);
});
-test("the cache: -c under the cache dir, incremental the next time, the key and the ancestry checked, busy or cut-off locks handled", async () => {
+test("loggedCommits: the commits the log links, in order, once each", () => {
+ const a = "a".repeat(40);
+ const b = "b".repeat(40);
+ const log = `<tr><td><a href="commit/${a}.html">x</a></td></tr>\n<tr><td><a href="commit/${b}.html">y</a></td></tr>\n<a href="commit/${a}.html">again</a> says href="commit/${"c".repeat(40)}.html"`;
+ assert.deepEqual(loggedCommits(log), [a, b]);
+});
+
+test("the cap: past it, -l keeps the newest in the log, and only their pages are published", async () => {
+ const { gitDir, work } = bareRepo(5);
+ const newest = gitIn(TMP, "--git-dir", gitDir, "rev-list", "--max-count=3", "main").split("\n");
+ const log = path.join(dir("log"), "calls");
+ const stagit = fakeStagit(log);
+
+ // No cache: `-l 3`, no `-c`.
+ let o = renderOpts(gitDir, stagit, { maxCommits: 3 });
+ let r = await renderHistory(o);
+ assert.equal(calls(log).at(-1), `cache= limit=3 base=${o.baseUrl} repo=archilyzer.git`);
+ assert.equal(r.shown, 3);
+ assert.equal(r.files, 3 + 5 + 1);
+ assert.deepEqual(readdirSync(path.join(o.dest, "commit")).sort(), newest.map((c) => `${c}.html`).sort());
+ assert.equal(readdirSync(path.join(o.scratch, "history", "commit")).length, 5, "stagit wrote all five; three are published");
+ assert.match(readFileSync(path.join(o.dest, "log.html"), "utf8"), /2 more commits remaining, fetch the repository/);
+
+ // With the cache: still `-l 3` (stagit refuses -c with -l). A new commit:
+ // its page is the one rendered, and the oldest of the three drops out.
+ const cacheDir = path.join(dir("cache-home"), "archilyzer", "source-history");
+ o = renderOpts(gitDir, stagit, { maxCommits: 3, cacheDir });
+ r = await renderHistory(o);
+ assert.equal(calls(log).at(-1), `cache= limit=3 base=${o.baseUrl} repo=archilyzer.git`);
+ assert.equal(r.rendered, 5);
+ addCommit(work, 9);
+ gitIn(TMP, "--git-dir", gitDir, "fetch", "-q", work, "main:main");
+ const now = gitIn(TMP, "--git-dir", gitDir, "rev-list", "--max-count=3", "main").split("\n");
+ o = renderOpts(gitDir, stagit, { maxCommits: 3, cacheDir });
+ r = await renderHistory(o);
+ assert.equal(r.cached, true);
+ assert.equal(r.rendered, 1);
+ assert.equal(r.shown, 3);
+ assert.deepEqual(readdirSync(path.join(o.dest, "commit")).sort(), now.map((c) => `${c}.html`).sort());
+ assert.match(readFileSync(path.join(o.dest, "log.html"), "utf8"), /3 more commits remaining/);
+});
+
+test("the cache: -c in the cache dir, incremental the next time, the key and the log's ancestry checked, busy or cut-off locks handled", async () => {
const { gitDir, work } = bareRepo(2);
const log = path.join(dir("log"), "calls");
const stagit = fakeStagit(log);
- const cacheDir = path.join(dir("scratch-root"), "archilyzer-source-history");
+ const cacheDir = path.join(dir("cache-home"), "archilyzer", "source-history");
const cacheFile = path.join(cacheDir, "stagit.cache");
- const cacheArgs = (o: RenderHistoryOpts) => `cache=${cacheFile} base=${o.baseUrl} repo=archilyzer.git`;
+ const cacheArgs = (o: RenderHistoryOpts) => `cache=${cacheFile} limit= base=${o.baseUrl} repo=archilyzer.git`;
- // First: everything rendered, the cache kept (key, cache file, output) and
- // unlocked; no per-file pages and no stagit leftovers are kept.
+ // First: everything rendered, the cache made (recursively) and kept (key,
+ // cache file, output), unlocked; no per-file pages, no stagit leftovers.
let o = renderOpts(gitDir, stagit, { cacheDir });
let r = await renderHistory(o);
assert.equal(r.cached, false);
@@ -370,9 +391,9 @@ test("the cache: -c under the cache dir, incremental the next time, the key and
assert.equal(r.cached, true);
assert.equal(r.rendered, 1);
assert.equal(r.files, 3 + 5 + 1);
+ assert.equal(loggedCommits(readFileSync(path.join(o.dest, "log.html"), "utf8")).length, 3, "the new line and the cached ones");
- // Another key (other rules, another stagit…): every page rendered again,
- // and a page the cache held that is not in history is gone.
+ // Another key (other rules, another stagit…): every page rendered again.
const junk = path.join(cacheDir, "out", "commit", `${"f".repeat(40)}.html`);
writeFileSync(junk, "junk");
o = renderOpts(gitDir, stagit, { cacheDir, cacheKey: "k2" });
@@ -380,22 +401,34 @@ test("the cache: -c under the cache dir, incremental the next time, the key and
assert.equal(r.cached, false);
assert.equal(r.rendered, 3);
assert.ok(!existsSync(junk));
- assert.ok(!existsSync(path.join(o.dest, "commit", `${"f".repeat(40)}.html`)));
- // A cache whose commit is not an ancestor of the head (a rewritten main).
- writeFileSync(cacheFile, `${"e".repeat(40)}\n`);
+ // Log lines ending at a commit that is not an ancestor of the head (a
+ // rewritten main): the cache file goes, the pages stay, the log is whole.
+ writeFileSync(cacheFile, `${"e".repeat(40)}\n<tr><td><a href="commit/${"e".repeat(40)}.html">x</a></td></tr>\n`);
o = renderOpts(gitDir, stagit, { cacheDir, cacheKey: "k2" });
r = await renderHistory(o);
- assert.equal(r.cached, false);
+ assert.equal(r.cached, true);
+ assert.equal(r.rendered, 0);
+ assert.equal(loggedCommits(readFileSync(path.join(o.dest, "log.html"), "utf8")).length, 3);
- // A cache that looks sound but holds a page history does not: the render
- // notices (pages ≠ commits) and renders every page again.
+ // A page of a commit history does not have is never published.
writeFileSync(junk, "junk");
o = renderOpts(gitDir, stagit, { cacheDir, cacheKey: "k2" });
r = await renderHistory(o);
+ assert.equal(r.files, 3 + 5 + 1);
+ assert.ok(!existsSync(path.join(o.dest, "commit", `${"f".repeat(40)}.html`)));
+
+ // A listed commit's page gone from the cache: noticed, and every page is
+ // rendered again.
+ const head = gitIn(TMP, "--git-dir", gitDir, "rev-parse", "main");
+ const oldest = gitIn(TMP, "--git-dir", gitDir, "rev-list", "main").split("\n").at(-1)!;
+ assert.notEqual(oldest, head);
+ rmSync(path.join(cacheDir, "out", "commit", `${oldest}.html`));
+ o = renderOpts(gitDir, stagit, { cacheDir, cacheKey: "k2" });
+ r = await renderHistory(o);
assert.equal(r.cached, false);
- assert.match(o.logs.join("\n"), /stagit left 4 commit pages for 3 commits from the cache; rendering every page again/);
- assert.ok(!existsSync(junk));
+ assert.match(o.logs.join("\n"), new RegExp(`the log lists ${oldest.slice(0, 12)}, whose page is not there from the cache; rendering every page again`));
+ assert.ok(existsSync(path.join(o.dest, "commit", `${oldest}.html`)));
// `fresh` (--force) renders every page, whatever the cache holds.
o = renderOpts(gitDir, stagit, { cacheDir, cacheKey: "k2", fresh: true });
@@ -408,7 +441,7 @@ test("the cache: -c under the cache dir, incremental the next time, the key and
o = renderOpts(gitDir, stagit, { cacheDir, cacheKey: "k2" });
r = await renderHistory(o);
assert.equal(r.cached, false);
- assert.equal(calls(log).at(-1), `cache= base=${o.baseUrl} repo=archilyzer.git`, "rendered without -c");
+ assert.equal(calls(log).at(-1), `cache= limit= base=${o.baseUrl} repo=archilyzer.git`, "rendered without -c");
assert.match(o.logs.join("\n"), /the render cache is in use by another publish; rendering without it/);
assert.ok(existsSync(path.join(cacheDir, "lock")), "the other holder's lock is left alone");
@@ -464,4 +497,16 @@ test("the real stagit: a page per commit, every link into file/ now into the tre
assert.match(logHtml, /<title>Log - archilyzer - Archilyzer<\/title>/);
assert.match(logHtml, /git clone <a href="https:\/\/archilyzer\.pages\.dev\/source\/archilyzer\.git">/);
assert.match(readFileSync(path.join(o.dest, "atom.xml"), "utf8"), /href="https:\/\/archilyzer\.pages\.dev\/source\/git\/commit\/[0-9a-f]{40}\.html"/);
+
+ // The cap, by stagit itself: -l keeps the NEWEST in the log (and says how
+ // many more), writes a page for every commit, and only the listed are
+ // published.
+ const capped = renderOpts(gitDir, stagit, { maxCommits: 2 });
+ const rc = await renderHistory(capped);
+ const newest = gitIn(TMP, "--git-dir", gitDir, "rev-list", "--max-count=2", "main").split("\n");
+ assert.equal(rc.shown, 2);
+ assert.deepEqual(loggedCommits(readFileSync(path.join(capped.dest, "log.html"), "utf8")), newest);
+ assert.deepEqual(readdirSync(path.join(capped.dest, "commit")).sort(), newest.map((c) => `${c}.html`).sort());
+ assert.equal(readdirSync(path.join(capped.scratch, "history", "commit")).length, 3);
+ assert.match(readFileSync(path.join(capped.dest, "log.html"), "utf8"), /1 more commits remaining, fetch the repository/);
});
diff --git a/common/publish/sourceHistory.ts b/common/publish/sourceHistory.ts
@@ -27,18 +27,25 @@
// stagit's header (the name, the description, the clone line, Log | Files |
// Refs) stays as stagit writes it.
//
+// THE CAP. At most SOURCE_HISTORY_MAX_COMMITS commits, the newest, get a page:
+// past it stagit runs with `-l`, the log lists that many and says how many
+// more there are, and only the listed commits' pages are published (stagit
+// still writes one for every commit). The /source/ page then says "the latest
+// N of M". The 15,000-file drop in source.ts stays, as the last resort.
+//
// THE CACHE. stagit's `-c <cachefile>` renders incrementally: it walks from
// HEAD to the commit the cache names and keeps every commit page already in
-// its output directory. So the cache is a DIRECTORY kept between publishes —
-// `<ARCHILYZER_SOURCE_SCRATCH>/archilyzer-source-history/` (outside the
-// checkout and the public dir; the scratch root's own check) holding the
-// cache file, stagit's output and a key. It is trusted only when the key
-// matches (the scrub rules and step, filter-repo, stagit, the header text),
-// the commit it names is an ancestor of today's head, and the last run
-// finished (the key is removed before a render and written after it). One
-// publish holds it at a time (`lock`, the holder's pid); a second renders
-// without it. `--force` renders it afresh, `--check` never touches it, and a
-// refusal removes it.
+// its output directory (`-l` keeps them too; it cannot be combined with
+// `-c`). So the cache is a DIRECTORY kept between publishes —
+// `${XDG_CACHE_HOME:-~/.cache}/archilyzer/source-history/`
+// (paths.sourceHistoryCacheDir; never inside the checkout or the public dir)
+// holding the cache file, stagit's output and a key. Its pages are kept only
+// when the key matches (the scrub rules and step, filter-repo, stagit, the
+// header text) and the last run finished (the key is removed before a render
+// and written after it); its log lines only when the commit they end at is an
+// ancestor of today's head. One publish holds it at a time (`lock`, the
+// holder's pid); a second renders without it. `--force` renders it afresh,
+// `--check` never touches it, and a refusal removes it.
import { createHash } from "node:crypto";
import { accessSync, constants, existsSync, statSync } from "node:fs";
@@ -53,8 +60,12 @@ import { onPath } from "./sourceAudit";
export const STAGIT_INSTALL =
"git clone git://git.codemadness.org/stagit && make -C stagit && cp stagit/stagit ~/.local/bin/";
-/** The cache directory's name under the scratch root. */
-export const HISTORY_CACHE_NAME = "archilyzer-source-history";
+/**
+ * The cap: at most this many commits — the newest — get a page and a log
+ * line (stagit `-l`); the log says how many more there are. One file per
+ * commit counts against the step's 15,000 (Pages' 20,000).
+ */
+export const SOURCE_HISTORY_MAX_COMMITS = 10_000;
/** The site's own 32 px icon, which stagit's logo.png and favicon.png become. */
export const SITE_ICON_HREF = "/icons/icon-32.png";
@@ -299,15 +310,30 @@ export function rewriteHistoryPage(html: string, themeScript: string): string {
}
/**
- * Copy the allowlist of stagit's output from `work` into `dest`, each page
+ * The commits the log lists, newest first: every `href="commit/<sha>.html"`
+ * in log.html (stagit writes one per log line; a commit's own text cannot
+ * fake one, since stagit encodes every `"` it prints as `"`). These, and
+ * only these, have their pages published.
+ */
+export function loggedCommits(logHtml: string): string[] {
+ const seen = new Set<string>();
+ for (const m of logHtml.matchAll(/<a href="commit\/([0-9a-f]{40})\.html">/g)) seen.add(m[1]);
+ return [...seen];
+}
+
+/**
+ * Copy the allowlist of stagit's output from `work` into `dest` — the
+ * top-level pages and feeds, and the page of each commit in `commits` (the
+ * ones the log lists; any other page in `work` stays there) — each page
* through the post-pass (byte for byte otherwise: read and written as latin1,
* so a diff of a file that is not UTF-8 keeps its bytes), then style.css.
- * The feeds are copied as they are.
+ * The feeds are copied as they are. A listed page that is not there is a
+ * HistoryProblem.
*/
export async function stageHistory(
work: string,
dest: string,
- o: { themeScript: string; stylesheet: string },
+ o: { themeScript: string; stylesheet: string; commits: readonly string[] },
): Promise<{ files: number; bytes: number; largest: { rel: string; bytes: number } }> {
if (/[^\x00-\x7f]/.test(o.themeScript)) throw new Error("the theme script must be ASCII");
const rels: string[] = [];
@@ -315,9 +341,11 @@ export async function stageHistory(
if (!existsSync(path.join(work, f))) throw new HistoryProblem(`stagit wrote no ${f}`);
rels.push(f);
}
- const commitDir = path.join(work, "commit");
- for (const f of existsSync(commitDir) ? await readdir(commitDir) : []) {
- if (COMMIT_PAGE.test(f)) rels.push(`commit/${f}`);
+ for (const sha of o.commits) {
+ const rel = `commit/${sha}.html`;
+ if (!COMMIT_PAGE.test(`${sha}.html`)) throw new HistoryProblem(`the log names ${sha.slice(0, 40)}, not a commit id`);
+ if (!existsSync(path.join(work, rel))) throw new HistoryProblem(`the log lists ${sha.slice(0, 12)}, whose page is not there`);
+ rels.push(rel);
}
await mkdir(path.join(dest, "commit"), { recursive: true });
let bytes = 0;
@@ -355,16 +383,20 @@ export type HistoryRun = (
export type RenderHistoryOpts = {
stagit: string;
// The scrubbed bare clone — a directory named archilyzer.git (stagit names
- // the repository after it) — at `head`, with `commits` commits.
+ // the repository after it) — at `head`, with `commits` commits in all.
gitDir: string;
head: string;
commits: number;
+ // The cap: at most this many commits (the newest) get a page and a log
+ // line. SOURCE_HISTORY_MAX_COMMITS, but for the tests.
+ maxCommits: number;
// Where the published copy goes (stage/source/git).
dest: string;
// The publish's own scratch dir: the render's directory when there is no
// cache to use.
scratch: string;
- // The kept cache directory, or null (a `--check`).
+ // The kept cache directory, or null (a `--check`, or a cache dir the step
+ // may not use).
cacheDir: string | null;
// Changes whenever pages rendered before must not be kept.
cacheKey: string;
@@ -386,6 +418,8 @@ export type RenderedHistory = {
files: number;
bytes: number;
largest: { rel: string; bytes: number };
+ // The commits with a page — min(commits, maxCommits), the newest.
+ shown: number;
// Pages stagit wrote this run (all of them without a usable cache).
rendered: number;
cached: boolean;
@@ -401,9 +435,10 @@ const pidAlive = (pid: number): boolean => {
};
/**
- * Hold the cache directory, or say it is busy. The lock names its holder's
- * pid; a lock whose holder is gone means a render was cut off, so nothing in
- * the directory is trusted: it is emptied and taken.
+ * Hold the cache directory, or say it is busy. The directory is made on the
+ * way (recursively). The lock names its holder's pid; a lock whose holder is
+ * gone means a render was cut off, so nothing in the directory is trusted: it
+ * is emptied and taken.
*/
export async function holdHistoryCache(dir: string): Promise<boolean> {
await mkdir(dir, { recursive: true, mode: 0o700 });
@@ -450,6 +485,14 @@ async function countCommitPages(work: string): Promise<number> {
* Render the history into `o.dest`. Throws HistoryProblem when stagit fails
* or its output is not what it should be; the caller publishes without the
* history then.
+ *
+ * THE CAP. Up to `maxCommits` commits, stagit runs with `-c` (its log lines
+ * cached: a publish renders only the new commits). Past it, with `-l
+ * <maxCommits>`: the log lists the newest `maxCommits` and says how many more
+ * there are ("N more commits remaining, fetch the repository"). stagit refuses
+ * `-c` with `-l`, and `-l` still writes a page for EVERY commit — so what is
+ * published is the page of each commit the log lists, and no other. The
+ * pages already in the cache's directory are kept by stagit either way.
*/
export async function renderHistory(o: RenderHistoryOpts): Promise<RenderedHistory> {
// stagit's header reads both from the repository directory. Neither is
@@ -469,8 +512,9 @@ export async function renderHistory(o: RenderHistoryOpts): Promise<RenderedHisto
const keyFile = path.join(dir, "key.json");
const cacheFile = path.join(dir, "stagit.cache");
const work = path.join(dir, "out");
- let usable = !o.fresh && (await cacheUsable(o, dir, keyFile, cacheFile, work));
+ let usable = !o.fresh && (await cacheUsable(o, keyFile, work));
if (!usable) await emptyCache(dir);
+ else await dropStaleLogCache(o, dir, cacheFile);
// The key goes before the render and comes back after it: a render cut
// off leaves none, and the next publish starts over.
await rm(keyFile, { force: true });
@@ -503,29 +547,33 @@ async function emptyCache(dir: string): Promise<void> {
}
}
-async function cacheUsable(
- o: RenderHistoryOpts,
- dir: string,
- keyFile: string,
- cacheFile: string,
- work: string,
-): Promise<boolean> {
+// The cache's pages can be kept: the same key, and a last run that finished.
+// (A page is its commit's, by id; pages of commits no longer in history are
+// never published, since only the log's commits are.)
+async function cacheUsable(o: RenderHistoryOpts, keyFile: string, work: string): Promise<boolean> {
let key: unknown = null;
try {
key = (JSON.parse(await readFile(keyFile, "utf8")) as { key?: unknown }).key;
} catch {
return false;
}
- if (key !== o.cacheKey || !existsSync(path.join(work, "log.html"))) return false;
+ return key === o.cacheKey && existsSync(path.join(work, "log.html"));
+}
+
+// stagit's `-c` file holds the log lines down to the commit it names, and
+// stagit appends them to the new ones: when that commit is not an ancestor
+// of today's head (a main rewritten since), the lines are of another history,
+// and the file goes (the pages stay).
+async function dropStaleLogCache(o: RenderHistoryOpts, dir: string, cacheFile: string): Promise<void> {
+ if (!existsSync(cacheFile)) return;
const last = (await readFile(cacheFile, "utf8").catch(() => "")).split("\n")[0].trim();
- if (!/^[0-9a-f]{40}$/.test(last)) return false;
- // History that was rewritten since (a force-push to the private main)
- // leaves pages of commits that are no longer in it.
- const r = await o.run("git", ["--git-dir", o.gitDir, "merge-base", "--is-ancestor", last, o.head], {
- cwd: dir,
- timeoutMs: 30_000,
- });
- return r.code === 0;
+ const ancestor =
+ /^[0-9a-f]{40}$/.test(last) &&
+ (await o.run("git", ["--git-dir", o.gitDir, "merge-base", "--is-ancestor", last, o.head], {
+ cwd: dir,
+ timeoutMs: 30_000,
+ })).code === 0;
+ if (!ancestor) await rm(cacheFile, { force: true });
}
async function renderOnce(
@@ -535,7 +583,14 @@ async function renderOnce(
): Promise<Omit<RenderedHistory, "cached">> {
await mkdir(work, { recursive: true });
const before = await countCommitPages(work);
- const args = [...(cacheFile ? ["-c", cacheFile] : []), "-u", o.baseUrl, o.gitDir];
+ const capped = o.commits > o.maxCommits;
+ const shown = Math.min(o.commits, o.maxCommits);
+ const args = [
+ ...(capped ? ["-l", String(o.maxCommits)] : cacheFile ? ["-c", cacheFile] : []),
+ "-u",
+ o.baseUrl,
+ o.gitDir,
+ ];
const r = await o.run(o.stagit, args, { cwd: work, timeoutMs: 600_000 });
// The per-file pages are never published, and stagit writes them all again
// on every run: none is kept.
@@ -544,17 +599,20 @@ async function renderOnce(
const tail = r.out.filter((l) => l.trim()).slice(-2).join(" / ");
throw new HistoryProblem(`stagit exited ${r.code}${tail ? `: ${tail}` : ""}`);
}
- const pages = await countCommitPages(work);
- if (pages !== o.commits) {
- throw new HistoryProblem(`stagit left ${pages} commit pages for ${o.commits} commits`);
+ const logPath = path.join(work, "log.html");
+ if (!existsSync(logPath)) throw new HistoryProblem("stagit wrote no log.html");
+ const commits = loggedCommits((await readFile(logPath)).toString("latin1"));
+ if (commits.length !== shown) {
+ throw new HistoryProblem(`stagit's log lists ${commits.length} commits, not ${shown} (${o.commits} in all, at most ${o.maxCommits})`);
}
- const staged = await stageHistory(work, o.dest, { themeScript: o.themeScript, stylesheet: o.stylesheet });
- return { ...staged, rendered: cacheFile ? pages - before : pages };
-}
-
-/** The cache directory under a scratch root. */
-export function historyCacheDir(scratchRoot: string): string {
- return path.join(scratchRoot, HISTORY_CACHE_NAME);
+ if (commits[0] !== o.head) throw new HistoryProblem(`stagit's log does not start at the head`);
+ const staged = await stageHistory(work, o.dest, {
+ themeScript: o.themeScript,
+ stylesheet: o.stylesheet,
+ commits,
+ });
+ const after = await countCommitPages(work);
+ return { ...staged, shown, rendered: cacheFile ? after - before : after };
}
/**
