and the header. await writeFile(path.join(o.gitDir, "description"), o.description); await writeFile(path.join(o.gitDir, "url"), `${o.cloneUrl}\n`); const withoutCache = async (): Promise<RenderedHistory> => ({ ...(await renderOnce(o, path.join(o.scratch, "history"), null)), cached: false, }); const dir = o.cacheDir; if (!dir) return withoutCache(); // A cache that cannot be used (unwritable, read-only, full) is one line and // a render without it: never a failed build. const unusable = (err: unknown) => o.onLog(`[source] history: the render cache ${tildify(dir)} is unusable (${ioCode(err)}); rendering without it`); let held: boolean; try { held = await holdHistoryCache(dir, o.onLog); } catch (err) { if (!isIoError(err)) throw err; unusable(err); return withoutCache(); } if (!held) { o.onLog("[source] history: the render cache is in use by another publish; rendering without it"); return withoutCache(); } try { const keyFile = path.join(dir, "key.json"); const cacheFile = path.join(dir, "stagit.cache"); const work = path.join(dir, "out"); let usable: boolean; try { usable = !o.fresh && (await cacheUsable(o, keyFile, work)); if (!usable) await emptyCache(dir); else await dropStaleLogCache(o, dir, cacheFile); // The key goes before the render and comes back after it: a render cut // off leaves none, and the next publish starts over. await rm(keyFile, { force: true }); } catch (err) { if (!isIoError(err)) throw err; unusable(err); return await withoutCache(); } try { const r = await renderOnce(o, work, cacheFile); await writeKey(o, keyFile); return { ...r, cached: usable }; } catch (err) { if (!(err instanceof HistoryProblem) || !usable) throw err; // Not a fault: stagit's -c walk is in commit-date order and stops at the // head it rendered last, so the commits of a merge that are older than // that head are left out, and the log comes up short. Once more, every // page. o.onLog( `[source] history: the cached render does not cover this head (${err.message}) — stagit's -c stops at the last head it rendered, and a merge of older commits falls behind it; rendering every page again`, ); try { await emptyCache(dir); } catch (ioErr) { if (!isIoError(ioErr)) throw ioErr; unusable(ioErr); return await withoutCache(); } const r = await renderOnce(o, work, cacheFile); await writeKey(o, keyFile); return { ...r, cached: false }; } } catch (err) { await emptyCache(dir).catch(() => {}); throw err; } finally { await releaseHistoryCache(dir).catch(() => {}); } } // An I/O error from the file system (it carries an errno code), as opposed to // the history's own problems, a cancel, or a bug. function isIoError(err: unknown): err is NodeJS.ErrnoException { return err instanceof Error && !(err instanceof HistoryProblem) && typeof (err as NodeJS.ErrnoException).code === "string"; } function ioCode(err: unknown): string { return (err as NodeJS.ErrnoException)?.code ?? "an I/O error"; } // The key, after a render that finished. A key that cannot be written is no // key: the next publish renders every page again. async function writeKey(o: RenderHistoryOpts, keyFile: string): Promise<void> { try { await writeFile(keyFile, JSON.stringify({ key: o.cacheKey }) + "\n"); } catch (err) { if (!isIoError(err)) throw err; o.onLog(`[source] history: the render cache's key could not be written (${ioCode(err)}); the next publish renders every page again`); } } // Everything in the cache but its lock. async function emptyCache(dir: string): Promise<void> { for (const f of await readdir(dir).catch(() => [] as string[])) { if (f !== "lock") await rm(path.join(dir, f), { recursive: true, force: true }); } } // The cache's pages can be kept: the same key, and a last run that finished. // (A page is its commit's, by id; pages of commits no longer in history are // never published, since only the log's commits are.) async function cacheUsable(o: RenderHistoryOpts, keyFile: string, work: string): Promise<boolean> { let key: unknown = null; try { key = (JSON.parse(await readFile(keyFile, "utf8")) as { key?: unknown }).key; } catch { return false; } return key === o.cacheKey && existsSync(path.join(work, "log.html")); } // stagit's `-c` file holds the log lines down to the commit it names, and // stagit appends them to the new ones: when that commit is not an ancestor // of today's head (a main rewritten since), the lines are of another history, // and the file goes (the pages stay). async function dropStaleLogCache(o: RenderHistoryOpts, dir: string, cacheFile: string): Promise<void> { if (!existsSync(cacheFile)) return; const last = (await readFile(cacheFile, "utf8").catch(() => "")).split("\n")[0].trim(); const ancestor = /^[0-9a-f]{40}$/.test(last) && (await o.run("git", ["--git-dir", o.gitDir, "merge-base", "--is-ancestor", last, o.head], { cwd: dir, timeoutMs: 30_000, })).code === 0; if (!ancestor) await rm(cacheFile, { force: true }); } async function renderOnce( o: RenderHistoryOpts, work: string, cacheFile: string | null, ): Promise<Omit<RenderedHistory, "cached">> { try { await mkdir(work, { recursive: true }); } catch (err) { if (!isIoError(err)) throw err; throw new HistoryProblem(`the render directory cannot be made (${ioCode(err)})`); } const before = await countCommitPages(work); const capped = o.commits > o.maxCommits; const shown = Math.min(o.commits, o.maxCommits); const args = [ ...(capped ? ["-l", String(o.maxCommits)] : cacheFile ? ["-c", cacheFile] : []), "-u", o.baseUrl, o.gitDir, ]; const r = await o.run(o.stagit, args, { cwd: work, timeoutMs: 600_000 }); // The per-file pages are never published, and stagit writes them all again // on every run: none is kept. await rm(path.join(work, "file"), { recursive: true, force: true }).catch(() => {}); if (r.code !== 0) { const tail = r.out.filter((l) => l.trim()).slice(-2).join(" / "); throw new HistoryProblem(`stagit exited ${r.code}${tail ? `: ${tail}` : ""}`); } const logPath = path.join(work, "log.html"); if (!existsSync(logPath)) throw new HistoryProblem("stagit wrote no log.html"); const commits = loggedCommits((await readFile(logPath)).toString("latin1")); if (commits.length !== shown) { throw new HistoryProblem(`stagit's log lists ${commits.length} commits, not ${shown} (${o.commits} in all, at most ${o.maxCommits})`); } if (commits[0] !== o.head) throw new HistoryProblem(`stagit's log does not start at the head`); const staged = await stageHistory(work, o.dest, { themeScript: o.themeScript, stylesheet: o.stylesheet, commits, treeFiles: o.treeFiles, }); const after = await countCommitPages(work); return { ...staged, shown, rendered: cacheFile ? after - before : after }; } /** * The cache's key: whatever, changed, makes a page rendered before wrong — the * rules (with the step's version) and filter-repo that made the ids, the * stagit that wrote the pages, and the header text every page carries. */ export function historyCacheKey(parts: { rulesHash: string; filterRepo: string; stagit: string; description: string; cloneUrl: string; baseUrl: string; }): string { return createHash("sha256").update(JSON.stringify(parts)).digest("hex"); }

// The source's history pages: stagit's rendering of the scrubbed mirror, // published at /source/git/ by `archilyzer source publish` (source.ts), which // runs this after the mirror is built and its objects audited, and before the // staged files are audited — so every page goes through the same gate as the // mirror, the tree and the tarball. // // stagit (codemadness.org, C over libgit2) is an OPERATOR-INSTALLED tool, like // git-filter-repo: never vendored, never committed. It is found as STAGIT_BIN, // else `stagit` on PATH, else ~/.local/bin/stagit (resolveStagit). WITHOUT IT // THE PUBLISH GOES ON: one log line says so and how to install it, and the // source is published without /source/git/ (the /source/ page then shows no // History links). A render that fails, or pages over the host's limits, are // the same: a WARNING, and no history — never a failed build. // // WHAT IS PUBLISHED, from an allowlist of what stagit writes: // log.html, files.html, refs.html, atom.xml, tags.xml, commit/.html // and style.css, written here from common/styles/tokens.css. NOT stagit's // per-file pages (file/…): browsing is the raw tree at /source/tree/, so // every link into file/ is rewritten to it and file/ is never staged. // // THE POST-PASS (rewriteHistoryPage) touches every .html page and never the // two feeds. It adds exactly two things — the homepage's pre-paint theme // script (lib/themeConfig.ts buildThemeScript, the string the homepage's // ThemeScript emits), and one line at the top linking back to /source/ — and // points two kinds of stagit link somewhere that exists: `…file/.html` // at the raw tree, and its logo.png / favicon.png at the site's own icon. // stagit's header (the name, the description, the clone line, Log | Files | // Refs) stays as stagit writes it. // // THE CAP. At most SOURCE_HISTORY_MAX_COMMITS commits, the newest, get a page: // past it stagit runs with `-l`, the log lists that many and says how many // more there are, and only the listed commits' pages are published (stagit // still writes one for every commit). The /source/ page then says "the latest // N of M". The 15,000-file drop in source.ts stays, as the last resort. // // THE CACHE. stagit's `-c ` renders incrementally: it walks from // HEAD to the commit the cache names and keeps every commit page already in // its output directory (`-l` keeps them too; it cannot be combined with // `-c`). So the cache is a DIRECTORY kept between publishes — // `${XDG_CACHE_HOME:-~/.cache}/archilyzer/source-history/` // (paths.sourceHistoryCacheDir; never inside the checkout or the public dir) // holding the cache file, stagit's output and a key. Its pages are kept only // when the key matches (the scrub rules and step, filter-repo, stagit, the // header text) and the last run finished (the key is removed before a render // and written after it); its log lines only when the commit they end at is an // ancestor of today's head. One publish holds it at a time (`lock`, the // holder's pid); a second renders without it, and a lock whose pid is not // running or that is over an hour old is stale and replaced. A cache that // cannot be written (EACCES, EROFS, ENOSPC) is one line and a render without // it. `--force` renders it afresh, `--check` never touches it, and a refusal // removes it. // // LINKS TO WHAT IS NOT PUBLISHED become text: a diff's file that main no // longer has, a commit with no page (past the cap). The raw tree's file list // decides (`treeFiles`). import { createHash } from "node:crypto"; import { accessSync, constants, existsSync, statSync } from "node:fs"; import { cp, mkdir, readdir, readFile, realpath, rm, stat, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { PROJECT_NAME } from "../lib/project"; import { buildThemeScript, HOMEPAGE_DEFAULT_BASE } from "../lib/themeConfig"; import { onPath, tildify } from "./sourceAudit"; /** How to install stagit, as the log line and the doctor say it. */ export const STAGIT_INSTALL = "git clone git://git.codemadness.org/stagit && make -C stagit && cp stagit/stagit ~/.local/bin/"; /** * The cap: at most this many commits — the newest — get a page and a log * line (stagit `-l`); the log says how many more there are. One file per * commit counts against the step's 15,000 (Pages' 20,000). */ export const SOURCE_HISTORY_MAX_COMMITS = 10_000; /** The site's own 32 px icon, which stagit's logo.png and favicon.png become. */ export const SITE_ICON_HREF = "/icons/icon-32.png"; /** * The one line put at the top of every page. ASCII only (`·`): the * pages are rewritten byte for byte (latin1 in, latin1 out), so anything * injected must be the same bytes in every encoding. */ export const HISTORY_BACK_LINK = `

${PROJECT_NAME} · Source