// A REPORT'S REVISION HISTORY — per report, never site-wide (plans/report-sites.md, // "History"). Readers can check that a report is the one published, and see // every edit with its hashes. // // THE STORE. A bare git repository beside the report's report.json: // `sites//reports//history-git/` (never a path segment named // `.git` — wrangler drops one). It sits inside the corpus's own tree; the // operator adds `history-git/` to that tree's .gitignore (nothing here writes // it). One branch, `main`; one commit per revision, holding // // report.json the report.json as it was on disk, byte for byte // report.md the Markdown export of that revision (lib/report/exportMarkdown.ts) // exports.json the sha256 and size of every export file made for that // revision and of its citations JSON and CSV // // with the message `Revision N` and the change summary against the revision // before (lib/report/revisions.ts reportChangeSummary). // // WHEN A REVISION IS MADE. `reports export` (./reportExports.ts) commits one // when the sha256 of report.json differs from the newest revision's; an export // of an unchanged report commits nothing (its files may still differ — a PDF // is not byte-stable — and exports.json keeps the hashes of the revision's own // export). // // WHO AND WHEN. Every commit's author and committer is the SITE — its title, // `noreply@.invalid` — dated in UTC (`+0000`). git runs with a minimal // environment: no system or global config, HOME pointing nowhere, none of the // operator's GIT_* variables (cleanGitEnv) — so no operator name, email, time // zone or signing key reaches a commit. // // THE FOOTER SCHEME. An export's footer (`exportFooterFor`) prints `Revision N` // and the sha256 of the report.json it was made from — never the commit: a // commit holds report.md, whose footer would have to name the commit holding // it. The history page and history.json map each revision's report sha256 to // its commit, so a reader goes footer → sha256 → commit. export.json (the // local manifest) records the commit once it is made. // // PUBLISHING. Compose (./composeReports.ts) reads the revisions into // `history.json` (ReportHistoryView) and stages a dumb-HTTP clone at // `reports//history/repo/`, built as the source mirror is // (./source.ts): a fresh bare clone, repacked, `update-server-info`, then an // ALLOWLISTED copy — HEAD, packed-refs, info/refs, objects/info/packs, the // packs, refs/heads/main — so no config, hook or log of the store is // published. import { createHash } from "node:crypto"; import { cp, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { execa } from "execa"; import { publishFileSizeProblem } from "../lib/builtExport"; import type { Paths } from "../lib/paths"; import type { Site } from "../lib/site"; import type { Report } from "../lib/report/schema"; import { REPORT_HISTORY_BRANCH, REPORT_HISTORY_FORMAT, REPORT_HISTORY_REPO_FILE_RE, REPORT_HISTORY_VERSION, claimChanges, reportChangeSummary, reportHistoryPagePath, reportHistoryRepoPath, reportHistoryViewPath, revisionCommitMessage, revisionSummaryOf, type ReportHistoryRef, type ReportHistoryView, type ReportRevisionView, } from "../lib/report/revisions"; import { cleanGitEnv } from "./sourceAudit"; export { REPORT_HISTORY_REPO_FILE_RE }; import { siteReportDir } from "./reportMedia"; export const REPORT_HISTORY_GIT_DIRNAME = "history-git"; export const REVISION_EXPORTS_FILENAME = "exports.json"; export const REVISION_EXPORTS_FORMAT = "archilyzer-report-revision-exports"; export const REVISION_EXPORTS_VERSION = 1; // The published clone's packs are split below this (the publish limit is 24 MiB). const PACK_SIZE = "20m"; const HEAD_REF = `refs/heads/${REPORT_HISTORY_BRANCH}`; const ZERO_OID = "0".repeat(40); // `sites//reports//history-git/`. export function reportHistoryGitDir(paths: Paths, siteId: string, reportId: string): string { return path.join(siteReportDir(paths, siteId, reportId), REPORT_HISTORY_GIT_DIRNAME); } // ─── git, with nothing of the operator's ─── export type HistoryIdentity = { name: string; email: string }; // The site, as the author and committer of its reports' revisions. export function historyIdentity(site: Pick): HistoryIdentity { return { name: site.siteTitle?.trim() || site.siteId, email: `noreply@${site.siteId}.invalid` }; } // `@ +0000`: git's own date form, in UTC. export function gitDateUtc(d: Date): string { return `@${Math.floor(d.getTime() / 1000)} +0000`; } // The whole environment git runs with: PATH, and nothing else of this // process's — no HOME (so no ~/.gitconfig), no system or global config, no // GIT_* variable, no EMAIL — then the identity and date when committing. export function historyGitEnv(identity?: HistoryIdentity, date?: Date): NodeJS.ProcessEnv { const env: NodeJS.ProcessEnv = cleanGitEnv({ PATH: process.env.PATH, HOME: path.join(os.tmpdir(), "archilyzer-report-history-nohome"), XDG_CONFIG_HOME: path.join(os.tmpdir(), "archilyzer-report-history-nohome"), GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: os.devNull, GIT_TERMINAL_PROMPT: "0", TZ: "UTC", LC_ALL: "C", LANG: "C", } as unknown as NodeJS.ProcessEnv); // Set here, for the git child only (written, never read). return { ...env, ...(identity ? { GIT_AUTHOR_NAME: identity.name, GIT_AUTHOR_EMAIL: identity.email, GIT_COMMITTER_NAME: identity.name, GIT_COMMITTER_EMAIL: identity.email, } : {}), ...(date ? { GIT_AUTHOR_DATE: gitDateUtc(date), GIT_COMMITTER_DATE: gitDateUtc(date) } : {}), }; } // Config every invocation carries on its command line, over whatever a // repository's own config could say. const GIT_FLAGS = ["-c", "commit.gpgSign=false", "-c", "core.hooksPath=/dev/null", "-c", "core.logAllRefUpdates=false"]; type GitOpts = { input?: string | Uint8Array; env?: NodeJS.ProcessEnv; cwd?: string }; async function gitRaw(args: string[], opts: GitOpts = {}): Promise { const r = await execa("git", [...GIT_FLAGS, ...args], { env: opts.env ?? historyGitEnv(), extendEnv: false, input: opts.input, cwd: opts.cwd, encoding: "buffer", // A blob's bytes are its bytes: its final newline included. stripFinalNewline: false, reject: false, timeout: 120_000, }); if (r.exitCode !== 0) { const err = Buffer.from(r.stderr as Uint8Array).toString("utf8").trim().split("\n")[0]; throw new Error(`git ${args.find((a) => !a.startsWith("-") && !a.includes("/")) ?? args[0]} failed (exit ${r.exitCode}): ${err}`); } return Buffer.from(r.stdout as Uint8Array); } async function git(gitDir: string, args: string[], opts: GitOpts = {}): Promise { return (await gitRaw(["--git-dir", gitDir, ...args], opts)).toString("utf8").trim(); } const sha256 = (data: string | Uint8Array) => createHash("sha256").update(data).digest("hex"); async function isRepo(gitDir: string): Promise { return (await stat(path.join(gitDir, "HEAD")).catch(() => null))?.isFile() ?? false; } async function headCommit(gitDir: string): Promise { if (!(await isRepo(gitDir))) return null; try { return await git(gitDir, ["rev-parse", "--verify", "--quiet", `${HEAD_REF}^{commit}`]); } catch { return null; } } // ─── Reading ─── export type StoredRevision = { revision: number; commit: string; // ISO 8601, UTC, to the second. date: string; reportSha256: string; // The report.json's bytes. reportJson: Buffer; message: string; }; const isoSeconds = (secs: number) => new Date(secs * 1000).toISOString().replace(/\.\d{3}Z$/, "Z"); // Every revision in the store, oldest first; [] when there is no store. export async function readStoredRevisions(gitDir: string): Promise { if (!(await headCommit(gitDir))) return []; const log = await gitRaw(["--git-dir", gitDir, "log", "--reverse", "--format=%H%x1f%ct%x1f%B%x1e", HEAD_REF]); const entries = log .toString("utf8") .split("\x1e") .map((e) => e.replace(/^\n/, "")) .filter((e) => e.trim()); const out: StoredRevision[] = []; for (const [i, e] of entries.entries()) { const [commit, ct, message] = e.split("\x1f"); const reportJson = await gitRaw(["--git-dir", gitDir, "cat-file", "blob", `${commit}:report.json`]); out.push({ revision: i + 1, commit, date: isoSeconds(Number(ct)), reportSha256: sha256(reportJson), reportJson, message: message.trimEnd(), }); } return out; } export type RevisionHead = { revision: number; commit: string; date: string; reportSha256: string; summary: string[] }; // The newest revision, or null when there is none. export async function readRevisionHead(gitDir: string): Promise { const head = await headCommit(gitDir); if (!head) return null; const count = Number(await git(gitDir, ["rev-list", "--count", HEAD_REF])); const [ct, ...msg] = (await git(gitDir, ["log", "-1", "--format=%ct%n%B", HEAD_REF])).split("\n"); const reportJson = await gitRaw(["--git-dir", gitDir, "cat-file", "blob", `${head}:report.json`]); return { revision: count, commit: head, date: isoSeconds(Number(ct)), reportSha256: sha256(reportJson), summary: revisionSummaryOf(msg.join("\n")), }; } // The revision a report.json of `reportSha256` is: the newest when it is that // report.json (`changed` false), else the next one to be committed. export async function pendingRevision( gitDir: string, reportSha256: string, ): Promise<{ revision: number; changed: boolean; head: RevisionHead | null }> { const head = await readRevisionHead(gitDir); if (!head) return { revision: 1, changed: true, head }; if (head.reportSha256 === reportSha256) return { revision: head.revision, changed: false, head }; return { revision: head.revision + 1, changed: true, head }; } // ─── Committing ─── export type RevisionExports = { format: typeof REVISION_EXPORTS_FORMAT; version: typeof REVISION_EXPORTS_VERSION; reportId: string; reportSha256: string; // By file name: report.html, report.pdf, report.md, evidence-pack.zip (each // that was made), citations.json, citations.csv. files: Record; }; export function revisionExports( reportId: string, reportSha256: string, files: Record, ): RevisionExports { const out: RevisionExports["files"] = {}; for (const name of Object.keys(files).sort()) { const data = typeof files[name] === "string" ? Buffer.from(files[name] as string) : (files[name] as Uint8Array); out[name] = { bytes: data.length, sha256: sha256(data) }; } return { format: REVISION_EXPORTS_FORMAT, version: REVISION_EXPORTS_VERSION, reportId, reportSha256, files: out }; } export type CommittedRevision = RevisionHead & { // Whether this call made the commit (false: report.json had not changed). committed: boolean; }; function parseReport(data: Buffer): Report | null { try { const v = JSON.parse(data.toString("utf8")) as Report; return v && typeof v === "object" && Array.isArray(v.sections) ? v : null; } catch { return null; } } async function ensureRepo(gitDir: string): Promise { if (await isRepo(gitDir)) return; await mkdir(path.dirname(gitDir), { recursive: true }); // No template: no hooks, no description, no sample files. await gitRaw(["init", "--bare", "--quiet", `--initial-branch=${REPORT_HISTORY_BRANCH}`, "--template=", gitDir]); } // Commit `reportJson` (with its Markdown export and exports.json) as the next // revision — when it is not the newest revision's report.json already, in // which case nothing is written and the newest is answered. export async function commitReportRevision(o: { gitDir: string; site: Pick; reportJson: Uint8Array; markdown: string; exports: RevisionExports; now: Date; }): Promise { const reportJson = Buffer.from(o.reportJson); const sha = sha256(reportJson); const pending = await pendingRevision(o.gitDir, sha); if (!pending.changed && pending.head) return { ...pending.head, committed: false }; await ensureRepo(o.gitDir); const g = o.gitDir; const prev = pending.head ? parseReport(await gitRaw(["--git-dir", g, "cat-file", "blob", `${pending.head.commit}:report.json`])) : null; const next = parseReport(reportJson); const summary = next ? reportChangeSummary(pending.head ? prev ?? emptyReport(next) : null, next) : ["report.json is not a readable report"]; const blob = (data: string | Uint8Array) => git(g, ["hash-object", "-w", "--stdin"], { input: data }); const entries = [ [REVISION_EXPORTS_FILENAME, await blob(`${JSON.stringify(o.exports, null, 2)}\n`)], ["report.json", await blob(reportJson)], ["report.md", await blob(o.markdown)], ]; const tree = await git(g, ["mktree"], { input: entries.map(([name, id]) => `100644 blob ${id}\t${name}\n`).join("") }); const env = historyGitEnv(historyIdentity(o.site), o.now); const commit = await git( g, ["commit-tree", "--no-gpg-sign", tree, ...(pending.head ? ["-p", pending.head.commit] : []), "-F", "-"], { input: revisionCommitMessage(pending.revision, summary), env }, ); await git(g, ["update-ref", "-m", `revision ${pending.revision}`, HEAD_REF, commit, pending.head?.commit ?? ZERO_OID]); return { revision: pending.revision, commit, date: isoSeconds(Math.floor(o.now.getTime() / 1000)), reportSha256: sha, summary, committed: true, }; } // A report with nothing in it: what an unreadable earlier revision is // compared as. function emptyReport(like: Report): Report { return { ...like, title: "", series: undefined, subtitle: undefined, summary: undefined, method: undefined, citations: {}, sections: [] }; } // ─── The history page's data ─── // Every revision with its summary and claim diff, or null when the report has // no revisions. `clone` is what `git clone` takes. export async function readReportHistoryView( gitDir: string, o: { reportId: string; siteUrl?: string }, ): Promise { const stored = await readStoredRevisions(gitDir); if (stored.length === 0) return null; const revisions: ReportRevisionView[] = []; let prev: Report | null = null; let newest: Report | null = null; for (const s of stored) { const report = parseReport(s.reportJson); revisions.push({ revision: s.revision, date: s.date, commit: s.commit, reportSha256: s.reportSha256, summary: revisionSummaryOf(s.message), claims: prev && report ? claimChanges(prev, report) : [], }); if (report) newest = report; prev = report; } const base = o.siteUrl?.trim().replace(/\/+$/, ""); const repo = reportHistoryRepoPath(o.reportId); return { format: REPORT_HISTORY_FORMAT, version: REPORT_HISTORY_VERSION, reportId: o.reportId, ...(newest?.series ? { series: newest.series } : {}), title: newest?.title ?? o.reportId, branch: REPORT_HISTORY_BRANCH, clone: base ? `${base}${repo}` : repo, revisions, }; } // What the report's page shows: the newest revision, current when the // report.json now is that revision's. export function reportHistoryRef(view: ReportHistoryView, currentSha256: string | null): ReportHistoryRef { const last = view.revisions[view.revisions.length - 1]; return { revision: last.revision, date: last.date, href: reportHistoryPagePath(view.reportId), current: currentSha256 === last.reportSha256, }; } // ─── The published clone ─── // Stage a dumb-HTTP clone of the store at `destDir` (replaced whole): a fresh // bare clone of the one branch, repacked, its server info written, copied // from an allowlist. Throws when a file would be over the publish limit. // Answers the files written, relative to `destDir`. export async function stageReportHistoryRepo(gitDir: string, destDir: string): Promise { const tmp = await mkdtemp(path.join(os.tmpdir(), "report-history-stage-")); try { const clone = path.join(tmp, "clone"); await gitRaw( ["clone", "--bare", "--quiet", "--no-local", "--single-branch", "--branch", REPORT_HISTORY_BRANCH, "--template=", gitDir, clone], { cwd: tmp }, ); await git(clone, ["repack", "-a", "-d", "-q", `--max-pack-size=${PACK_SIZE}`]); await git(clone, ["prune-packed"]); await git(clone, ["pack-refs", "--all"]); await git(clone, ["update-server-info"]); const refs = (await git(clone, ["for-each-ref", "--format=%(refname)"])).split("\n").filter(Boolean); if (refs.length !== 1 || refs[0] !== HEAD_REF) { throw new Error(`the history clone must hold ${HEAD_REF} alone, and holds ${refs.join(", ") || "no refs"}`); } const head = await git(clone, ["rev-parse", HEAD_REF]); await rm(destDir, { recursive: true, force: true }); for (const d of ["info", "objects/info", "objects/pack", "refs/heads"]) { await mkdir(path.join(destDir, d), { recursive: true }); } const files: string[] = []; for (const f of ["HEAD", "packed-refs", "info/refs", "objects/info/packs"]) { await cp(path.join(clone, f), path.join(destDir, f)); files.push(f); } await writeFile(path.join(destDir, "refs", "heads", REPORT_HISTORY_BRANCH), `${head}\n`); files.push(`refs/heads/${REPORT_HISTORY_BRANCH}`); for (const f of (await readdir(path.join(clone, "objects", "pack"))).sort()) { if (!/^pack-[0-9a-f]+\.(pack|idx)$/.test(f)) continue; await cp(path.join(clone, "objects", "pack", f), path.join(destDir, "objects", "pack", f)); files.push(`objects/pack/${f}`); } for (const f of files) { if (!REPORT_HISTORY_REPO_FILE_RE.test(f)) throw new Error(`the history clone would publish ${f}, which it may not`); const problem = publishFileSizeProblem(f, (await stat(path.join(destDir, f))).size); if (problem) throw new Error(`the history clone's ${problem}`); } return files; } finally { await rm(tmp, { recursive: true, force: true }); } } // Publish a report's history under `publicDir`: history.json and the clone, // at `reports//history/`. Answers the clone's files. export async function publishReportHistory(o: { gitDir: string; publicDir: string; view: ReportHistoryView; }): Promise { const viewFile = path.join(o.publicDir, ...reportHistoryViewPath(o.view.reportId).split("/").filter(Boolean)); await mkdir(path.dirname(viewFile), { recursive: true }); await writeFile(viewFile, `${JSON.stringify(o.view, null, 2)}\n`); const repoDir = path.join(o.publicDir, ...reportHistoryRepoPath(o.view.reportId).split("/").filter(Boolean)); return stageReportHistoryRepo(o.gitDir, repoDir); } // The sha256 of a file, or null when it cannot be read. export async function fileSha256(file: string): Promise { try { return sha256(await readFile(file)); } catch { return null; } }