commit 38d9ca564e16f5189815cc76f879121bfba66440
parent 8fe1f18d4aa9cc74ddbb2ceb315de08df6126bf0
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 28 Sep 2026 13:12:09 -0400
common: `publishSource` — a scrubbed, dumb-HTTP git mirror, the raw tree and the tarball
A fresh `--no-local --bare --single-branch` clone of the git common dir's main,
rewritten by git-filter-repo (the home-dir rule first, then the operator's
source-scrub.txt, over file contents AND commit messages), repacked into
≤20 MB packs with info/refs and objects/info/packs; audited object by object,
then file by file; staged from an allowlist (no config, hooks, filter-repo/ or
private ids); installed link-safe with the manifest removed first and written
last. Skips when main and the rules are unchanged — the rules hash lives in
homepage/.source-publish.json, never in the published manifest, because a hash
of the denylist would confirm a guess at it. Refuses at 15,000 files / 24 MiB.
getPaths() gains ARCHILYZER_CONFIG_DIR, SOURCE_SCRUB_FILE, SOURCE_DENYLIST_FILE
and ARCHILYZER_SOURCE_SCRATCH; ENVIRONMENT.md regenerated (TRANSCRIPTS_DIR
also says umtool reads it).
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
5 files changed, 1155 insertions(+), 4 deletions(-)
diff --git a/ENVIRONMENT.md b/ENVIRONMENT.md
@@ -12,7 +12,7 @@ The one override surface for where things live and which binary runs. Every one
| Variable | Default | What it does | Read by |
|---|---|---|---|
-| `TRANSCRIPTS_DIR` | `<repo>/transcripts` | The corpus: channels, sites, the LMDB index, job logs, the saved-video store. | common/lib/paths.ts (getPaths) |
+| `TRANSCRIPTS_DIR` | `<repo>/transcripts` | The corpus: channels, sites, the LMDB index, job logs, the saved-video store. umtool reads `<it>/channels` too, when its own `CHANNELS_DIR` is unset. | common/lib/paths.ts (getPaths) |
| `SAVED_VIDEOS_DIR` | `<TRANSCRIPTS_DIR>/saved-videos` | The persisted source-video store, when it should live on another disk. | common/lib/paths.ts (getPaths) |
| `SITES_DIR` | `<TRANSCRIPTS_DIR>/sites` | Per-site config (`<id>/site.json`, every key in [SITE.md](SITE.md)) and the homepage's `_homepage/`. | common/lib/paths.ts (getPaths) |
| `SETTINGS_FILE` | `<repo>/settings.json` | The settings file (every key in [SETTINGS.md](SETTINGS.md)). | common/lib/paths.ts (getPaths) |
@@ -39,6 +39,10 @@ The one override surface for where things live and which binary runs. Every one
| `GALLERY_DL_BIN` | `gallery-dl` on PATH | The X/Twitter post fetcher, for social channels. | common/lib/paths.ts (getPaths) |
| `OLLAMA_URL` | `http://127.0.0.1:11434` | The local ollama server, the local digest and attribution engine. | common/lib/paths.ts (getPaths) |
| `CLAUDE_BIN` | `claude` on PATH | The `claude` CLI, driving the opt-in metered digest lane. | common/lib/paths.ts (getPaths) |
+| `ARCHILYZER_CONFIG_DIR` | `~/.config/archilyzer` | The operator's private config dir, outside the repo: the two inputs of `archilyzer source publish` below. Never committed. | common/lib/paths.ts (getPaths) |
+| `SOURCE_SCRUB_FILE` | `<ARCHILYZER_CONFIG_DIR>/source-scrub.txt` | git-filter-repo `lhs==>rhs` rules applied to file contents AND commit messages when the source mirror is generated (`<home dir>==>/home/user` is built in and runs first). Every rule's left side is also denied. See [PUBLISH.md](PUBLISH.md). | common/lib/paths.ts (getPaths) |
+| `SOURCE_DENYLIST_FILE` | `<ARCHILYZER_CONFIG_DIR>/source-denylist.txt` | Literals the published source must never contain, one per line (`i:` = any case). One hit anywhere in the mirror, the tree or the tarball refuses the publish. | common/lib/paths.ts (getPaths) |
+| `ARCHILYZER_SOURCE_SCRATCH` | the OS temp dir | Where `source publish` makes its scratch clone and stage (removed afterwards unless `--keep-scratch`). | common/lib/paths.ts (getPaths) |
## Runtime
@@ -123,7 +127,7 @@ The publish pipeline sets these for a process it spawns. Listed so a reader know
| `INSTANCE_MODE` | a site | `hub` makes the export build the hub. Set by `archilyzer build hub`. | export/app/lib/mode.ts, common/lib/archive/contract.ts |
| `BUILD_ARCHIVES` | on | `0` skips archive-zip generation for one build (`--skip-archives`). | common/bin/compose-site.ts, common/bin/build-archives.ts |
| `ARCHIVES_READONLY` | off | `1` inside a docker-mode build container: materialize archives, never write the shared cache. | common/bin/compose-site.ts |
-| `HOMEPAGE_PUBLIC_DIR` | `<repo>/homepage/public` | Where `compose homepage` writes. | common/bin/compose-homepage.ts |
+| `HOMEPAGE_PUBLIC_DIR` | `<repo>/homepage/public` | Where `compose homepage` and `source publish` write. | common/bin/compose-homepage.ts, common/publish/source.ts |
## Docker
diff --git a/common/lib/envVars.ts b/common/lib/envVars.ts
@@ -53,7 +53,7 @@ const paths = (name: string, def: string, doc: string): EnvVarDecl => ({
const DECLARED: EnvVarDecl[] = [
// ── paths: getPaths() ──────────────────────────────────────────────────
- paths("TRANSCRIPTS_DIR", "`<repo>/transcripts`", "The corpus: channels, sites, the LMDB index, job logs, the saved-video store."),
+ paths("TRANSCRIPTS_DIR", "`<repo>/transcripts`", "The corpus: channels, sites, the LMDB index, job logs, the saved-video store. umtool reads `<it>/channels` too, when its own `CHANNELS_DIR` is unset."),
paths("SAVED_VIDEOS_DIR", "`<TRANSCRIPTS_DIR>/saved-videos`", "The persisted source-video store, when it should live on another disk."),
paths("SITES_DIR", "`<TRANSCRIPTS_DIR>/sites`", "Per-site config (`<id>/site.json`, every key in [SITE.md](SITE.md)) and the homepage's `_homepage/`."),
paths("SETTINGS_FILE", "`<repo>/settings.json`", "The settings file (every key in [SETTINGS.md](SETTINGS.md))."),
@@ -80,6 +80,10 @@ const DECLARED: EnvVarDecl[] = [
paths("GALLERY_DL_BIN", "`gallery-dl` on PATH", "The X/Twitter post fetcher, for social channels."),
paths("OLLAMA_URL", "`http://127.0.0.1:11434`", "The local ollama server, the local digest and attribution engine."),
paths("CLAUDE_BIN", "`claude` on PATH", "The `claude` CLI, driving the opt-in metered digest lane."),
+ paths("ARCHILYZER_CONFIG_DIR", "`~/.config/archilyzer`", "The operator's private config dir, outside the repo: the two inputs of `archilyzer source publish` below. Never committed."),
+ paths("SOURCE_SCRUB_FILE", "`<ARCHILYZER_CONFIG_DIR>/source-scrub.txt`", "git-filter-repo `lhs==>rhs` rules applied to file contents AND commit messages when the source mirror is generated (`<home dir>==>/home/user` is built in and runs first). Every rule's left side is also denied. See [PUBLISH.md](PUBLISH.md)."),
+ paths("SOURCE_DENYLIST_FILE", "`<ARCHILYZER_CONFIG_DIR>/source-denylist.txt`", "Literals the published source must never contain, one per line (`i:` = any case). One hit anywhere in the mirror, the tree or the tarball refuses the publish."),
+ paths("ARCHILYZER_SOURCE_SCRATCH", "the OS temp dir", "Where `source publish` makes its scratch clone and stage (removed afterwards unless `--keep-scratch`)."),
// ── runtime ────────────────────────────────────────────────────────────
{ name: "WORKER_TOKEN", audience: "runtime", default: "unset (both surfaces off)", readBy: "common/lib/workerToken.ts, scripts/archilyzer-ops.mjs, mcp/src/fetchClip.ts", doc: "Bearer token for the remote-worker API and for `/api/ops/*` (`pnpm ops`, the MCP's `fetch_clip`). Set the same value on both ends." },
@@ -132,7 +136,7 @@ const DECLARED: EnvVarDecl[] = [
{ name: "INSTANCE_MODE", audience: "internal", default: "a site", readBy: "export/app/lib/mode.ts, common/lib/archive/contract.ts", doc: "`hub` makes the export build the hub. Set by `archilyzer build hub`." },
{ name: "BUILD_ARCHIVES", audience: "internal", default: "on", readBy: "common/bin/compose-site.ts, common/bin/build-archives.ts", doc: "`0` skips archive-zip generation for one build (`--skip-archives`)." },
{ name: "ARCHIVES_READONLY", audience: "internal", default: "off", readBy: "common/bin/compose-site.ts", doc: "`1` inside a docker-mode build container: materialize archives, never write the shared cache." },
- { name: "HOMEPAGE_PUBLIC_DIR", audience: "internal", default: "`<repo>/homepage/public`", readBy: "common/bin/compose-homepage.ts", doc: "Where `compose homepage` writes." },
+ { name: "HOMEPAGE_PUBLIC_DIR", audience: "internal", default: "`<repo>/homepage/public`", readBy: "common/bin/compose-homepage.ts, common/publish/source.ts", doc: "Where `compose homepage` and `source publish` write." },
// ── docker: the container's set ────────────────────────────────────────
{ name: "ARCHILYZER_TRANSCRIBER", audience: "docker", default: "baked per image target (`whisper-cpp` in `runtime`)", readBy: "docker/entrypoint.sh", doc: "`whisper-cpp` or `parakeet`: which worker the first boot seeds and which model it fetches." },
diff --git a/common/lib/paths.ts b/common/lib/paths.ts
@@ -158,6 +158,16 @@ export type Paths = {
// settings.digest.remoteEnabled). Not bundled; install it separately and point
// CLAUDE_BIN at it if it isn't on PATH.
claudeBin: string;
+ // The operator's PRIVATE config dir, outside the repo (~/.config/archilyzer
+ // by default). Holds the two inputs of `archilyzer source publish`
+ // (common/publish/source.ts), which are never committed:
+ // sourceScrubFile — git-filter-repo `lhs==>rhs` rules for the mirror
+ // sourceDenylistFile — literals the published source must never contain
+ configDir: string;
+ sourceScrubFile: string;
+ sourceDenylistFile: string;
+ // Where `source publish` makes its scratch clone (removed afterwards).
+ sourceScratchDir: string;
};
let cached: Paths | null = null;
@@ -179,6 +189,9 @@ export function getPaths(): Paths {
const exportSharedDir = path.join(exportIndexDir, "shared");
const sitesDir = process.env.SITES_DIR ?? path.join(transcriptsDir, "sites");
const homepageDir = path.join(sitesDir, "_homepage");
+ const configDir =
+ process.env.ARCHILYZER_CONFIG_DIR ??
+ path.join(os.homedir(), ".config", "archilyzer");
cached = {
monorepoRoot,
transcriptsDir,
@@ -264,6 +277,13 @@ export function getPaths(): Paths {
"",
),
claudeBin: process.env.CLAUDE_BIN ?? "claude",
+ configDir,
+ sourceScrubFile:
+ process.env.SOURCE_SCRUB_FILE ?? path.join(configDir, "source-scrub.txt"),
+ sourceDenylistFile:
+ process.env.SOURCE_DENYLIST_FILE ??
+ path.join(configDir, "source-denylist.txt"),
+ sourceScratchDir: process.env.ARCHILYZER_SOURCE_SCRATCH ?? os.tmpdir(),
};
return cached;
}
diff --git a/common/publish/source.test.ts b/common/publish/source.test.ts
@@ -0,0 +1,331 @@
+import { test, after, before } from "node:test";
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { execFileSync } from "node:child_process";
+import {
+ existsSync,
+ lstatSync,
+ mkdirSync,
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ rmSync,
+ statSync,
+ symlinkSync,
+ writeFileSync,
+} from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import { CLONE_URL, MIRROR_DIR, TARBALL_HREF, TREE_HREF } from "../lib/sourceManifest";
+import {
+ MAX_FILES,
+ MAX_FILE_BYTES,
+ SourceRefusal,
+ clearPublishedSource,
+ limitProblem,
+ loadSourceRules,
+ parseScrubRules,
+ publishSource,
+ resolveFilterRepo,
+ type SourcePublishOpts,
+} from "./source";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common test
+//
+// `archilyzer source publish` (source.ts) over temp repos. The two tests that
+// rewrite history need git-filter-repo; they SKIP when resolveFilterRepo()
+// cannot find one (say so in a gate: the release gate requires they RAN).
+// Every repo, public dir and scratch dir is under the OS temp dir, and the git
+// variables a hook or a wrapper might export are cleared first.
+for (const key of ["GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_PREFIX"]) {
+ delete process.env[key];
+}
+
+const TMP = mkdtempSync(path.join(os.tmpdir(), "source-publish-"));
+after(() => rmSync(TMP, { recursive: true, force: true }));
+
+// Planted, never real: the home dir the built-in rule scrubs, and the paths.
+const HOME = "/home/not-a-real-user";
+const PLANTED = "plantedhome";
+const SECRET = "plantedsecret";
+
+let filterRepoProblem: string | null = null;
+before(async () => {
+ try {
+ await resolveFilterRepo({});
+ } catch (err) {
+ filterRepoProblem = (err as Error).message;
+ }
+});
+
+let n = 0;
+function dir(name: string): string {
+ const d = path.join(TMP, `${name}-${n++}`);
+ mkdirSync(d, { recursive: true });
+ return d;
+}
+
+function gitIn(cwd: string, ...args: string[]): string {
+ return execFileSync("git", args, { cwd, stdio: "pipe" }).toString().trim();
+}
+
+// A repo with `main` of three commits: a planted path in one blob and one
+// message, and the names the tree pages must encode.
+function sourceRepo(extra: Record<string, string> = {}): string {
+ const repo = dir("src");
+ gitIn(repo, "init", "-q", "-b", "main");
+ gitIn(repo, "config", "user.name", "source test");
+ gitIn(repo, "config", "user.email", "source@example.invalid");
+ gitIn(repo, "config", "commit.gpgsign", "false");
+ const put = (files: Record<string, string>, message: string) => {
+ for (const [f, text] of Object.entries(files)) {
+ mkdirSync(path.dirname(path.join(repo, f)), { recursive: true });
+ writeFileSync(path.join(repo, f), text);
+ }
+ gitIn(repo, "add", "-A");
+ gitIn(repo, "commit", "-q", "-m", message);
+ };
+ put(
+ {
+ "README.md": "hello\n",
+ "app/[slug]/page.tsx": "export default 1;\n",
+ "fonts/Archivo[wdth,wght].ttf": "not really a font\n",
+ },
+ "first",
+ );
+ put({ "notes.txt": `data lives at /srv/${PLANTED}/x\n`, ...extra }, "add notes");
+ put({ "README.md": "hello again\n" }, `moved from /srv/${PLANTED}`);
+ return repo;
+}
+
+function operatorFiles(scrub: string, deny: string): { scrubFile: string; denylistFile: string } {
+ const d = dir("config");
+ writeFileSync(path.join(d, "source-scrub.txt"), scrub);
+ writeFileSync(path.join(d, "source-denylist.txt"), deny);
+ return { scrubFile: path.join(d, "source-scrub.txt"), denylistFile: path.join(d, "source-denylist.txt") };
+}
+
+function opts(repo: string, files: { scrubFile: string; denylistFile: string }, logs: string[], extra: Partial<SourcePublishOpts> = {}): SourcePublishOpts {
+ return {
+ paths: { monorepoRoot: TMP } as Paths,
+ sourceRepo: path.join(repo, ".git"),
+ publicDir: path.join(dir("site"), "public"),
+ scratchRoot: path.join(TMP, "scratch"),
+ gitleaks: null,
+ homeDir: HOME,
+ onLog: (l) => logs.push(l),
+ now: () => new Date("2026-09-28T12:00:00.000Z"),
+ ...files,
+ ...extra,
+ };
+}
+
+const sha256 = (file: string) => createHash("sha256").update(readFileSync(file)).digest("hex");
+
+test("scrub rules: the home-dir rule first, comments and blanks dropped, every literal left side denied", () => {
+ const rules = parseScrubRules(
+ ["# a comment", "", `/srv/${PLANTED}==>/home/user`, " # indented comment", "literal:abc==>x", "regex:a+b==>c", "glob:*.x==>y", "bare-line", "a==>b==>c", "\r"].join("\n"),
+ HOME,
+ );
+ assert.deepEqual(rules.lines, [
+ `${HOME}==>/home/user`,
+ `/srv/${PLANTED}==>/home/user`,
+ "literal:abc==>x",
+ "regex:a+b==>c",
+ "glob:*.x==>y",
+ "bare-line",
+ "a==>b==>c",
+ ]);
+ // filter-repo splits at the LAST ==>; regex and glob rules deny nothing.
+ assert.deepEqual(rules.denied, [HOME, `/srv/${PLANTED}`, "abc", "bare-line", "a==>b"]);
+ // A home dir of `/`, or one that IS the replacement, gets no built-in rule.
+ assert.deepEqual(parseScrubRules("", "/").lines, []);
+ assert.deepEqual(parseScrubRules("", "/home/user").lines, []);
+});
+
+test("the operator's files: a missing one refuses by name; the denylist's i: and the implied left sides; the hash moves with them", async () => {
+ const d = dir("cfg");
+ await assert.rejects(
+ loadSourceRules({ scrubFile: path.join(d, "nope.txt"), denylistFile: path.join(d, "deny.txt"), homeDir: HOME }),
+ (e) => e instanceof SourceRefusal && /no scrub rules at .*nope\.txt — create it/.test(e.message),
+ );
+ writeFileSync(path.join(d, "scrub.txt"), `/srv/${PLANTED}==>/home/user\n`);
+ await assert.rejects(
+ loadSourceRules({ scrubFile: path.join(d, "scrub.txt"), denylistFile: path.join(d, "deny.txt"), homeDir: HOME }),
+ (e) => e instanceof SourceRefusal && /no denylist at .*deny\.txt/.test(e.message),
+ );
+ writeFileSync(path.join(d, "deny.txt"), `# mine\ni:${SECRET.toUpperCase()}\n`);
+ const a = await loadSourceRules({ scrubFile: path.join(d, "scrub.txt"), denylistFile: path.join(d, "deny.txt"), homeDir: HOME });
+ assert.deepEqual(
+ a.literals.map((l) => [l.bytes.toString(), l.ci]),
+ [[SECRET, true], [HOME, false], [`/srv/${PLANTED}`, false]],
+ );
+ writeFileSync(path.join(d, "deny.txt"), `i:${SECRET}\nanother\n`);
+ const b = await loadSourceRules({ scrubFile: path.join(d, "scrub.txt"), denylistFile: path.join(d, "deny.txt"), homeDir: HOME });
+ assert.notEqual(a.rulesHash, b.rulesHash, "a new literal must defeat the skip");
+});
+
+test("the limits: 15,000 files and 24 MiB a file, inside Pages' 20,000 and 25 MiB", () => {
+ assert.equal(MAX_FILES, 15_000);
+ assert.equal(MAX_FILE_BYTES, 24 * 1024 * 1024);
+ assert.equal(limitProblem([{ rel: "a", bytes: MAX_FILE_BYTES }]), null);
+ assert.match(limitProblem([{ rel: "big.pack", bytes: MAX_FILE_BYTES + 1 }])!, /^big\.pack is 24\.0 MiB, over the step's limit of 24\.0 MiB/);
+ const many = Array.from({ length: MAX_FILES + 1 }, (_, i) => ({ rel: `f${i}`, bytes: 1 }));
+ assert.match(limitProblem(many)!, /^15001 files to publish, over the step's limit of 15000/);
+ assert.equal(limitProblem(many.slice(1)), null);
+});
+
+test("skip: an unchanged main with unchanged rules does nothing; a changed rule does not skip", async () => {
+ const repo = sourceRepo();
+ const files = operatorFiles(`/srv/${PLANTED}==>/home/user\n`, "");
+ const logs: string[] = [];
+ // filter-repo is `false`: reaching it would refuse, so a 0 proves the skip.
+ const o = opts(repo, files, logs, { filterRepo: ["false"] });
+ const pub = o.publicDir!;
+ const sourceCommit = gitIn(repo, "rev-parse", "main");
+ const rules = await loadSourceRules({ ...files, homeDir: HOME });
+ mkdirSync(path.join(pub, "source", MIRROR_DIR, "info"), { recursive: true });
+ mkdirSync(path.join(pub, "downloads"), { recursive: true });
+ writeFileSync(path.join(pub, "source", MIRROR_DIR, "info", "refs"), "x\trefs/heads/main\n");
+ writeFileSync(path.join(pub, "downloads", path.basename(TARBALL_HREF)), "tarball");
+ writeFileSync(
+ path.join(pub, "source", "manifest.json"),
+ JSON.stringify({
+ version: 1, generatedAt: "x", branch: "main", sourceCommit, mirrorHead: "a".repeat(40), subject: "s",
+ files: 1, bytes: 1, mirror: {}, tree: {}, tarball: { href: TARBALL_HREF, bytes: 7, sha256: "b".repeat(64) },
+ audit: {}, tools: {},
+ }),
+ );
+ writeFileSync(path.join(path.dirname(pub), ".source-publish.json"), JSON.stringify({ sourceCommit, rulesHash: rules.rulesHash }));
+ assert.equal(await publishSource(o), 0);
+ assert.match(logs.join("\n"), new RegExp(`up to date at ${sourceCommit.slice(0, 12)}; skipping`));
+
+ writeFileSync(files.denylistFile, "a-new-literal\n");
+ logs.length = 0;
+ assert.equal(await publishSource(o), 1, "the new rule reached the rewrite");
+ assert.match(logs.join("\n"), /REFUSED: false --force --quiet exited 1/);
+ assert.equal(readdirSync(o.scratchRoot!).length, 0, "the scratch dir is removed");
+});
+
+test("round trip: --check writes nothing; publish; a dumb clone of the mirror is main, scrubbed, from static files", async (t) => {
+ if (filterRepoProblem) return t.skip(`git-filter-repo unavailable: ${filterRepoProblem}`);
+ const repo = sourceRepo();
+ const files = operatorFiles(`# the planted path\n/srv/${PLANTED}==>/home/user\n`, `i:${PLANTED}\n`);
+ const logs: string[] = [];
+ const o = opts(repo, files, logs);
+ const pub = o.publicDir!;
+ const site = path.dirname(pub);
+
+ assert.equal(await publishSource({ ...o, check: true }), 0, logs.join("\n"));
+ assert.match(logs.join("\n"), /check passed — would publish main .* nothing written/);
+ assert.ok(!existsSync(pub), "--check wrote nothing");
+ assert.ok(!existsSync(path.join(site, ".source-publish.json")));
+
+ logs.length = 0;
+ assert.equal(await publishSource(o), 0, logs.join("\n"));
+ assert.match(logs.join("\n"), /\[source\] audit clean: \d+ objects \(3 commits\), \d+ staged files against 3 denied literals; gitleaks skipped/);
+ assert.match(logs.join("\n"), /\[source\] published main [0-9a-f]{12} as [0-9a-f]{12}: \d+ files/);
+ const manifest = JSON.parse(readFileSync(path.join(pub, "source", "manifest.json"), "utf8"));
+ const sourceCommit = gitIn(repo, "rev-parse", "main");
+ assert.equal(manifest.version, 1);
+ assert.equal(manifest.branch, "main");
+ assert.equal(manifest.sourceCommit, sourceCommit);
+ assert.notEqual(manifest.mirrorHead, sourceCommit, "scrubbed history, different ids");
+ assert.equal(manifest.subject, "moved from /home/user");
+ assert.equal(manifest.cloneUrl, CLONE_URL);
+ assert.equal(manifest.treeHref, TREE_HREF);
+ assert.deepEqual(manifest.tree, { files: 4, dirs: 4, bytes: manifest.tree.bytes });
+ assert.ok(manifest.mirror.packs >= 1);
+ assert.equal(manifest.audit.commits, 3);
+ assert.equal(manifest.audit.gitleaks, "skipped");
+ assert.ok(!("rulesHash" in manifest), "the rules hash is never published");
+ assert.ok(existsSync(path.join(site, ".source-publish.json")), "…it is kept beside public/");
+
+ // The mirror: the allowlist and the dumb-HTTP files.
+ const mirror = path.join(pub, "source", MIRROR_DIR);
+ for (const f of ["config", "hooks", "description", "filter-repo", "logs", "info/exclude"]) {
+ assert.ok(!existsSync(path.join(mirror, f)), `${f} is never published`);
+ }
+ assert.equal(readFileSync(path.join(mirror, "HEAD"), "utf8"), "ref: refs/heads/main\n");
+ assert.equal(readFileSync(path.join(mirror, "info", "refs"), "utf8"), `${manifest.mirrorHead}\trefs/heads/main\n`);
+ assert.match(readFileSync(path.join(mirror, "objects", "info", "packs"), "utf8"), /^P pack-[0-9a-f]+\.pack$/m);
+ assert.ok(!existsSync(path.join(pub, "source", "index.html")), "the /source/ page keeps its route");
+
+ // The clone.
+ const clone = path.join(dir("clone"), "c");
+ execFileSync("git", ["clone", "-q", `file://${mirror}`, clone], { stdio: "pipe" });
+ assert.equal(gitIn(clone, "rev-parse", "HEAD"), manifest.mirrorHead);
+ assert.equal(gitIn(clone, "log", "-1", "--format=%s"), "moved from /home/user");
+ assert.equal(readFileSync(path.join(clone, "notes.txt"), "utf8"), "data lives at /home/user/x\n");
+ const revs = gitIn(clone, "rev-list", "--all").split("\n");
+ assert.equal(revs.length, 3);
+ assert.throws(
+ () => execFileSync("git", ["grep", "-F", PLANTED, ...revs], { cwd: clone, stdio: "pipe" }),
+ (e: { status?: number }) => e.status === 1,
+ "no revision holds the planted path",
+ );
+ assert.ok(!gitIn(clone, "log", "--all", "--format=%B%an%ae").includes(PLANTED));
+
+ // The tarball and its sidecar agree with the manifest and the bytes.
+ const tarball = path.join(pub, "downloads", path.basename(TARBALL_HREF));
+ const snapshot = JSON.parse(readFileSync(path.join(pub, "downloads", "snapshot.json"), "utf8"));
+ assert.equal(snapshot.sha256, sha256(tarball));
+ assert.equal(snapshot.sha256, manifest.tarball.sha256);
+ assert.equal(snapshot.bytes, statSync(tarball).size);
+ assert.equal(snapshot.commit, manifest.mirrorHead);
+ assert.equal(snapshot.subject, "moved from /home/user");
+
+ // The tree pages.
+ const tree = path.join(pub, "source", "tree");
+ for (const d of ["", "app", "app/[slug]", "fonts"]) assert.ok(existsSync(path.join(tree, d, "index.html")), d);
+ assert.match(readFileSync(path.join(tree, "fonts", "index.html"), "utf8"), /href="Archivo%5Bwdth%2Cwght%5D\.ttf"/);
+ assert.equal(readFileSync(path.join(tree, "notes.txt"), "utf8"), "data lives at /home/user/x\n");
+ assert.equal(readdirSync(o.scratchRoot!).length, 0, "the scratch dir is removed");
+
+ // Nothing changed: the next publish skips.
+ logs.length = 0;
+ assert.equal(await publishSource(o), 0);
+ assert.match(logs.join("\n"), /up to date at/);
+});
+
+test("a denied literal no rule removes: refused, nothing written, the report never prints it", async (t) => {
+ if (filterRepoProblem) return t.skip(`git-filter-repo unavailable: ${filterRepoProblem}`);
+ const repo = sourceRepo({ "keys.txt": `the ${SECRET} is here\n` });
+ const files = operatorFiles(`/srv/${PLANTED}==>/home/user\n`, `${SECRET}\n`);
+ const logs: string[] = [];
+ const o = opts(repo, files, logs);
+ const downloads = path.join(o.publicDir!, "downloads");
+ mkdirSync(downloads, { recursive: true });
+ writeFileSync(path.join(downloads, "snapshot.json"), "yesterday's");
+ const before = statSync(path.join(downloads, "snapshot.json")).mtimeMs;
+ assert.equal(await publishSource(o), 1);
+ const report = logs.join("\n");
+ assert.ok(!report.includes(SECRET), report);
+ assert.match(report, /AUDIT REFUSED: 1 hit in \d+ objects/);
+ assert.match(report, /#1 \(p…, len 13\): 1 in blob/);
+ assert.match(report, /blob [0-9a-f]{12} keys\.txt #1 \(p…, len 13\): the \[REDACTED\] is here\./);
+ assert.match(report, /add a rule to .*source-scrub\.txt or drop the file from history, then re-run\.$/);
+ assert.ok(!existsSync(path.join(o.publicDir!, "source")), "no manifest, no mirror");
+ assert.equal(readFileSync(path.join(downloads, "snapshot.json"), "utf8"), "yesterday's");
+ assert.equal(statSync(path.join(downloads, "snapshot.json")).mtimeMs, before);
+});
+
+test("--no-source's clear: the manifest, mirror, tree, tarball and skip key go; a linked downloads/ goes as a link", async () => {
+ const site = dir("clear");
+ const pub = path.join(site, "public");
+ mkdirSync(path.join(pub, "source", MIRROR_DIR), { recursive: true });
+ writeFileSync(path.join(pub, "source", "manifest.json"), "{}");
+ const elsewhere = dir("elsewhere");
+ writeFileSync(path.join(elsewhere, "snapshot.json"), "the other checkout's");
+ symlinkSync(elsewhere, path.join(pub, "downloads"));
+ writeFileSync(path.join(site, ".source-publish.json"), "{}");
+ const logs: string[] = [];
+ await clearPublishedSource({ paths: {} as Paths, publicDir: pub, onLog: (l) => logs.push(l) });
+ assert.ok(!existsSync(path.join(pub, "source")));
+ assert.ok(!existsSync(path.join(site, ".source-publish.json")));
+ assert.equal(lstatSync(path.join(pub, "downloads"), { throwIfNoEntry: false }), undefined);
+ assert.equal(readFileSync(path.join(elsewhere, "snapshot.json"), "utf8"), "the other checkout's");
+ assert.match(logs.join("\n"), /previously published source .* was removed/);
+});
diff --git a/common/publish/source.ts b/common/publish/source.ts
@@ -0,0 +1,792 @@
+// `archilyzer source publish` — the repo on the project site, read-only.
+//
+// The private repository is never rewritten. Every publish makes a FRESH bare
+// clone of its `main`, rewrites that copy with git-filter-repo (the operator's
+// scrub rules over file contents AND commit messages), repacks it for git's
+// dumb-HTTP protocol, and publishes three things under homepage/public:
+//
+// source/archilyzer.git/ a clonable mirror: HEAD, refs, info/refs,
+// objects/info/packs, the packs — static files only
+// source/tree/ the tracked files of main, raw, with an
+// index.html per directory (sourceTree.ts)
+// downloads/ archilyzer-source.tar.gz + snapshot.json, the
+// tarball the Downloads page has always offered
+// source/manifest.json written LAST: what was published, from what
+//
+// THE GATE. Before anything is staged for the site, every object of the
+// rewritten mirror, and then every staged file, is searched for every denied
+// literal (sourceAudit.ts): the operator's denylist plus every scrub rule's
+// left side. One hit and nothing is written; the report names the literal by
+// number, never by its bytes.
+//
+// OPERATOR-PRIVATE INPUTS live outside the repo, in
+// `${ARCHILYZER_CONFIG_DIR ?? ~/.config/archilyzer}/`: source-scrub.txt
+// (git-filter-repo `lhs==>rhs` lines; `<home dir>==>/home/user` is always
+// applied first) and source-denylist.txt (one literal per line, `i:` = any
+// case). A missing file is a refusal naming it. Nothing here names a user.
+//
+// `buildHomepage` runs this between compose and `next build`, so `archilyzer
+// build homepage`, the runbooks' home scripts and the editor's /sites homepage
+// jobs all publish it; an unchanged main with unchanged rules skips.
+
+import { createHash } from "node:crypto";
+import { createReadStream, existsSync } from "node:fs";
+import { cp, lstat, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { runChildIntoLog } from "../jobs/runChild";
+import { copyPublicFile, ownDir, writePublicFile } from "../bin/_publicFile";
+import { getPaths, type Paths } from "../lib/paths";
+import {
+ CLONE_URL,
+ MIRROR_DIR,
+ SOURCE_MANIFEST_VERSION,
+ TARBALL_HREF,
+ TREE_HREF,
+ parseSourceManifest,
+ type SourceManifest,
+} from "../lib/sourceManifest";
+import {
+ SourceRefusal,
+ auditBare,
+ auditFiles,
+ cleanGitEnv,
+ dedupeLiterals,
+ formatAuditReport,
+ maskLiterals,
+ onPath,
+ parseDenylist,
+ tildify,
+ type Literal,
+} from "./sourceAudit";
+import { writeTreeIndexes } from "./sourceTree";
+// Type-only: build.ts imports THIS module lazily, and must not load it (or the
+// AWS SDK this would drag in) at import time.
+import type { PublishOpts } from "./build";
+
+export { SourceRefusal } from "./sourceAudit";
+
+/** The one branch mirrored (an operator decision: no other refs, no tags). */
+export const SOURCE_BRANCH = "main";
+
+/** What `pipx run` fetches when `git filter-repo` is not installed. */
+export const FILTER_REPO_PIPX_SPEC = "git-filter-repo==2.47.0";
+export const FILTER_REPO_INSTALL = "pipx install git-filter-repo";
+
+/** The home-directory rule's replacement (always the first scrub rule). */
+export const HOME_REPLACEMENT = "/home/user";
+
+// Cloudflare Pages allows 20,000 files per deployment and 25 MiB per file;
+// the step refuses well inside both, leaving the rest of the site its room.
+export const MAX_FILES = 15_000;
+export const MAX_FILE_BYTES = 24 * 1024 * 1024;
+
+// Packs are split at this size (under the per-file cap, with room to grow).
+const PACK_SIZE = "20m";
+
+const TARBALL_NAME = path.basename(TARBALL_HREF);
+
+export type SourcePublishOpts = PublishOpts & {
+ // Rebuild even when main and the rules are unchanged.
+ force?: boolean;
+ // Build, audit and count everything, then write NOTHING.
+ check?: boolean;
+ // Leave the scratch dir (the rewritten bare clone, the stage) for a look.
+ keepScratch?: boolean;
+ // The repository to mirror. Default: this checkout's git COMMON dir, so a
+ // worktree build mirrors the primary's main.
+ sourceRepo?: string;
+ // Default: HOMEPAGE_PUBLIC_DIR, else <repo>/homepage/public.
+ publicDir?: string;
+ scrubFile?: string;
+ denylistFile?: string;
+ // The filter-repo argv; default: resolveFilterRepo().
+ filterRepo?: string[] | null;
+ // The gitleaks binary; null skips the secret scan (tests). Default "gitleaks".
+ gitleaks?: string | null;
+ // Where the scratch dir is made. Default: paths.sourceScratchDir.
+ scratchRoot?: string;
+ // The environment the children run in (PATH decides which tools). Default:
+ // process.env.
+ env?: NodeJS.ProcessEnv;
+ now?: () => Date;
+ // The home dir the built-in rule scrubs. Default: os.homedir().
+ homeDir?: string;
+};
+
+// ── the operator's files ────────────────────────────────────────────────────
+
+export type ScrubRules = {
+ // replace.txt as filter-repo will read it: the built-in rule first, then
+ // the operator's rules in order (comments and blank lines dropped —
+ // filter-repo itself would treat a `#` line as a literal to replace).
+ lines: string[];
+ // Every rule's LITERAL left side: denied, exact case, by implication.
+ denied: string[];
+};
+
+/**
+ * The scrub file's text as rules. A line is `lhs==>rhs` (split at the LAST
+ * `==>`, as filter-repo splits it), `literal:lhs==>rhs`, `regex:…==>…` or
+ * `glob:…==>…`; a line with no `==>` is replaced by filter-repo's
+ * `***REMOVED***`. Lines whose first non-blank character is `#` are comments.
+ */
+export function parseScrubRules(text: string, homeDir: string): ScrubRules {
+ const lines: string[] = [];
+ // A home dir of `/` (a container user) would scrub every slash, and one
+ // that IS the replacement would deny the replacement itself.
+ if (homeDir.length > 1 && homeDir !== HOME_REPLACEMENT) {
+ lines.push(`${homeDir}==>${HOME_REPLACEMENT}`);
+ }
+ for (const raw of text.split("\n")) {
+ const line = raw.replace(/\r$/, "");
+ const t = line.trim();
+ if (t === "" || t.startsWith("#")) continue;
+ lines.push(line);
+ }
+ const denied: string[] = [];
+ for (const line of lines) {
+ const i = line.lastIndexOf("==>");
+ let lhs = i === -1 ? line : line.slice(0, i);
+ if (lhs.startsWith("regex:") || lhs.startsWith("glob:")) continue;
+ if (lhs.startsWith("literal:")) lhs = lhs.slice("literal:".length);
+ if (lhs) denied.push(lhs);
+ }
+ return { lines, denied };
+}
+
+export type SourceRules = {
+ scrub: ScrubRules;
+ literals: Literal[];
+ // Changes whenever a rule or a literal does: part of the skip key. Never
+ // published (a hash of the denylist would confirm a guess at it).
+ rulesHash: string;
+};
+
+async function readOperatorFile(file: string, what: string, how: string): Promise<string> {
+ try {
+ return await readFile(file, "utf8");
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+ throw new SourceRefusal(`no ${what} at ${tildify(file)} — ${how}`);
+ }
+ throw err;
+ }
+}
+
+/** Both operator files, parsed, with the literal list the gate searches for. */
+export async function loadSourceRules(opts: {
+ scrubFile: string;
+ denylistFile: string;
+ homeDir?: string;
+}): Promise<SourceRules> {
+ const scrubText = await readOperatorFile(
+ opts.scrubFile,
+ "scrub rules",
+ "create it (git-filter-repo `lhs==>rhs` lines; the home-directory rule is built in, so it may be empty) or point SOURCE_SCRUB_FILE at one",
+ );
+ const denyText = await readOperatorFile(
+ opts.denylistFile,
+ "denylist",
+ "create it (one literal per line, `i:` for any case; every scrub rule's left side is denied too, so it may be empty) or point SOURCE_DENYLIST_FILE at one",
+ );
+ const scrub = parseScrubRules(scrubText, opts.homeDir ?? os.homedir());
+ const literals = dedupeLiterals([
+ ...parseDenylist(denyText),
+ ...scrub.denied.map((d) => ({ bytes: Buffer.from(d, "utf8"), ci: false })),
+ ]);
+ const rulesHash = createHash("sha256")
+ .update(
+ JSON.stringify({
+ v: SOURCE_MANIFEST_VERSION,
+ rules: scrub.lines,
+ literals: literals.map((l) => `${l.ci ? "i" : "x"}:${l.bytes.toString("hex")}`),
+ }),
+ )
+ .digest("hex");
+ return { scrub, literals, rulesHash };
+}
+
+// ── children ────────────────────────────────────────────────────────────────
+
+type Ctx = {
+ onLog: (line: string) => void;
+ signal: AbortSignal;
+ env: NodeJS.ProcessEnv;
+ // Every echoed or quoted child line is masked with these once they are known.
+ literals: readonly Literal[];
+};
+
+class Cancelled extends Error {}
+
+/**
+ * One child through runChildIntoLog, with a timeout. Returns its combined
+ * output. A non-zero exit or a timeout is a refusal quoting its last lines
+ * (masked); a cancel throws Cancelled.
+ */
+async function run(
+ ctx: Ctx,
+ command: string,
+ args: string[],
+ o: { cwd: string; timeoutMs: number; echo?: boolean; allowFail?: boolean },
+): Promise<{ code: number; out: string[] }> {
+ const out: string[] = [];
+ const timeout = AbortSignal.timeout(o.timeoutMs);
+ const code = await runChildIntoLog(
+ (line) => {
+ out.push(line);
+ if (o.echo) ctx.onLog(`[source] ${maskLiterals(line, ctx.literals)}`);
+ },
+ AbortSignal.any([ctx.signal, timeout]),
+ { command, args, cwd: o.cwd, env: ctx.env },
+ );
+ if (ctx.signal.aborted) throw new Cancelled();
+ // `git --git-dir <path> repack …` is named by its verb, not the path.
+ const what = `${command} ${(args[0] === "--git-dir" ? args.slice(2, 3) : args.slice(0, 2)).join(" ")}`;
+ if (timeout.aborted) {
+ throw new SourceRefusal(`${what} timed out after ${Math.round(o.timeoutMs / 1000)} s`);
+ }
+ if (code !== 0 && !o.allowFail) {
+ const tail = out.slice(-3).map((l) => maskLiterals(l, ctx.literals)).join(" / ");
+ throw new SourceRefusal(`${what} exited ${code}${tail ? `: ${tail}` : ""}`);
+ }
+ return { code, out };
+}
+
+const lastLine = (out: string[]) => (out.filter((l) => l.trim()).at(-1) ?? "").trim();
+const OID = /^[0-9a-f]{40}(?:[0-9a-f]{24})?$/;
+
+async function revParse(ctx: Ctx, gitDir: string, rev: string): Promise<string> {
+ const { out } = await run(ctx, "git", ["--git-dir", gitDir, "rev-parse", "--verify", rev], {
+ cwd: gitDir,
+ timeoutMs: 30_000,
+ });
+ const oid = lastLine(out);
+ if (!OID.test(oid)) throw new SourceRefusal(`git rev-parse ${rev}: not an object id`);
+ return oid;
+}
+
+export type FilterRepoChoice = { argv: string[]; label: string; version: string };
+
+/**
+ * Which git-filter-repo runs: an installed `git filter-repo`, else `pipx run`
+ * of the pinned version (network on first use), else a refusal with the
+ * install line.
+ */
+export async function resolveFilterRepo(opts: {
+ env?: NodeJS.ProcessEnv;
+ signal?: AbortSignal;
+ onLog?: (line: string) => void;
+}): Promise<FilterRepoChoice> {
+ const ctx: Ctx = {
+ onLog: opts.onLog ?? (() => {}),
+ signal: opts.signal ?? new AbortController().signal,
+ env: cleanGitEnv(opts.env ?? process.env),
+ literals: [],
+ };
+ const cwd = os.tmpdir();
+ const installed = await run(ctx, "git", ["filter-repo", "--version"], {
+ cwd,
+ timeoutMs: 30_000,
+ allowFail: true,
+ });
+ if (installed.code === 0) {
+ return { argv: ["git", "filter-repo"], label: "git filter-repo", version: lastLine(installed.out) };
+ }
+ if (!onPath("pipx", ctx.env.PATH)) {
+ throw new SourceRefusal(
+ `git-filter-repo is not installed and pipx is not on PATH — install it once: \`${FILTER_REPO_INSTALL}\` (pipx comes from your OS's packages)`,
+ );
+ }
+ const argv = ["pipx", "run", "--spec", FILTER_REPO_PIPX_SPEC, "git-filter-repo"];
+ const viaPipx = await run(ctx, argv[0], [...argv.slice(1), "--version"], {
+ cwd,
+ timeoutMs: 300_000,
+ allowFail: true,
+ });
+ if (viaPipx.code !== 0) {
+ throw new SourceRefusal(
+ `\`pipx run --spec ${FILTER_REPO_PIPX_SPEC}\` failed (exit ${viaPipx.code}; it needs the network on first use) — install it once: \`${FILTER_REPO_INSTALL}\``,
+ );
+ }
+ return {
+ argv,
+ label: `pipx run --spec ${FILTER_REPO_PIPX_SPEC} git-filter-repo`,
+ version: lastLine(viaPipx.out),
+ };
+}
+
+// ── helpers ─────────────────────────────────────────────────────────────────
+
+function terminalLog(line: string): void {
+ process.stdout.write(line.endsWith("\n") ? line : `${line}\n`);
+}
+
+function sha256File(file: string): Promise<string> {
+ return new Promise((resolve, reject) => {
+ const h = createHash("sha256");
+ createReadStream(file)
+ .on("data", (c) => h.update(c))
+ .on("error", reject)
+ .on("end", () => resolve(h.digest("hex")));
+ });
+}
+
+async function walkFiles(dir: string): Promise<Array<{ rel: string; bytes: number }>> {
+ const out: Array<{ rel: string; bytes: number }> = [];
+ const walk = async (rel: string) => {
+ for (const ent of await readdir(path.join(dir, rel), { withFileTypes: true })) {
+ const r = rel ? `${rel}/${ent.name}` : ent.name;
+ if (ent.isDirectory()) await walk(r);
+ else out.push({ rel: r, bytes: (await stat(path.join(dir, r))).size });
+ }
+ };
+ await walk("");
+ return out;
+}
+
+const mb = (bytes: number) => (bytes / (1024 * 1024)).toFixed(1);
+
+/**
+ * Why `files` may not be published on Pages, as one sentence — or null. The
+ * step's limits sit inside the host's: 15,000 files of its 20,000 per
+ * deployment (the rest of the site needs room), 24 MiB of its 25 MiB per file.
+ */
+export function limitProblem(files: ReadonlyArray<{ rel: string; bytes: number }>): string | null {
+ if (files.length > MAX_FILES) {
+ return `${files.length} files to publish, over the step's limit of ${MAX_FILES} (Pages allows 20,000 per deployment)`;
+ }
+ const big = files.find((f) => f.bytes > MAX_FILE_BYTES);
+ if (big) {
+ return `${big.rel} is ${mb(big.bytes)} MiB, over the step's limit of ${mb(MAX_FILE_BYTES)} MiB (Pages allows 25 MiB per file)`;
+ }
+ return null;
+}
+
+/** Where publishSource writes, for a given checkout. */
+export function sourcePublicDir(paths: Paths, override?: string): string {
+ return override ?? process.env.HOMEPAGE_PUBLIC_DIR ?? path.join(paths.monorepoRoot, "homepage", "public");
+}
+
+// The skip key, kept BESIDE the public dir, never in it: it holds the rules
+// hash, and public/ is deployed.
+function statePath(publicDir: string): string {
+ return path.join(path.dirname(publicDir), ".source-publish.json");
+}
+
+type PublishState = { sourceCommit: string; rulesHash: string };
+
+async function readJson(file: string): Promise<unknown> {
+ try {
+ return JSON.parse(await readFile(file, "utf8"));
+ } catch {
+ return null;
+ }
+}
+
+/** The last published manifest in `publicDir`, or null. */
+export async function readPublishedManifest(publicDir: string): Promise<SourceManifest | null> {
+ return parseSourceManifest(await readJson(path.join(publicDir, "source", "manifest.json")));
+}
+
+// ── the step ────────────────────────────────────────────────────────────────
+
+/**
+ * Publish the source (see the header). Returns 0 when published, skipped or
+ * checked, 1 when refused or cancelled; the refusal's reason (and the audit's
+ * redacted report) is in the log. Throws only on a bug or an I/O failure.
+ */
+export async function publishSource(opts: SourcePublishOpts = {}): Promise<number> {
+ const paths = opts.paths ?? getPaths();
+ const onLog = opts.onLog ?? terminalLog;
+ const signal = opts.signal ?? new AbortController().signal;
+ const ctx: Ctx = { onLog, signal, env: cleanGitEnv(opts.env ?? process.env), literals: [] };
+ try {
+ return await publish(opts, paths, ctx);
+ } catch (err) {
+ if (err instanceof Cancelled || signal.aborted) {
+ onLog("[source] cancelled — nothing published");
+ return 1;
+ }
+ if (err instanceof SourceRefusal) {
+ onLog(`[source] REFUSED: ${err.message}`);
+ return 1;
+ }
+ throw err;
+ }
+}
+
+async function publish(opts: SourcePublishOpts, paths: Paths, ctx: Ctx): Promise<number> {
+ const { onLog } = ctx;
+ const started = Date.now();
+ const publicDir = sourcePublicDir(paths, opts.publicDir);
+ const pubSource = path.join(publicDir, "source");
+ const pubDownloads = path.join(publicDir, "downloads");
+
+ // 1. The private main.
+ const sourceRepo =
+ opts.sourceRepo ??
+ lastLine(
+ (
+ await run(ctx, "git", ["rev-parse", "--path-format=absolute", "--git-common-dir"], {
+ cwd: paths.monorepoRoot,
+ timeoutMs: 30_000,
+ })
+ ).out,
+ );
+ const sourceCommit = await revParse(ctx, sourceRepo, `refs/heads/${SOURCE_BRANCH}^{commit}`);
+
+ // 2. The operator's rules.
+ const rules = await loadSourceRules({
+ scrubFile: opts.scrubFile ?? paths.sourceScrubFile,
+ denylistFile: opts.denylistFile ?? paths.sourceDenylistFile,
+ homeDir: opts.homeDir,
+ });
+ ctx.literals = rules.literals;
+
+ // 3. Nothing changed: skip.
+ if (!opts.force && !opts.check) {
+ const manifest = await readPublishedManifest(publicDir);
+ const state = (await readJson(statePath(publicDir))) as PublishState | null;
+ if (
+ manifest?.sourceCommit === sourceCommit &&
+ state?.sourceCommit === sourceCommit &&
+ state.rulesHash === rules.rulesHash &&
+ existsSync(path.join(pubSource, MIRROR_DIR, "info", "refs")) &&
+ existsSync(path.join(pubDownloads, TARBALL_NAME))
+ ) {
+ onLog(`[source] up to date at ${sourceCommit.slice(0, 12)}; skipping (--force to rebuild)`);
+ return 0;
+ }
+ }
+ if (existsSync(path.join(pubSource, "index.html"))) {
+ throw new SourceRefusal(
+ `${tildify(path.join(pubSource, "index.html"))} exists and would replace the /source/ page — remove it`,
+ );
+ }
+
+ // 4. The tools.
+ const filterRepo: FilterRepoChoice =
+ opts.filterRepo && opts.filterRepo.length > 0
+ ? { argv: opts.filterRepo, label: opts.filterRepo.join(" "), version: "(given)" }
+ : await resolveFilterRepo({ env: ctx.env, signal: ctx.signal });
+ const gitVersion = lastLine((await run(ctx, "git", ["--version"], { cwd: os.tmpdir(), timeoutMs: 30_000 })).out)
+ .replace(/^git version /, "");
+ onLog(`[source] main ${sourceCommit.slice(0, 12)}; git ${gitVersion}; ${filterRepo.label} ${filterRepo.version}`);
+
+ const scratchRoot = opts.scratchRoot ?? paths.sourceScratchDir;
+ await mkdir(scratchRoot, { recursive: true });
+ const scratch = await mkdtemp(path.join(scratchRoot, "archilyzer-source-"));
+ try {
+ const bare = path.join(scratch, "bare");
+
+ // 5. A fresh bare clone of main alone. --no-local: through upload-pack, not
+ // a copy of the object dir (which would carry every loose leftover).
+ await run(
+ ctx,
+ "git",
+ ["clone", "--no-local", "--bare", "--single-branch", "--no-tags", "--branch", SOURCE_BRANCH, "--quiet", sourceRepo, bare],
+ { cwd: scratch, timeoutMs: 120_000 },
+ );
+ await run(ctx, "git", ["--git-dir", bare, "remote", "remove", "origin"], { cwd: bare, timeoutMs: 30_000 });
+
+ // 6. The rewrite. --force: filter-repo wants a fresh clone with an origin.
+ const replace = path.join(scratch, "replace.txt");
+ await writeFile(replace, rules.scrub.lines.join("\n") + "\n");
+ onLog(`[source] rewriting history (${rules.scrub.lines.length} scrub rule${rules.scrub.lines.length === 1 ? "" : "s"})…`);
+ await run(
+ ctx,
+ filterRepo.argv[0],
+ [
+ ...filterRepo.argv.slice(1),
+ "--force",
+ "--quiet",
+ "--replace-refs",
+ "delete-no-add",
+ "--replace-text",
+ replace,
+ "--replace-message",
+ replace,
+ ],
+ { cwd: bare, timeoutMs: 900_000, echo: true },
+ );
+ // commit-map / ref-map hold the PRIVATE ids.
+ await rm(path.join(bare, "filter-repo"), { recursive: true, force: true });
+
+ // 7. Packed for dumb HTTP.
+ const g = (args: string[], timeoutMs = 60_000) =>
+ run(ctx, "git", ["--git-dir", bare, ...args], { cwd: bare, timeoutMs });
+ await g(["repack", "-a", "-d", "-q", `--max-pack-size=${PACK_SIZE}`], 300_000);
+ await g(["prune-packed"]);
+ await g(["pack-refs", "--all"]);
+ await g(["update-server-info"]);
+ const counts = (await g(["count-objects", "-v"])).out;
+ const loose = Number(/^count:\s*(\d+)/m.exec(counts.join("\n"))?.[1] ?? NaN);
+ if (loose !== 0) throw new SourceRefusal(`the repacked mirror still has ${loose} loose objects`);
+ const packsList = await readFile(path.join(bare, "objects", "info", "packs"), "utf8").catch(() => "");
+ const packs = packsList.split("\n").filter((l) => l.startsWith("P ")).length;
+ if (packs === 0) throw new SourceRefusal("the repacked mirror lists no packs in objects/info/packs");
+ const refs = (await g(["for-each-ref", "--format=%(refname)"])).out.filter((l) => l.trim());
+ if (refs.length !== 1 || refs[0] !== `refs/heads/${SOURCE_BRANCH}`) {
+ throw new SourceRefusal(`the mirror must hold refs/heads/${SOURCE_BRANCH} alone, and holds ${refs.length} refs`);
+ }
+ const head = (await readFile(path.join(bare, "HEAD"), "utf8")).trim();
+ if (head !== `ref: refs/heads/${SOURCE_BRANCH}`) {
+ throw new SourceRefusal(`the mirror's HEAD is not refs/heads/${SOURCE_BRANCH}`);
+ }
+
+ // 8. What it became.
+ const mirrorHead = await revParse(ctx, bare, `refs/heads/${SOURCE_BRANCH}`);
+ const subject = lastLine((await g(["log", "-1", "--format=%s", `refs/heads/${SOURCE_BRANCH}`])).out);
+
+ // 9. THE GATE, over every object.
+ onLog(`[source] auditing ${mirrorHead.slice(0, 12)} for ${rules.literals.length} denied literals…`);
+ const audit = await auditBare(bare, rules.literals, {
+ scratch,
+ onLog,
+ signal: ctx.signal,
+ gitleaks: opts.gitleaks === undefined ? "gitleaks" : opts.gitleaks,
+ env: ctx.env,
+ });
+ const scrubFile = opts.scrubFile ?? paths.sourceScrubFile;
+ if (audit.hits.length > 0) {
+ for (const l of formatAuditReport(audit, rules.literals, { scrubFile })) onLog(l);
+ return 1;
+ }
+
+ const now = opts.now?.() ?? new Date();
+ const generatedAt = now.toISOString();
+ const stage = path.join(scratch, "stage");
+ const stageSource = path.join(stage, "source");
+ const stageMirror = path.join(stageSource, MIRROR_DIR);
+ const stageTree = path.join(stageSource, "tree");
+ const stageDownloads = path.join(stage, "downloads");
+ await mkdir(stageTree, { recursive: true });
+ await mkdir(stageDownloads, { recursive: true });
+
+ // 10. The raw tree, with its directory pages.
+ const treeTar = path.join(scratch, "tree.tar");
+ await g(["archive", "--format=tar", "-o", treeTar, `refs/heads/${SOURCE_BRANCH}`], 120_000);
+ await run(ctx, "tar", ["-xf", treeTar, "-C", stageTree], { cwd: scratch, timeoutMs: 120_000 });
+ await rm(treeTar, { force: true });
+ const tree = await writeTreeIndexes(stageTree, { mirrorHead, generatedAt });
+
+ // 11. The tarball and its sidecar (the Snapshot shape the Downloads page reads).
+ const tarball = path.join(stageDownloads, TARBALL_NAME);
+ await g(
+ ["archive", "--format=tar.gz", "-9", "--prefix=archilyzer/", "-o", tarball, `refs/heads/${SOURCE_BRANCH}`],
+ 120_000,
+ );
+ const tarBytes = (await stat(tarball)).size;
+ const tarSha = await sha256File(tarball);
+ const snapshotText =
+ JSON.stringify(
+ { generatedAt, commit: mirrorHead, subject, bytes: tarBytes, sha256: tarSha },
+ null,
+ 2,
+ ) + "\n";
+ await writeFile(path.join(stageDownloads, "snapshot.json"), snapshotText);
+
+ // 12. The mirror, from an ALLOWLIST: never config (it names the clone's
+ // origin path), hooks/, description, logs/, filter-repo/, *.rev, *.bitmap.
+ // refs/heads/<branch> is written beside packed-refs so the directory is a
+ // git dir to git itself too (a file:// clone, `source audit`): git wants a
+ // refs/ directory, and an empty one would not survive the deploy.
+ await mkdir(path.join(stageMirror, "info"), { recursive: true });
+ await mkdir(path.join(stageMirror, "objects", "info"), { recursive: true });
+ await mkdir(path.join(stageMirror, "objects", "pack"), { recursive: true });
+ await mkdir(path.join(stageMirror, "refs", "heads"), { recursive: true });
+ for (const f of ["HEAD", "packed-refs", "info/refs", "objects/info/packs"]) {
+ await cp(path.join(bare, f), path.join(stageMirror, f));
+ }
+ await writeFile(path.join(stageMirror, "refs", "heads", SOURCE_BRANCH), `${mirrorHead}\n`);
+ for (const f of await readdir(path.join(bare, "objects", "pack"))) {
+ if (/^pack-[0-9a-f]+\.(pack|idx)$/.test(f)) {
+ await cp(path.join(bare, "objects", "pack", f), path.join(stageMirror, "objects", "pack", f));
+ }
+ }
+ const mirrorFiles = await walkFiles(stageMirror);
+
+ // The manifest, staged with the rest so the file sweep reads it too.
+ const staged = await walkFiles(stage);
+ const manifest: SourceManifest = {
+ version: SOURCE_MANIFEST_VERSION,
+ generatedAt,
+ branch: SOURCE_BRANCH,
+ sourceCommit,
+ mirrorHead,
+ subject,
+ files: staged.length,
+ bytes: staged.reduce((n, f) => n + f.bytes, 0),
+ mirror: {
+ files: mirrorFiles.length,
+ bytes: mirrorFiles.reduce((n, f) => n + f.bytes, 0),
+ packs,
+ },
+ tree,
+ tarball: { href: TARBALL_HREF, bytes: tarBytes, sha256: tarSha },
+ cloneUrl: CLONE_URL,
+ treeHref: TREE_HREF,
+ audit: {
+ literals: rules.literals.length,
+ objects: audit.objects,
+ commits: audit.commits,
+ gitleaks: audit.gitleaks === "clean" ? "clean" : "skipped",
+ },
+ tools: { git: gitVersion, filterRepo: `${filterRepo.label} ${filterRepo.version}` },
+ };
+ const manifestText = JSON.stringify(manifest, null, 2) + "\n";
+ await writeFile(path.join(stageSource, "manifest.json"), manifestText);
+
+ // 13. THE GATE, over every staged file and path (packs excepted: step 9
+ // read their objects).
+ await auditFiles(stage, rules.literals, { result: audit });
+ if (audit.hits.length > 0) {
+ for (const l of formatAuditReport(audit, rules.literals, { scrubFile })) onLog(l);
+ return 1;
+ }
+ for (const l of formatAuditReport(audit, rules.literals, { scrubFile })) onLog(l);
+
+ // 14. The host's limits.
+ const all = await walkFiles(stage);
+ const tooMuch = limitProblem(all);
+ if (tooMuch) throw new SourceRefusal(maskLiterals(tooMuch, rules.literals));
+ const totalBytes = all.reduce((n, f) => n + f.bytes, 0);
+ const summary =
+ `main ${sourceCommit.slice(0, 12)} as ${mirrorHead.slice(0, 12)}: ${all.length} files, ${mb(totalBytes)} MB ` +
+ `(mirror ${packs} pack${packs === 1 ? "" : "s"}, tree ${tree.dirs} dirs), tarball ${mb(tarBytes)} MB sha256 ${tarSha.slice(0, 12)}`;
+
+ // 15. --check writes nothing.
+ if (opts.check) {
+ onLog(`[source] check passed — would publish ${summary}; nothing written (${elapsed(started)})`);
+ return 0;
+ }
+
+ // 16. Install, link-safe. The manifest goes first and comes back last: a
+ // crash mid-copy leaves the page's empty state, never a manifest over a
+ // half-written tree.
+ await ownDir(pubSource);
+ await rm(path.join(pubSource, "manifest.json"), { force: true });
+ await rm(path.join(pubSource, MIRROR_DIR), { recursive: true, force: true });
+ await rm(path.join(pubSource, "tree"), { recursive: true, force: true });
+ await cp(stageMirror, path.join(pubSource, MIRROR_DIR), { recursive: true });
+ await cp(stageTree, path.join(pubSource, "tree"), { recursive: true });
+ await ownDir(pubDownloads);
+ await copyPublicFile(tarball, path.join(pubDownloads, TARBALL_NAME));
+ await writePublicFile(path.join(pubDownloads, "snapshot.json"), snapshotText);
+ const state: PublishState = { sourceCommit, rulesHash: rules.rulesHash };
+ await writeFile(statePath(publicDir), JSON.stringify(state, null, 2) + "\n");
+ await writePublicFile(path.join(pubSource, "manifest.json"), manifestText);
+
+ // 17.
+ onLog(`[source] published ${summary} (${elapsed(started)})`);
+ return 0;
+ } finally {
+ if (opts.keepScratch) onLog(`[source] scratch kept at ${scratch}`);
+ else await rm(scratch, { recursive: true, force: true });
+ }
+}
+
+function elapsed(started: number): string {
+ return `${Math.round((Date.now() - started) / 1000)} s`;
+}
+
+/**
+ * `build homepage --no-source`: remove what an earlier publish left (the
+ * manifest first, so the page never describes a half-removed tree), because
+ * it was audited against the rules of ITS day. The pages then show their
+ * empty states. Link-safe like the install: a linked directory is replaced,
+ * never followed.
+ */
+export async function clearPublishedSource(
+ opts: PublishOpts & { publicDir?: string } = {},
+): Promise<void> {
+ const paths = opts.paths ?? getPaths();
+ const onLog = opts.onLog ?? terminalLog;
+ const publicDir = sourcePublicDir(paths, opts.publicDir);
+ const pubSource = path.join(publicDir, "source");
+ const pubDownloads = path.join(publicDir, "downloads");
+ const had = existsSync(path.join(pubSource, "manifest.json")) || existsSync(path.join(pubDownloads, TARBALL_NAME));
+ await rm(path.join(pubSource, "manifest.json"), { force: true });
+ await rm(statePath(publicDir), { force: true });
+ // fs.rm reads the path with lstat: a linked public/source goes as a link.
+ await rm(pubSource, { recursive: true, force: true });
+ // A linked downloads/ (a worktree's, into the primary) is dropped as a link,
+ // never reached through: its files are the other checkout's.
+ const linked = await lstat(pubDownloads).then((s) => s.isSymbolicLink(), () => false);
+ if (linked) {
+ await rm(pubDownloads, { force: true });
+ } else {
+ await rm(path.join(pubDownloads, "snapshot.json"), { force: true });
+ await rm(path.join(pubDownloads, TARBALL_NAME), { force: true });
+ }
+ onLog(
+ had
+ ? "[notice] --no-source: the previously published source (mirror, tree, tarball) was removed — this build ships none.\n"
+ : "[notice] --no-source: no source published in this build.\n",
+ );
+}
+
+// ── `archilyzer source audit` ───────────────────────────────────────────────
+
+/**
+ * The gate alone, over any git dir — by default the published mirror; the
+ * rollout runs it on a LIVE clone. 0 clean, 1 hits (the redacted report is
+ * logged) or refused.
+ */
+export async function auditSource(
+ opts: PublishOpts & {
+ gitDir?: string;
+ publicDir?: string;
+ scrubFile?: string;
+ denylistFile?: string;
+ gitleaks?: string | null;
+ scratchRoot?: string;
+ env?: NodeJS.ProcessEnv;
+ homeDir?: string;
+ } = {},
+): Promise<number> {
+ const paths = opts.paths ?? getPaths();
+ const onLog = opts.onLog ?? terminalLog;
+ const signal = opts.signal ?? new AbortController().signal;
+ const env = cleanGitEnv(opts.env ?? process.env);
+ const gitDir = path.resolve(
+ opts.gitDir ?? path.join(sourcePublicDir(paths, opts.publicDir), "source", MIRROR_DIR),
+ );
+ const scrubFile = opts.scrubFile ?? paths.sourceScrubFile;
+ let scratch: string | null = null;
+ try {
+ const rules = await loadSourceRules({
+ scrubFile,
+ denylistFile: opts.denylistFile ?? paths.sourceDenylistFile,
+ homeDir: opts.homeDir,
+ });
+ const ctx: Ctx = { onLog, signal, env, literals: rules.literals };
+ const head = await revParse(ctx, gitDir, "HEAD");
+ const scratchRoot = opts.scratchRoot ?? paths.sourceScratchDir;
+ await mkdir(scratchRoot, { recursive: true });
+ scratch = await mkdtemp(path.join(scratchRoot, "archilyzer-source-audit-"));
+ onLog(`[source] auditing ${tildify(gitDir)} (HEAD ${head.slice(0, 12)}) for ${rules.literals.length} denied literals…`);
+ const audit = await auditBare(gitDir, rules.literals, {
+ scratch,
+ onLog,
+ signal,
+ gitleaks: opts.gitleaks === undefined ? "gitleaks" : opts.gitleaks,
+ env,
+ });
+ for (const l of formatAuditReport(audit, rules.literals, { scrubFile })) onLog(l);
+ return audit.hits.length > 0 ? 1 : 0;
+ } catch (err) {
+ if (err instanceof Cancelled || signal.aborted) {
+ onLog("[source] cancelled");
+ return 1;
+ }
+ if (err instanceof SourceRefusal) {
+ onLog(`[source] REFUSED: ${err.message}`);
+ return 1;
+ }
+ throw err;
+ } finally {
+ if (scratch) await rm(scratch, { recursive: true, force: true });
+ }
+}
