commit 296b8a0418074971f8dd415228800e7339bc15ce
parent 52858224b72bd9eede5b47a6f89ba2a0624cb062
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 28 Sep 2026 14:55:19 -0400
common: refusals masked, a digest of every published file and the gitleaks identity in the key (re-review R2-L1, L2, L4), and the key's tests bite (R2-L3)
R2-L1: `[source] REFUSED: …` goes through maskLiterals with the loaded literals
(publish and audit alike), as do the kept-scratch path and the report's scrub-
file path: a literal spanning path components (`a/b`), invisible to the object
walk, was printed in full by the tree's symlink / index.html / 404.html
refusal. Child stderr was already masked where quoted. R2-L2: the state
carries `contentDigest`, sourceDigest() over every published file (sorted
path, size, streamed sha256; a symlink or a missing piece only differs); the
skip recomputes it over public/, and the deploy check over out/
(`source/archilyzer.git/**`, `source/tree/**`, `source/manifest.json`, the
tarball, snapshot.json) — a mixed or edited out/ is refused. R2-L4: the state
carries gitleaksIdentity() ("skipped", "absent", or the version line plus the
binary's sha256, since a distribution build can print one line for every
release); the skip and the deploy check compare it. SOURCE_STEP_VERSION → 3.
R2-L3: every part of the skip key is tested through a path that reaches the
skip; the rules hash is shown to move with the step version (rulesHashOf); the
deploy check's mirror-head and gitleaks comparisons each refuse by their own
sentence; a swapped tree file and a flipped pack byte are refused.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
3 files changed, 258 insertions(+), 35 deletions(-)
diff --git a/common/publish/source.test.ts b/common/publish/source.test.ts
@@ -3,6 +3,7 @@ import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { execFileSync } from "node:child_process";
import {
+ chmodSync,
cpSync,
existsSync,
lstatSync,
@@ -23,6 +24,7 @@ import {
MAX_FILES,
MAX_FILE_BYTES,
NO_REPOSITORY,
+ SOURCE_STEP_VERSION,
SourceRefusal,
clearPublishedSource,
limitProblem,
@@ -30,8 +32,11 @@ import {
parseScrubRules,
publishSource,
publishedSourceProblem,
+ gitleaksIdentity,
resolveFilterRepo,
+ rulesHashOf,
scratchRootProblem,
+ sourceDigest,
type SourcePublishOpts,
} from "./source";
@@ -217,37 +222,66 @@ test("the limits: 15,000 files and 24 MiB a file, inside Pages' 20,000 and 25 Mi
assert.equal(limitProblem(many.slice(1)), null);
});
-test("skip: an unchanged main with unchanged rules and tools does nothing; a changed rule refuses AND withdraws the last publish", async () => {
+test("skip: an unchanged main with unchanged rules, tools and files does nothing; each part of the key defeats it; a refusal withdraws", async () => {
const repo = sourceRepo();
const files = operatorFiles(`/srv/${PLANTED}==>/home/user\n`, "");
const logs: string[] = [];
- // filter-repo is `false`: reaching it would refuse, so a 0 proves the skip.
+ // filter-repo is `false`: reaching it would refuse, so a 0 proves the skip
+ // and a 1 proves the key did NOT match (review R2-L3: every case below goes
+ // through the skip, never `check: true`, which skips nothing).
const o = opts(repo, files, logs, { filterRepo: ["false"] });
const pub = o.publicDir!;
const sourceCommit = gitIn(repo, "rev-parse", "main");
const rules = await loadSourceRules({ ...files, homeDir: HOME });
const mirrorHead = "a".repeat(40);
- mkdirSync(path.join(pub, "source", MIRROR_DIR, "info"), { recursive: true });
- mkdirSync(path.join(pub, "downloads"), { recursive: true });
- writeFileSync(path.join(pub, "source", MIRROR_DIR, "info", "refs"), "x\trefs/heads/main\n");
- writeFileSync(path.join(pub, "downloads", path.basename(TARBALL_HREF)), "tarball");
- writeFileSync(path.join(pub, "downloads", "snapshot.json"), "{}");
- writeFileSync(path.join(pub, "source", "manifest.json"), fakeManifest(sourceCommit, mirrorHead));
const state = path.join(path.dirname(pub), ".source-publish.json");
- const key = { sourceCommit, mirrorHead, rulesHash: rules.rulesHash, filterRepo: "false (given)" };
- // Another filter-repo (an upgrade) does not skip…
- writeFileSync(state, JSON.stringify({ ...key, filterRepo: "git filter-repo 0.1" }));
- assert.equal(await publishSource({ ...o, check: true }), 1, "--check never skips, and reached `false`");
- assert.ok(existsSync(path.join(pub, "source", "manifest.json")), "--check withdraws nothing");
- // …the same one does.
- writeFileSync(state, JSON.stringify(key));
- logs.length = 0;
- assert.equal(await publishSource(o), 0);
+ const plant = async (tweak: Record<string, string> = {}) => {
+ mkdirSync(path.join(pub, "source", MIRROR_DIR, "info"), { recursive: true });
+ mkdirSync(path.join(pub, "downloads"), { recursive: true });
+ writeFileSync(path.join(pub, "source", MIRROR_DIR, "info", "refs"), "x\trefs/heads/main\n");
+ writeFileSync(path.join(pub, "downloads", path.basename(TARBALL_HREF)), "tarball");
+ writeFileSync(path.join(pub, "downloads", "snapshot.json"), "{}");
+ writeFileSync(path.join(pub, "source", "manifest.json"), fakeManifest(sourceCommit, mirrorHead));
+ const key = {
+ sourceCommit,
+ mirrorHead,
+ rulesHash: rules.rulesHash,
+ filterRepo: "false (given)",
+ gitleaks: "skipped",
+ contentDigest: await sourceDigest(pub),
+ };
+ writeFileSync(state, JSON.stringify({ ...key, ...tweak }));
+ };
+ const run = async () => {
+ logs.length = 0;
+ return publishSource(o);
+ };
+
+ await plant();
+ assert.equal(await run(), 0);
assert.match(logs.join("\n"), new RegExp(`up to date at ${sourceCommit.slice(0, 12)}; skipping`));
+ // Each part of the key, changed alone, reaches the rewrite (and the refusal
+ // withdraws the planted publish, so it is planted again each time).
+ for (const [what, tweak] of [
+ ["another filter-repo", { filterRepo: "git filter-repo 0.1" }],
+ ["another gitleaks", { gitleaks: "gitleaks 8.0 (abc)" }],
+ ["other files", { contentDigest: "0".repeat(64) }],
+ ["other rules", { rulesHash: "0".repeat(64) }],
+ ] as Array<[string, Record<string, string>]>) {
+ await plant(tweak);
+ assert.equal(await run(), 1, `${what} must not skip`);
+ assert.match(logs.join("\n"), /REFUSED: false --force --quiet exited 1/, what);
+ }
+ // A published file edited in place defeats the skip too.
+ await plant();
+ writeFileSync(path.join(pub, "downloads", "snapshot.json"), '{"edited":true}');
+ assert.equal(await run(), 1, "an edited public/ file must not skip");
+
+ // A changed rule refuses AND withdraws the last publish.
+ await plant();
writeFileSync(files.denylistFile, "a-new-literal\n");
- logs.length = 0;
- assert.equal(await publishSource(o), 1, "the new rule reached the rewrite");
+ assert.equal(await run(), 1, "the new rule reached the rewrite");
assert.match(logs.join("\n"), /REFUSED: false --force --quiet exited 1/);
assert.match(logs.join("\n"), /the previous publish was WITHDRAWN/);
assert.ok(!existsSync(path.join(pub, "source")), "the last publish is withdrawn");
@@ -257,6 +291,32 @@ test("skip: an unchanged main with unchanged rules and tools does nothing; a cha
assert.equal(readdirSync(o.scratchRoot!).length, 0, "the scratch dir is removed");
});
+test("the rules hash moves with the step's version (review R2-L3), and loadSourceRules uses the current one", async () => {
+ const lines = ["a==>b"];
+ const literals = [{ bytes: Buffer.from("a"), ci: false }];
+ assert.notEqual(rulesHashOf(lines, literals, 1), rulesHashOf(lines, literals, 2));
+ assert.notEqual(rulesHashOf(lines, literals, 1), rulesHashOf(lines, [{ bytes: Buffer.from("a"), ci: true }], 1));
+ const files = operatorFiles("a==>b\n", "");
+ const rules = await loadSourceRules({ ...files, homeDir: "/" });
+ assert.equal(rules.rulesHash, rulesHashOf(["a==>b"], rules.literals, SOURCE_STEP_VERSION));
+ assert.ok(SOURCE_STEP_VERSION >= 3);
+});
+
+test("gitleaksIdentity: skipped, absent, or the version line with the binary's sha256 (review R2-L4)", async () => {
+ const bin = dir("gl-bin");
+ assert.equal(await gitleaksIdentity(null, { PATH: bin }), "skipped");
+ assert.equal(await gitleaksIdentity("gitleaks", { PATH: bin }), "absent");
+ const fake = path.join(bin, "gitleaks");
+ writeFileSync(fake, "#!/bin/sh\necho 'version is set by build process'\n");
+ chmodSync(fake, 0o755);
+ const a = await gitleaksIdentity("gitleaks", { PATH: bin });
+ assert.match(a, /^version is set by build process \([0-9a-f]{12}\)$/);
+ // Same version line, another binary (an upgrade on a distribution that
+ // prints a constant line): another identity.
+ writeFileSync(fake, "#!/bin/sh\necho 'version is set by build process'\n# 8.30\n");
+ assert.notEqual(await gitleaksIdentity("gitleaks", { PATH: bin }), a);
+});
+
test("the scratch root: refused inside the checkout or the public dir, through a symlink too (review L12)", async () => {
const checkout = dir("chk");
const pub = path.join(dir("site"), "public");
@@ -403,7 +463,7 @@ test("round trip: --check writes nothing; publish; a dumb clone of the mirror is
// The deploy check (M1): a build's out/ that is this publish deploys…
const out = path.join(dir("out"), "out");
- const check = { ...files, homeDir: HOME, publicDir: pub, sourceRepo: path.join(repo, ".git") };
+ const check = { ...files, homeDir: HOME, publicDir: pub, sourceRepo: path.join(repo, ".git"), gitleaks: null };
const checkPaths = o.paths!;
// A build's out/ always has the /source page; a --no-source one, nothing more.
mkdirSync(path.join(out, "source"), { recursive: true });
@@ -412,6 +472,34 @@ test("round trip: --check writes nothing; publish; a dumb clone of the mirror is
cpSync(path.join(pub, "source"), path.join(out, "source"), { recursive: true });
cpSync(path.join(pub, "downloads"), path.join(out, "downloads"), { recursive: true });
assert.equal(await publishedSourceProblem(checkPaths, out, check), null);
+ // Review R2-L3: a state naming another mirror head, or another gitleaks,
+ // refuses — each by its own sentence (the digest below would refuse
+ // neither, since out/ is untouched).
+ const stateFile = path.join(site, ".source-publish.json");
+ const good = readFileSync(stateFile, "utf8");
+ const stateWith = (tweak: object) => writeFileSync(stateFile, JSON.stringify({ ...JSON.parse(good), ...tweak }));
+ stateWith({ mirrorHead: "f".repeat(40) });
+ assert.match((await publishedSourceProblem(checkPaths, out, check))!, /is not the last publish \(ffffffffffff\)/);
+ stateWith({ gitleaks: "gitleaks 8.0 (abc)" });
+ assert.match((await publishedSourceProblem(checkPaths, out, check))!, /scanned by another gitleaks/);
+ writeFileSync(stateFile, good);
+ // Review R2-L2: a mixed out/ — one tree file swapped, or one pack byte
+ // flipped — does not match the digest of what was audited.
+ const readme = path.join(out, "source", "tree", "README.md");
+ const readmeWas = readFileSync(readme);
+ writeFileSync(readme, "another publish's README\n");
+ assert.match((await publishedSourceProblem(checkPaths, out, check))!, /not the ones that were audited/);
+ writeFileSync(readme, readmeWas);
+ const packDirOut = path.join(out, "source", MIRROR_DIR, "objects", "pack");
+ const pack = path.join(packDirOut, readdirSync(packDirOut).find((f) => f.endsWith(".pack"))!);
+ const packWas = readFileSync(pack);
+ chmodSync(pack, 0o644); // git writes packs read-only; the digest reads bytes, not modes
+ const flipped = Buffer.from(packWas);
+ flipped[Math.floor(flipped.length / 2)] ^= 0x01;
+ writeFileSync(pack, flipped);
+ assert.match((await publishedSourceProblem(checkPaths, out, check))!, /not the ones that were audited/);
+ writeFileSync(pack, packWas);
+ assert.equal(await publishedSourceProblem(checkPaths, out, check), null, "restored, it deploys again");
// …a tampered tarball does not…
writeFileSync(path.join(out, "downloads", path.basename(TARBALL_HREF)), "other bytes");
assert.match((await publishedSourceProblem(checkPaths, out, check))!, /tarball is not the one its manifest describes/);
@@ -466,6 +554,36 @@ test("a denied literal no rule removes: refused, nothing written, the report nev
rmSync(kept[1], { recursive: true, force: true });
});
+test("a refusal naming a tree path masks a literal that spans path components (review R2-L1)", async (t) => {
+ if (filterRepoProblem) return t.skip(`git-filter-repo unavailable: ${filterRepoProblem}`);
+ // `plant/secret` is invisible to the object walk (which reads entry names
+ // one at a time), so the first refusal to name it is the tree's own: a
+ // tracked symlink below it, then a tracked index.html.
+ const spanning = "plant/secret";
+ const repo = sourceRepo();
+ mkdirSync(path.join(repo, "plant", "secret"), { recursive: true });
+ symlinkSync("../../README.md", path.join(repo, "plant", "secret", "link"));
+ gitIn(repo, "add", "-A");
+ gitIn(repo, "commit", "-q", "-m", "a link");
+ const files = operatorFiles(`/srv/${PLANTED}==>/home/user\n`, `${spanning}\n`);
+ const logs: string[] = [];
+ assert.equal(await publishSource(opts(repo, files, logs)), 1);
+ let log = logs.join("\n");
+ assert.match(log, /REFUSED: the tree holds a symlink \(\[REDACTED\]\/link\)/);
+ assert.ok(!log.includes(spanning), log);
+
+ gitIn(repo, "rm", "-q", "plant/secret/link");
+ mkdirSync(path.join(repo, "plant", "secret"), { recursive: true });
+ writeFileSync(path.join(repo, "plant", "secret", "index.html"), "<p>x</p>");
+ gitIn(repo, "add", "-A");
+ gitIn(repo, "commit", "-q", "-m", "an index");
+ logs.length = 0;
+ assert.equal(await publishSource(opts(repo, files, logs)), 1);
+ log = logs.join("\n");
+ assert.match(log, /REFUSED: the tree already has \[REDACTED\]\/index\.html/);
+ assert.ok(!log.includes(spanning), log);
+});
+
test("--no-source's clear: the manifest, mirror, tree, tarball and skip key go; a linked downloads/ goes as a link", async () => {
const site = dir("clear");
const pub = path.join(site, "public");
diff --git a/common/publish/source.ts b/common/publish/source.ts
@@ -29,6 +29,7 @@
// build homepage`, the runbooks' home scripts and the editor's /sites homepage
// jobs all publish it; an unchanged main with unchanged rules skips.
+import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { createReadStream, existsSync } from "node:fs";
import { cp, lstat, mkdir, mkdtemp, readdir, readFile, realpath, rm, stat, writeFile } from "node:fs/promises";
@@ -79,8 +80,10 @@ export const SOURCE_BRANCH = "main";
* 1 — release 12 slice R.
* 2 — release 12 slice R review: BOM/CRLF/trailing-slash parsing, empty
* left sides refused, `404.html` refused, no context bytes.
+ * 3 — the re-review: masked refusal messages, a digest of every published
+ * file and the gitleaks identity in the key.
*/
-export const SOURCE_STEP_VERSION = 2;
+export const SOURCE_STEP_VERSION = 3;
/** What `pipx run` fetches when `git filter-repo` is not installed. */
export const FILTER_REPO_PIPX_SPEC = "git-filter-repo==2.47.0";
@@ -230,16 +233,23 @@ export async function loadSourceRules(opts: {
...parseDenylist(denyText),
...scrub.denied.map((d) => ({ bytes: Buffer.from(d.text, "utf8"), ci: false, from: d.from })),
]);
- const rulesHash = createHash("sha256")
+ return { scrub, literals, rulesHash: rulesHashOf(scrub.lines, literals, SOURCE_STEP_VERSION) };
+}
+
+/**
+ * The rules hash: the scrub lines, every literal (and whether it is `i:`),
+ * and the step's version — so a fix to the step moves it like a new rule does.
+ */
+export function rulesHashOf(lines: readonly string[], literals: readonly Literal[], step: number): string {
+ return createHash("sha256")
.update(
JSON.stringify({
- step: SOURCE_STEP_VERSION,
- rules: scrub.lines,
+ step,
+ rules: lines,
literals: literals.map((l) => `${l.ci ? "i" : "x"}:${l.bytes.toString("hex")}`),
}),
)
.digest("hex");
- return { scrub, literals, rulesHash };
}
// ── children ────────────────────────────────────────────────────────────────
@@ -417,8 +427,72 @@ type PublishState = {
rulesHash: string;
// "<label> <version>": an upgrade may rewrite every id, so it re-publishes.
filterRepo: string;
+ // gitleaksIdentity(): a newer gitleaks may find what the old one did not,
+ // so it re-publishes, and an out/ it never scanned does not deploy.
+ gitleaks: string;
+ // sourceDigest() over every published file: an out/ (or public/) holding
+ // another publish's mirror or tree beside this manifest does not match.
+ contentDigest: string;
};
+/**
+ * One digest over every file a publish puts on the site, under `root` laid
+ * out as public/ and out/ both are: `source/archilyzer.git/**`,
+ * `source/tree/**`, `source/manifest.json`, `downloads/archilyzer-source.tar.gz`
+ * and `downloads/snapshot.json`. Each file contributes its relative path, size
+ * and sha256 (streamed), in sorted path order; a symlink or a missing piece
+ * contributes a marker, so it can only differ. The /source PAGE and anything
+ * else Next writes beside them are not part of it.
+ */
+export async function sourceDigest(root: string): Promise<string> {
+ const entries: string[] = [];
+ const walk = async (rel: string): Promise<void> => {
+ const abs = path.join(root, rel);
+ const st = await lstat(abs).catch(() => null);
+ if (!st) {
+ entries.push(`${rel}\0missing`);
+ } else if (st.isSymbolicLink()) {
+ entries.push(`${rel}\0symlink`);
+ } else if (st.isDirectory()) {
+ for (const name of await readdir(abs)) await walk(`${rel}/${name}`);
+ } else {
+ entries.push(`${rel}\0${st.size}\0${await sha256File(abs)}`);
+ }
+ };
+ for (const top of [
+ `source/${MIRROR_DIR}`,
+ "source/tree",
+ "source/manifest.json",
+ `downloads/${TARBALL_NAME}`,
+ "downloads/snapshot.json",
+ ]) {
+ await walk(top);
+ }
+ entries.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
+ const h = createHash("sha256");
+ for (const e of entries) h.update(`${e}\n`);
+ return h.digest("hex");
+}
+
+/**
+ * Which gitleaks the audit runs: "skipped" (none asked for), "absent" (not on
+ * PATH), else its version line and the sha256 of its binary — the version
+ * line alone is not enough (a distribution build can print the same line for
+ * every release).
+ */
+export async function gitleaksIdentity(bin: string | null, env: NodeJS.ProcessEnv): Promise<string> {
+ if (bin === null) return "skipped";
+ const found = onPath(bin, env.PATH);
+ if (!found) return "absent";
+ const version = await new Promise<string>((resolve) => {
+ execFile(found, ["version"], { env, timeout: 10_000 }, (_err, stdout, stderr) =>
+ resolve(String(stdout || stderr).trim().split("\n")[0] ?? ""),
+ );
+ });
+ const sha = await sha256File(await realpath(found));
+ return `${version || "unknown version"} (${sha.slice(0, 12)})`;
+}
+
async function readJson(file: string): Promise<unknown> {
try {
return JSON.parse(await readFile(file, "utf8"));
@@ -557,7 +631,9 @@ export async function publishSource(opts: SourcePublishOpts = {}): Promise<numbe
onLog("[source] cancelled — nothing published");
code = 1;
} else if (err instanceof SourceRefusal) {
- onLog(`[source] REFUSED: ${err.message}`);
+ // A refusal may name a tree path or quote a child's stderr: masked, so
+ // a literal spanning path components (`a/b`) is never printed.
+ onLog(`[source] REFUSED: ${maskLiterals(err.message, ctx.literals)}`);
code = 1;
} else {
await withdraw().catch(() => {});
@@ -606,8 +682,11 @@ async function publish(
? { argv: opts.filterRepo, label: opts.filterRepo.join(" "), version: "(given)" }
: await resolveFilterRepo({ env: ctx.env, signal: ctx.signal });
const filterRepoId = `${filterRepo.label} ${filterRepo.version}`;
+ const gitleaksBin = opts.gitleaks === undefined ? "gitleaks" : opts.gitleaks;
+ const gitleaksId = await gitleaksIdentity(gitleaksBin, ctx.env);
- // 4. Nothing changed: skip.
+ // 4. Nothing changed: skip. The key is main, the rules (with the step's
+ // version), both tools, and the published files themselves.
if (!opts.force && !opts.check) {
const manifest = await readPublishedManifest(publicDir);
const state = (await readJson(statePath(publicDir))) as Partial<PublishState> | null;
@@ -616,9 +695,11 @@ async function publish(
state?.sourceCommit === sourceCommit &&
state.rulesHash === rules.rulesHash &&
state.filterRepo === filterRepoId &&
+ state.gitleaks === gitleaksId &&
state.mirrorHead === manifest.mirrorHead &&
existsSync(path.join(pubSource, MIRROR_DIR, "info", "refs")) &&
- existsSync(path.join(pubDownloads, TARBALL_NAME))
+ existsSync(path.join(pubDownloads, TARBALL_NAME)) &&
+ state.contentDigest === (await sourceDigest(publicDir))
) {
onLog(`[source] up to date at ${sourceCommit.slice(0, 12)}; skipping (--force to rebuild)`);
return 0;
@@ -709,7 +790,7 @@ async function publish(
scratch,
onLog,
signal: ctx.signal,
- gitleaks: opts.gitleaks === undefined ? "gitleaks" : opts.gitleaks,
+ gitleaks: gitleaksBin,
env: ctx.env,
});
const scrubFile = opts.scrubFile ?? paths.sourceScrubFile;
@@ -803,6 +884,9 @@ async function publish(
};
const manifestText = JSON.stringify(manifest, null, 2) + "\n";
await writeFile(path.join(stageSource, "manifest.json"), manifestText);
+ // Every published file, as it will land: the state carries it, and the
+ // skip and the deploy check recompute it over public/ and out/.
+ const contentDigest = await sourceDigest(stage);
// 13. THE GATE, over every staged file and path (packs excepted: step 9
// read their objects).
@@ -840,7 +924,14 @@ async function publish(
await ownDir(pubDownloads);
await copyPublicFile(tarball, path.join(pubDownloads, TARBALL_NAME));
await writePublicFile(path.join(pubDownloads, "snapshot.json"), snapshotText);
- const state: PublishState = { sourceCommit, mirrorHead, rulesHash: rules.rulesHash, filterRepo: filterRepoId };
+ const state: PublishState = {
+ sourceCommit,
+ mirrorHead,
+ rulesHash: rules.rulesHash,
+ filterRepo: filterRepoId,
+ gitleaks: gitleaksId,
+ contentDigest,
+ };
await writeFile(statePath(publicDir), JSON.stringify(state, null, 2) + "\n");
await writePublicFile(path.join(pubSource, "manifest.json"), manifestText);
@@ -851,7 +942,7 @@ async function publish(
if (opts.keepScratch) {
// The scrub rules in plain text never outlive the run.
await rm(replace, { force: true });
- onLog(`[source] scratch kept at ${scratch} (replace.txt, the scrub rules, deleted)`);
+ onLog(`[source] scratch kept at ${maskLiterals(tildify(scratch), ctx.literals)} (replace.txt, the scrub rules, deleted)`);
} else {
await rm(scratch, { recursive: true, force: true });
}
@@ -901,6 +992,8 @@ export async function publishedSourceProblem(
homeDir?: string;
sourceRepo?: string;
env?: NodeJS.ProcessEnv;
+ // The gitleaks the audit would run (default "gitleaks"; null: none).
+ gitleaks?: string | null;
} = {},
): Promise<string | null> {
const outSource = path.join(outDir, "source");
@@ -923,7 +1016,7 @@ export async function publishedSourceProblem(
const manifest = parseSourceManifest(await readJson(path.join(outSource, "manifest.json")));
if (!manifest) return `homepage/out holds a source publish without a valid manifest — ${rebuild}`;
const state = (await readJson(statePath(sourcePublicDir(paths, opts.publicDir)))) as Partial<PublishState> | null;
- if (!state?.rulesHash || !state.mirrorHead || !state.sourceCommit) {
+ if (!state?.rulesHash || !state.mirrorHead || !state.sourceCommit || !state.contentDigest || !state.gitleaks) {
return `homepage/out's source has no record of the rules it was audited under — ${rebuild}`;
}
let rules: SourceRules;
@@ -963,6 +1056,16 @@ export async function publishedSourceProblem(
if (existsSync(outTarball) && (await sha256File(outTarball)) !== manifest.tarball.sha256) {
return `homepage/out's source tarball is not the one its manifest describes — ${rebuild}`;
}
+ const env = cleanGitEnv(opts.env ?? process.env);
+ const gitleaks = await gitleaksIdentity(opts.gitleaks === undefined ? "gitleaks" : opts.gitleaks, env);
+ if (state.gitleaks !== gitleaks) {
+ return `homepage/out's source was scanned by another gitleaks (${state.gitleaks}; this machine has ${gitleaks}) — ${rebuild}`;
+ }
+ // Last, and the one that binds every byte: the mirror, the tree, the
+ // manifest and both downloads, against the digest of what was audited.
+ if ((await sourceDigest(outDir)) !== state.contentDigest) {
+ return `homepage/out's source files are not the ones that were audited (a mixed or edited out/) — ${rebuild}`;
+ }
return null;
}
@@ -994,18 +1097,20 @@ export async function auditSource(
);
const scrubFile = opts.scrubFile ?? paths.sourceScrubFile;
let scratch: string | null = null;
+ let literals: readonly Literal[] = [];
try {
const rules = await loadSourceRules({
scrubFile,
denylistFile: opts.denylistFile ?? paths.sourceDenylistFile,
homeDir: opts.homeDir,
});
+ literals = rules.literals;
const ctx: Ctx = { onLog, signal, env, literals: rules.literals };
const head = await revParse(ctx, gitDir, "HEAD");
const scratchRoot = opts.scratchRoot ?? paths.sourceScratchDir;
await mkdir(scratchRoot, { recursive: true });
scratch = await mkdtemp(path.join(scratchRoot, "archilyzer-source-audit-"));
- onLog(`[source] auditing ${tildify(gitDir)} (HEAD ${head.slice(0, 12)}) for ${rules.literals.length} denied literals…`);
+ onLog(`[source] auditing ${maskLiterals(tildify(gitDir), literals)} (HEAD ${head.slice(0, 12)}) for ${rules.literals.length} denied literals…`);
const audit = await auditBare(gitDir, rules.literals, {
scratch,
onLog,
@@ -1021,7 +1126,7 @@ export async function auditSource(
return 1;
}
if (err instanceof SourceRefusal) {
- onLog(`[source] REFUSED: ${err.message}`);
+ onLog(`[source] REFUSED: ${maskLiterals(err.message, literals)}`);
return 1;
}
throw err;
diff --git a/common/publish/sourceAudit.ts b/common/publish/sourceAudit.ts
@@ -644,7 +644,7 @@ export function formatAuditReport(
lines.push(`[source] … and ${result.hits.length - shown.length} more`);
}
lines.push(
- `[source] add a rule to ${tildify(opts.scrubFile)} or drop the file from history, then re-run.`,
+ `[source] add a rule to ${maskLiterals(tildify(opts.scrubFile), literals)} or drop the file from history, then re-run.`,
);
return lines;
}
