commit 3b2ae052365d78e072708c05938bb706e09fa5f7
parent aab32fe5f987522b39bce42934849eea556bc30d
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 28 Sep 2026 14:22:47 -0400
common: a refusal withdraws the last publish, a build's out/ copy and any deploy of it (review M1), and the Lows
M1: once the rules are loaded, any outcome of `source publish` but success —
an audit hit, a limit, a missing tool, a cancel, a crash — removes the last
publish from homepage/public (manifest first, then the mirror, the tree, the
tarball, snapshot.json and the skip key); `--check` still writes nothing.
buildHomepage also removes out/source and the two download files from
homepage/out when the step refuses. deployHomepage asks publishedSourceProblem
first: an out/ holding a source ships only when the skip key says it was
audited under today's rules and step, of today's main, and is that publish
(its mirror head, its tarball's sha256); a --no-source out/ deploys as before.
L2: BOM, CRLF and a trailing `/` on the home dir are dropped before the rules
are built; an empty left side is a refusal. L3: the manifest no longer carries
the literal count. L8: a checkout with no git repository builds with the
/source empty state (`noRepository: "empty"`, which buildHomepage passes); the
CLI exits 1 with the same sentence. L10: parseSourceManifest checks every
number the page reads. L11: SOURCE_STEP_VERSION (2) is in the rules hash and
the filter-repo version in the skip key. L12: a scratch root inside the
checkout or the public dir is refused; `--keep-scratch` deletes replace.txt.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
7 files changed, 644 insertions(+), 127 deletions(-)
diff --git a/common/bin/doctor.test.ts b/common/bin/doctor.test.ts
@@ -262,8 +262,9 @@ test("the source publish block: the tools, the operator files by count and mode
writeFileSync(path.join(pub, "manifest.json"), JSON.stringify({
version: 1, generatedAt: "2026-09-28T12:00:00.000Z", branch: "main",
sourceCommit: "1".repeat(40), mirrorHead: "2".repeat(40), subject: "s", files: 2400, bytes: 1,
- mirror: {}, tree: {}, tarball: { href: "/downloads/archilyzer-source.tar.gz", bytes: 1, sha256: "3".repeat(64) },
- audit: {}, tools: {},
+ mirror: { files: 9, bytes: 1, packs: 2 }, tree: { files: 1, dirs: 1, bytes: 1 },
+ tarball: { href: "/downloads/archilyzer-source.tar.gz", bytes: 1, sha256: "3".repeat(64) },
+ audit: { objects: 1, commits: 1, gitleaks: "clean" }, tools: {},
}));
const before = tree(c.root);
r = await collectDoctorReport(deps({ filterRepo: { via: "pipx", version: "1.15.0" }, gitleaks: { version: "8.28.0" } }));
diff --git a/common/lib/sourceManifest.test.ts b/common/lib/sourceManifest.test.ts
@@ -0,0 +1,50 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { parseSourceManifest, TARBALL_HREF } from "./sourceManifest";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common test
+//
+// The homepage believes a manifest only through parseSourceManifest, and the
+// /source/ page reads its numbers during `next build`: a malformed or
+// foreign one must be the empty state (null), never a crash (review L10).
+
+const good = () => ({
+ version: 1,
+ generatedAt: "2026-09-28T12:00:00.000Z",
+ branch: "main",
+ sourceCommit: "1".repeat(40),
+ mirrorHead: "2".repeat(40),
+ subject: "s",
+ files: 10,
+ bytes: 100,
+ mirror: { files: 9, bytes: 80, packs: 2 },
+ tree: { files: 5, dirs: 2, bytes: 20 },
+ tarball: { href: TARBALL_HREF, bytes: 7, sha256: "3".repeat(64) },
+ cloneUrl: "x",
+ treeHref: "/source/tree/",
+ audit: { objects: 3, commits: 1, gitleaks: "clean" },
+ tools: { git: "2", filterRepo: "x" },
+});
+
+test("a well-formed version-1 manifest parses", () => {
+ assert.deepEqual(parseSourceManifest(good()), good());
+});
+
+test("a wrong version, a bad id or sha, or any number the page reads that is not one is null", () => {
+ const broken: Array<[string, (m: ReturnType<typeof good>) => unknown]> = [
+ ["version 2", (m) => ({ ...m, version: 2 })],
+ ["short mirror head", (m) => ({ ...m, mirrorHead: "abc" })],
+ ["upper-case sha", (m) => ({ ...m, tarball: { ...m.tarball, sha256: "A".repeat(64) } })],
+ ["no tree", (m) => ({ ...m, tree: undefined })],
+ ["tree.files a string", (m) => ({ ...m, tree: { ...m.tree, files: "5" } })],
+ ["mirror.packs missing", (m) => ({ ...m, mirror: { files: 1, bytes: 1 } })],
+ ["mirror.bytes NaN", (m) => ({ ...m, mirror: { ...m.mirror, bytes: Number.NaN } })],
+ ["negative files", (m) => ({ ...m, files: -1 })],
+ ["tarball.bytes missing", (m) => ({ ...m, tarball: { href: TARBALL_HREF, sha256: "3".repeat(64) } })],
+ ["unparsable date", (m) => ({ ...m, generatedAt: "yesterday" })],
+ ["no audit", (m) => ({ ...m, audit: null })],
+ ];
+ for (const [what, make] of broken) assert.equal(parseSourceManifest(make(good())), null, what);
+ for (const junk of [null, 1, "x", [], {}]) assert.equal(parseSourceManifest(junk), null, JSON.stringify(junk));
+});
diff --git a/common/lib/sourceManifest.ts b/common/lib/sourceManifest.ts
@@ -58,10 +58,11 @@ export type SourceManifest = {
tarball: { href: string; bytes: number; sha256: string };
cloneUrl: string;
treeHref: string;
- // What the gate checked: how many denied literals, how many git objects
- // (commits among them) it read, and whether gitleaks ran.
+ // What the gate read: how many git objects (commits among them), and
+ // whether gitleaks ran. NOT how many literals it searched for: `plans/`
+ // publishes which ones the plan put in the files, so the count would say
+ // whether the operator added private ones.
audit: {
- literals: number;
objects: number;
commits: number;
gitleaks: "clean" | "skipped";
@@ -75,19 +76,30 @@ const HEX64 = /^[0-9a-f]{64}$/;
/**
* The manifest, or null when `value` is not one this code can trust: version
- * 1, 40-hex commit ids, a 64-hex tarball sha. The homepage additionally
- * requires the files it describes to be present (homepage/app/lib/source.ts).
+ * 1, 40-hex commit ids, a 64-hex tarball sha, and a finite number wherever a
+ * page reads one. A malformed or foreign manifest is the page's empty state,
+ * never a crash in `next build`. The homepage additionally requires the files
+ * it describes to be present (homepage/app/lib/source.ts).
*/
export function parseSourceManifest(value: unknown): SourceManifest | null {
if (!value || typeof value !== "object") return null;
const m = value as Partial<SourceManifest>;
+ const num = (x: unknown) => typeof x === "number" && Number.isFinite(x) && x >= 0;
+ const obj = (x: unknown): x is Record<string, unknown> => !!x && typeof x === "object";
if (m.version !== SOURCE_MANIFEST_VERSION) return null;
if (typeof m.mirrorHead !== "string" || !HEX40.test(m.mirrorHead)) return null;
if (typeof m.sourceCommit !== "string" || !HEX40.test(m.sourceCommit)) return null;
- if (!m.tarball || typeof m.tarball.sha256 !== "string" || !HEX64.test(m.tarball.sha256)) {
+ if (typeof m.subject !== "string" || typeof m.generatedAt !== "string") return null;
+ if (Number.isNaN(Date.parse(m.generatedAt))) return null;
+ if (!num(m.files) || !num(m.bytes)) return null;
+ if (!obj(m.tarball) || typeof m.tarball.sha256 !== "string" || !HEX64.test(m.tarball.sha256)) {
return null;
}
- if (typeof m.tarball.bytes !== "number" || typeof m.generatedAt !== "string") return null;
- if (!m.mirror || !m.tree || !m.audit || typeof m.subject !== "string") return null;
+ if (!num(m.tarball.bytes) || typeof m.tarball.href !== "string") return null;
+ if (!obj(m.mirror) || !num(m.mirror.files) || !num(m.mirror.bytes) || !num(m.mirror.packs)) {
+ return null;
+ }
+ if (!obj(m.tree) || !num(m.tree.files) || !num(m.tree.dirs) || !num(m.tree.bytes)) return null;
+ if (!obj(m.audit)) return null;
return m as SourceManifest;
}
diff --git a/common/publish/build.test.ts b/common/publish/build.test.ts
@@ -1,6 +1,6 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { chmodSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import type { Paths } from "../lib/paths";
@@ -162,6 +162,16 @@ test("buildHomepage: the source step sits between compose and next build; a refu
writeFileSync(next, "#!/bin/sh\necho NEXT RAN\n");
chmodSync(next, 0o755);
const fakePaths = { monorepoRoot: root } as Paths;
+ // The last good build's source, in out/ (M1: a refusal must take it out).
+ const out = path.join(home, "out");
+ const plantOut = () => {
+ mkdirSync(path.join(out, "source", "archilyzer.git"), { recursive: true });
+ mkdirSync(path.join(out, "downloads"), { recursive: true });
+ writeFileSync(path.join(out, "source", "manifest.json"), "{}");
+ writeFileSync(path.join(out, "downloads", "archilyzer-source.tar.gz"), "old");
+ writeFileSync(path.join(out, "downloads", "snapshot.json"), "{}");
+ writeFileSync(path.join(out, "downloads", "index.html"), "<p>the page</p>");
+ };
const run = async (o: Parameters<typeof buildHomepage>[0]) => {
const logs: string[] = [];
@@ -170,17 +180,24 @@ test("buildHomepage: the source step sits between compose and next build; a refu
};
const calls: string[] = [];
- // A refusal (1) fails the build before next build runs.
+ // A refusal (1) fails the build before next build runs, and takes the
+ // last build's source out of homepage/out, so a deploy-only ships none.
+ plantOut();
let r = await run({
publishSource: async (p) => {
- calls.push(`publish:${p.paths === fakePaths}`);
+ calls.push(`publish:${p.paths === fakePaths}:${p.noRepository}`);
return 1;
},
});
assert.equal(r.code, 1);
assert.match(r.logs, /COMPOSED/);
assert.doesNotMatch(r.logs, /NEXT RAN/);
- assert.deepEqual(calls, ["publish:true"]);
+ assert.deepEqual(calls, ["publish:true:empty"], "a checkout with no git builds with the empty state");
+ assert.ok(!existsSync(path.join(out, "source")), "out/source is withdrawn");
+ assert.ok(!existsSync(path.join(out, "downloads", "archilyzer-source.tar.gz")));
+ assert.ok(!existsSync(path.join(out, "downloads", "snapshot.json")));
+ assert.ok(existsSync(path.join(out, "downloads", "index.html")), "the page itself stays");
+ assert.match(r.logs, /removed from homepage\/out too/);
// Published: next build follows.
r = await run({ publishSource: async () => (calls.push("publish"), 0) });
@@ -206,6 +223,37 @@ test("buildHomepage: the source step sits between compose and next build; a refu
}
});
+// M1: a deploy-only ships homepage/out as the last build left it. Its source
+// ships only with a record that it was audited under today's rules (the
+// positive case is source.test.ts' round trip, over publishedSourceProblem).
+test("deployHomepage refuses an out/ whose source has no record of the rules it was audited under", async () => {
+ const root = mkdtempSync(path.join(os.tmpdir(), "deploy-homepage-"));
+ try {
+ const out = path.join(root, "homepage", "out");
+ mkdirSync(path.join(out, "source"), { recursive: true });
+ writeFileSync(path.join(out, "index.html"), "<p>home</p>");
+ writeFileSync(
+ path.join(out, "source", "manifest.json"),
+ JSON.stringify({
+ version: 1, generatedAt: "2026-09-28T12:00:00.000Z", branch: "main", sourceCommit: "1".repeat(40),
+ mirrorHead: "2".repeat(40), subject: "s", files: 1, bytes: 1, mirror: { files: 1, bytes: 1, packs: 1 },
+ tree: { files: 1, dirs: 1, bytes: 1 }, tarball: { href: "/downloads/archilyzer-source.tar.gz", bytes: 1, sha256: "3".repeat(64) },
+ audit: { objects: 1, commits: 1, gitleaks: "clean" }, tools: {},
+ }),
+ );
+ const p = { monorepoRoot: root } as Paths;
+ await assert.rejects(
+ deployHomepage({ paths: p, previewBranch: "r12-source" }),
+ /homepage\/out's source has no record of the rules it was audited under — run `archilyzer build homepage`/,
+ );
+ // A half-removed source (no manifest) refuses too.
+ rmSync(path.join(out, "source", "manifest.json"));
+ await assert.rejects(deployHomepage({ paths: p }), /without a valid manifest — run `archilyzer build homepage`/);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
test("deployHomepage refuses a bad preview branch before it looks for a build", async () => {
const noBuild = { monorepoRoot: "/nonexistent-repo" } as Paths;
await assert.rejects(
diff --git a/common/publish/build.ts b/common/publish/build.ts
@@ -972,18 +972,26 @@ export async function composeHomepage(opts: PublishOpts = {}): Promise<number> {
/**
* composeHomepage, then `archilyzer source publish` (the git mirror, the raw
* tree and the tarball into homepage/public — source.ts), then `next build` in
- * homepage/ (→ homepage/out). A source REFUSAL fails the build before `next
- * build`: nothing is deployed with a stale or missing source, and the redacted
- * audit report is in the log. `skipSource` (the CLI's `--no-source`) REMOVES
- * the published source instead (the mirror, the tree, the tarball): a copy
- * left from an earlier publish was audited against that day's rules, not
- * today's, so a build that skips the gate ships none — the pages show their
- * empty states. `publishSource` / `clearSource` are the test's seams.
+ * homepage/ (→ homepage/out).
+ *
+ * A source REFUSAL fails the build before `next build`, and withdraws the
+ * source twice over: the step itself removes the last publish from
+ * homepage/public, and this removes the last BUILD's copy from homepage/out
+ * (`out/source`, the tarball, `snapshot.json`), so a deploy-only cannot ship
+ * a source today's rules were never applied to (deployHomepage checks too).
+ * The audit report is in the log.
+ *
+ * `skipSource` (the CLI's `--no-source`) REMOVES the published source instead:
+ * a copy left from an earlier publish was audited against the rules of ITS
+ * day, so a build that skips the gate ships none — the pages show their empty
+ * states. A checkout with no git repository (the docker runtime, a tarball
+ * install) builds with the empty state too. `publishSource` / `clearSource`
+ * are the test's seams.
*/
export async function buildHomepage(
opts: PublishOpts & {
skipSource?: boolean;
- publishSource?: (o: PublishOpts) => Promise<number>;
+ publishSource?: (o: PublishOpts & { noRepository?: "refuse" | "empty" }) => Promise<number>;
clearSource?: (o: PublishOpts) => Promise<void>;
} = {},
): Promise<number> {
@@ -997,8 +1005,11 @@ export async function buildHomepage(
await clear({ paths, onLog, signal });
} else {
const publish = opts.publishSource ?? (await import("./source")).publishSource;
- const sourceCode = await publish({ paths, onLog, signal });
- if (sourceCode !== 0 || signal.aborted) return sourceCode || 1;
+ const sourceCode = await publish({ paths, onLog, signal, noRepository: "empty" });
+ if (sourceCode !== 0 || signal.aborted) {
+ await withdrawBuiltSource(paths, onLog);
+ return sourceCode || 1;
+ }
}
return runSteps(onLog, signal, [
{
@@ -1010,6 +1021,22 @@ export async function buildHomepage(
]);
}
+// The last build's copy of the source, out of homepage/out: a refused source
+// step must not leave it for a deploy-only to ship.
+async function withdrawBuiltSource(paths: Paths, onLog: (line: string) => void): Promise<void> {
+ const out = homepageOutDir(paths);
+ const had =
+ existsSync(path.join(out, "source", "manifest.json")) ||
+ existsSync(path.join(out, "downloads", "archilyzer-source.tar.gz"));
+ await rm(path.join(out, "source", "manifest.json"), { force: true });
+ await rm(path.join(out, "source"), { recursive: true, force: true });
+ await rm(path.join(out, "downloads", "archilyzer-source.tar.gz"), { force: true });
+ await rm(path.join(out, "downloads", "snapshot.json"), { force: true });
+ if (had) {
+ onLog("[source] the last build's source was removed from homepage/out too, so a deploy-only ships none.\n");
+ }
+}
+
/**
* The wrangler argv (after `pnpm dlx`) for a homepage deploy of `outDir`: the
* production branch `main` — what homepage/package.json's hardcoded `deploy`
@@ -1047,6 +1074,10 @@ export async function deployHomepage(
if (!existsSync(path.join(outDir, "index.html"))) {
throw new Error("homepage/out holds no build — run archilyzer build homepage first");
}
+ // The source in out/ ships only if it was audited under TODAY's rules, of
+ // today's main (source.ts). An out/ with no source deploys as before.
+ const sourceProblem = await (await import("./source")).publishedSourceProblem(paths, outDir);
+ if (sourceProblem) throw new Error(sourceProblem);
if (branch) onLog(`=== Deploy homepage (preview "${branch}") ===\n`);
const code = await runPagesDeployIntoLog(onLog, signal, {
outDir,
diff --git a/common/publish/source.test.ts b/common/publish/source.test.ts
@@ -3,6 +3,7 @@ import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { execFileSync } from "node:child_process";
import {
+ cpSync,
existsSync,
lstatSync,
mkdirSync,
@@ -21,13 +22,16 @@ import { CLONE_URL, MIRROR_DIR, TARBALL_HREF, TREE_HREF } from "../lib/sourceMan
import {
MAX_FILES,
MAX_FILE_BYTES,
+ NO_REPOSITORY,
SourceRefusal,
clearPublishedSource,
limitProblem,
loadSourceRules,
parseScrubRules,
publishSource,
+ publishedSourceProblem,
resolveFilterRepo,
+ scratchRootProblem,
type SourcePublishOpts,
} from "./source";
@@ -107,9 +111,11 @@ function operatorFiles(scrub: string, deny: string): { scrubFile: string; denyli
return { scrubFile: path.join(d, "source-scrub.txt"), denylistFile: path.join(d, "source-denylist.txt") };
}
+// A checkout of its own (never TMP itself, which holds the scratch root: the
+// step refuses a scratch dir inside the checkout).
function opts(repo: string, files: { scrubFile: string; denylistFile: string }, logs: string[], extra: Partial<SourcePublishOpts> = {}): SourcePublishOpts {
return {
- paths: { monorepoRoot: TMP } as Paths,
+ paths: { monorepoRoot: dir("checkout") } as Paths,
sourceRepo: path.join(repo, ".git"),
publicDir: path.join(dir("site"), "public"),
scratchRoot: path.join(TMP, "scratch"),
@@ -124,6 +130,16 @@ function opts(repo: string, files: { scrubFile: string; denylistFile: string },
const sha256 = (file: string) => createHash("sha256").update(readFileSync(file)).digest("hex");
+// A manifest the page would believe, for the tests that fake a publish.
+function fakeManifest(sourceCommit: string, mirrorHead: string, sha = "b".repeat(64)): string {
+ return JSON.stringify({
+ version: 1, generatedAt: "2026-09-28T12:00:00.000Z", branch: "main", sourceCommit, mirrorHead, subject: "s",
+ files: 1, bytes: 1, mirror: { files: 1, bytes: 1, packs: 1 }, tree: { files: 1, dirs: 1, bytes: 1 },
+ tarball: { href: TARBALL_HREF, bytes: 7, sha256: sha }, cloneUrl: CLONE_URL, treeHref: TREE_HREF,
+ audit: { objects: 1, commits: 1, gitleaks: "skipped" }, tools: { git: "x", filterRepo: "x" },
+ });
+}
+
test("scrub rules: the home-dir rule first, comments and blanks dropped, every literal left side denied", () => {
const rules = parseScrubRules(
["# a comment", "", `/srv/${PLANTED}==>/home/user`, " # indented comment", "literal:abc==>x", "regex:a+b==>c", "glob:*.x==>y", "bare-line", "a==>b==>c", "\r"].join("\n"),
@@ -139,10 +155,34 @@ test("scrub rules: the home-dir rule first, comments and blanks dropped, every l
"a==>b==>c",
]);
// filter-repo splits at the LAST ==>; regex and glob rules deny nothing.
- assert.deepEqual(rules.denied, [HOME, `/srv/${PLANTED}`, "abc", "bare-line", "a==>b"]);
- // A home dir of `/`, or one that IS the replacement, gets no built-in rule.
+ // Each denial is named by where it was written (the report's label).
+ assert.deepEqual(rules.denied, [
+ { text: HOME, from: "built-in home rule" },
+ { text: `/srv/${PLANTED}`, from: "scrub line 3 lhs" },
+ { text: "abc", from: "scrub line 5 lhs" },
+ { text: "bare-line", from: "scrub line 8 lhs" },
+ { text: "a==>b", from: "scrub line 9 lhs" },
+ ]);
+ // A home dir of `/`, or one that IS the replacement, gets no built-in rule;
+ // a trailing slash is dropped, or the rule would match nothing.
assert.deepEqual(parseScrubRules("", "/").lines, []);
assert.deepEqual(parseScrubRules("", "/home/user").lines, []);
+ assert.deepEqual(parseScrubRules("", `${HOME}/`).lines, [`${HOME}==>/home/user`]);
+});
+
+test("scrub rules: a byte-order mark and CRLF are dropped; an empty left side refuses (review L2)", () => {
+ // The reviewer's reproduction: a BOM made the first rule — and its implied
+ // denial — `\uFEFF/srv/…`, which matches nothing.
+ const bom = parseScrubRules(`\uFEFF/srv/${PLANTED}==>/home/user\r\nx==>y\r\n`, HOME);
+ assert.deepEqual(bom.lines, [`${HOME}==>/home/user`, `/srv/${PLANTED}==>/home/user`, "x==>y"]);
+ assert.deepEqual(bom.denied.map((d) => d.text), [HOME, `/srv/${PLANTED}`, "x"]);
+ for (const line of ["==>user", "literal:==>user", "regex:==>user", "glob:"]) {
+ assert.throws(
+ () => parseScrubRules(`ok==>fine\n${line}\n`, HOME),
+ (e) => e instanceof SourceRefusal && /scrub line 2 has an empty left side/.test(e.message),
+ line,
+ );
+ }
});
test("the operator's files: a missing one refuses by name; the denylist's i: and the implied left sides; the hash moves with them", async () => {
@@ -156,11 +196,11 @@ test("the operator's files: a missing one refuses by name; the denylist's i: and
loadSourceRules({ scrubFile: path.join(d, "scrub.txt"), denylistFile: path.join(d, "deny.txt"), homeDir: HOME }),
(e) => e instanceof SourceRefusal && /no denylist at .*deny\.txt/.test(e.message),
);
- writeFileSync(path.join(d, "deny.txt"), `# mine\ni:${SECRET.toUpperCase()}\n`);
+ writeFileSync(path.join(d, "deny.txt"), `\uFEFF# mine\r\ni:${SECRET.toUpperCase()}\r\n`);
const a = await loadSourceRules({ scrubFile: path.join(d, "scrub.txt"), denylistFile: path.join(d, "deny.txt"), homeDir: HOME });
assert.deepEqual(
- a.literals.map((l) => [l.bytes.toString(), l.ci]),
- [[SECRET, true], [HOME, false], [`/srv/${PLANTED}`, false]],
+ a.literals.map((l) => [l.bytes.toString(), l.ci, l.from]),
+ [[SECRET, true, "denylist line 2"], [HOME, false, "built-in home rule"], [`/srv/${PLANTED}`, false, "scrub line 1 lhs"]],
);
writeFileSync(path.join(d, "deny.txt"), `i:${SECRET}\nanother\n`);
const b = await loadSourceRules({ scrubFile: path.join(d, "scrub.txt"), denylistFile: path.join(d, "deny.txt"), homeDir: HOME });
@@ -177,7 +217,7 @@ test("the limits: 15,000 files and 24 MiB a file, inside Pages' 20,000 and 25 Mi
assert.equal(limitProblem(many.slice(1)), null);
});
-test("skip: an unchanged main with unchanged rules does nothing; a changed rule does not skip", async () => {
+test("skip: an unchanged main with unchanged rules and tools does nothing; a changed rule refuses AND withdraws the last publish", async () => {
const repo = sourceRepo();
const files = operatorFiles(`/srv/${PLANTED}==>/home/user\n`, "");
const logs: string[] = [];
@@ -186,19 +226,22 @@ test("skip: an unchanged main with unchanged rules does nothing; a changed rule
const pub = o.publicDir!;
const sourceCommit = gitIn(repo, "rev-parse", "main");
const rules = await loadSourceRules({ ...files, homeDir: HOME });
+ const mirrorHead = "a".repeat(40);
mkdirSync(path.join(pub, "source", MIRROR_DIR, "info"), { recursive: true });
mkdirSync(path.join(pub, "downloads"), { recursive: true });
writeFileSync(path.join(pub, "source", MIRROR_DIR, "info", "refs"), "x\trefs/heads/main\n");
writeFileSync(path.join(pub, "downloads", path.basename(TARBALL_HREF)), "tarball");
- writeFileSync(
- path.join(pub, "source", "manifest.json"),
- JSON.stringify({
- version: 1, generatedAt: "x", branch: "main", sourceCommit, mirrorHead: "a".repeat(40), subject: "s",
- files: 1, bytes: 1, mirror: {}, tree: {}, tarball: { href: TARBALL_HREF, bytes: 7, sha256: "b".repeat(64) },
- audit: {}, tools: {},
- }),
- );
- writeFileSync(path.join(path.dirname(pub), ".source-publish.json"), JSON.stringify({ sourceCommit, rulesHash: rules.rulesHash }));
+ writeFileSync(path.join(pub, "downloads", "snapshot.json"), "{}");
+ writeFileSync(path.join(pub, "source", "manifest.json"), fakeManifest(sourceCommit, mirrorHead));
+ const state = path.join(path.dirname(pub), ".source-publish.json");
+ const key = { sourceCommit, mirrorHead, rulesHash: rules.rulesHash, filterRepo: "false (given)" };
+ // Another filter-repo (an upgrade) does not skip…
+ writeFileSync(state, JSON.stringify({ ...key, filterRepo: "git filter-repo 0.1" }));
+ assert.equal(await publishSource({ ...o, check: true }), 1, "--check never skips, and reached `false`");
+ assert.ok(existsSync(path.join(pub, "source", "manifest.json")), "--check withdraws nothing");
+ // …the same one does.
+ writeFileSync(state, JSON.stringify(key));
+ logs.length = 0;
assert.equal(await publishSource(o), 0);
assert.match(logs.join("\n"), new RegExp(`up to date at ${sourceCommit.slice(0, 12)}; skipping`));
@@ -206,13 +249,68 @@ test("skip: an unchanged main with unchanged rules does nothing; a changed rule
logs.length = 0;
assert.equal(await publishSource(o), 1, "the new rule reached the rewrite");
assert.match(logs.join("\n"), /REFUSED: false --force --quiet exited 1/);
+ assert.match(logs.join("\n"), /the previous publish was WITHDRAWN/);
+ assert.ok(!existsSync(path.join(pub, "source")), "the last publish is withdrawn");
+ assert.ok(!existsSync(path.join(pub, "downloads", path.basename(TARBALL_HREF))));
+ assert.ok(!existsSync(path.join(pub, "downloads", "snapshot.json")));
+ assert.ok(!existsSync(state));
assert.equal(readdirSync(o.scratchRoot!).length, 0, "the scratch dir is removed");
});
+test("the scratch root: refused inside the checkout or the public dir, through a symlink too (review L12)", async () => {
+ const checkout = dir("chk");
+ const pub = path.join(dir("site"), "public");
+ const places: Array<[string, string]> = [["the checkout", checkout], ["the public dir", pub]];
+ assert.match((await scratchRootProblem(path.join(checkout, "tmp", "x"), places))!, /inside the checkout/);
+ assert.match((await scratchRootProblem(path.join(pub, "s"), places))!, /inside the public dir/);
+ const link = path.join(dir("links"), "into-checkout");
+ symlinkSync(checkout, link);
+ assert.match((await scratchRootProblem(path.join(link, "new"), places))!, /inside the checkout/);
+ assert.equal(await scratchRootProblem(path.join(TMP, "scratch"), places), null);
+ // …and publishSource says so before making anything.
+ const repo = sourceRepo();
+ const logs: string[] = [];
+ const o = opts(repo, operatorFiles("", ""), logs, { filterRepo: ["false"] });
+ assert.equal(await publishSource({ ...o, scratchRoot: path.join(o.publicDir!, "scratch") }), 1);
+ assert.match(logs.join("\n"), /REFUSED: the scratch root .* is inside the public dir/);
+ assert.ok(!existsSync(path.join(o.publicDir!, "scratch")));
+});
+
+test("no git repository here (a docker runtime, a tarball install): the build gets the empty state, the CLI a sentence (review L8)", async () => {
+ const bare = dir("no-git");
+ const pub = path.join(dir("site"), "public");
+ mkdirSync(path.join(pub, "source"), { recursive: true });
+ writeFileSync(path.join(pub, "source", "manifest.json"), "{}");
+ const logs: string[] = [];
+ const o: SourcePublishOpts = {
+ paths: { monorepoRoot: bare } as Paths,
+ publicDir: pub,
+ onLog: (l) => logs.push(l),
+ // No operator files at all: nothing is read before the repository check.
+ scrubFile: path.join(bare, "none.txt"),
+ denylistFile: path.join(bare, "none.txt"),
+ };
+ assert.equal(await publishSource({ ...o, check: true }), 1);
+ assert.ok(existsSync(path.join(pub, "source", "manifest.json")), "--check writes nothing");
+ logs.length = 0;
+ assert.equal(await publishSource(o), 1, "the CLI refuses");
+ assert.equal(logs[0], `[source] ${NO_REPOSITORY}`);
+ assert.ok(!logs.join("\n").includes("fatal"), "no raw git error");
+ assert.ok(!existsSync(path.join(pub, "source")), "an old publish is removed");
+ mkdirSync(path.join(pub, "source"), { recursive: true });
+ writeFileSync(path.join(pub, "source", "manifest.json"), "{}");
+ logs.length = 0;
+ assert.equal(await publishSource({ ...o, noRepository: "empty" }), 0, "buildHomepage builds on");
+ assert.equal(logs[0], `[source] ${NO_REPOSITORY}`);
+ assert.ok(!existsSync(path.join(pub, "source")));
+});
+
test("round trip: --check writes nothing; publish; a dumb clone of the mirror is main, scrubbed, from static files", async (t) => {
if (filterRepoProblem) return t.skip(`git-filter-repo unavailable: ${filterRepoProblem}`);
const repo = sourceRepo();
- const files = operatorFiles(`# the planted path\n/srv/${PLANTED}==>/home/user\n`, `i:${PLANTED}\n`);
+ // A byte-order mark and CRLF endings, as some editors write them: the rule
+ // must still scrub, and its left side must still be denied (review L2).
+ const files = operatorFiles(`\uFEFF# the planted path\r\n/srv/${PLANTED}==>/home/user\r\n`, `i:${PLANTED}\r\n`);
const logs: string[] = [];
const o = opts(repo, files, logs);
const pub = o.publicDir!;
@@ -241,6 +339,7 @@ test("round trip: --check writes nothing; publish; a dumb clone of the mirror is
assert.equal(manifest.audit.commits, 3);
assert.equal(manifest.audit.gitleaks, "skipped");
assert.ok(!("rulesHash" in manifest), "the rules hash is never published");
+ assert.ok(!("literals" in manifest.audit), "nor how many literals there are (review L3)");
assert.ok(existsSync(path.join(site, ".source-publish.json")), "…it is kept beside public/");
// The mirror: the allowlist and the dumb-HTTP files.
@@ -251,6 +350,19 @@ test("round trip: --check writes nothing; publish; a dumb clone of the mirror is
assert.equal(readFileSync(path.join(mirror, "HEAD"), "utf8"), "ref: refs/heads/main\n");
assert.equal(readFileSync(path.join(mirror, "info", "refs"), "utf8"), `${manifest.mirrorHead}\trefs/heads/main\n`);
assert.match(readFileSync(path.join(mirror, "objects", "info", "packs"), "utf8"), /^P pack-[0-9a-f]+\.pack$/m);
+ // Only packs and their indexes: git 2.55 also writes .rev files, which the
+ // allowlist leaves behind.
+ const packDir = readdirSync(path.join(mirror, "objects", "pack"));
+ assert.ok(packDir.length >= 2);
+ for (const f of packDir) assert.match(f, /^pack-[0-9a-f]+\.(pack|idx)$/);
+
+ // EVERY object in the published packs, reachable or not — what a dumb-HTTP
+ // reader gets — read from a plain copy of the published files (review L9).
+ const copy = path.join(dir("copy"), "m.git");
+ cpSync(mirror, copy, { recursive: true });
+ const all = execFileSync("git", ["--git-dir", copy, "cat-file", "--batch-all-objects", "--batch"], { maxBuffer: 1 << 26 });
+ assert.equal(all.indexOf(PLANTED), -1, "no object in the packs holds the planted path");
+ assert.ok(all.includes(Buffer.from("/home/user")), "…which was scrubbed, not dropped");
assert.ok(!existsSync(path.join(pub, "source", "index.html")), "the /source/ page keeps its route");
// The clone.
@@ -288,6 +400,39 @@ test("round trip: --check writes nothing; publish; a dumb clone of the mirror is
logs.length = 0;
assert.equal(await publishSource(o), 0);
assert.match(logs.join("\n"), /up to date at/);
+
+ // The deploy check (M1): a build's out/ that is this publish deploys…
+ const out = path.join(dir("out"), "out");
+ cpSync(path.join(pub, "source"), path.join(out, "source"), { recursive: true });
+ cpSync(path.join(pub, "downloads"), path.join(out, "downloads"), { recursive: true });
+ const check = { ...files, homeDir: HOME, publicDir: pub, sourceRepo: path.join(repo, ".git") };
+ const checkPaths = o.paths!;
+ assert.equal(await publishedSourceProblem(checkPaths, out, check), null);
+ // …a tampered tarball does not…
+ writeFileSync(path.join(out, "downloads", path.basename(TARBALL_HREF)), "other bytes");
+ assert.match((await publishedSourceProblem(checkPaths, out, check))!, /tarball is not the one its manifest describes/);
+ cpSync(path.join(pub, "downloads"), path.join(out, "downloads"), { recursive: true });
+ // …nor one of an older main.
+ writeFileSync(path.join(repo, "later.txt"), "x\n");
+ gitIn(repo, "add", "-A");
+ gitIn(repo, "commit", "-q", "-m", "later");
+ assert.match((await publishedSourceProblem(checkPaths, out, check))!, /main is now [0-9a-f]{12} — run `archilyzer build homepage`/);
+
+ // A REFUSAL WITHDRAWS THE PUBLISH (M1): deny a literal the mirror holds.
+ // The deploy check refuses first (other rules), then the publish refuses
+ // and removes public/source and the downloads.
+ writeFileSync(files.denylistFile, `i:${PLANTED}\nhello again\n`);
+ assert.match((await publishedSourceProblem(checkPaths, out, check))!, /audited under other rules/);
+ logs.length = 0;
+ assert.equal(await publishSource(o), 1);
+ assert.match(logs.join("\n"), /AUDIT REFUSED/);
+ assert.match(logs.join("\n"), /denylist line 2 \(len 11\)/);
+ assert.ok(!logs.join("\n").includes("hello again"), "the report never prints the literal");
+ assert.ok(!existsSync(path.join(pub, "source")), "public/source is withdrawn");
+ assert.ok(!existsSync(tarball), "the tarball is withdrawn");
+ assert.ok(!existsSync(path.join(pub, "downloads", "snapshot.json")));
+ assert.ok(!existsSync(path.join(site, ".source-publish.json")));
+ assert.match((await publishedSourceProblem(checkPaths, out, check))!, /no record of the rules/);
});
test("a denied literal no rule removes: refused, nothing written, the report never prints it", async (t) => {
@@ -299,17 +444,22 @@ test("a denied literal no rule removes: refused, nothing written, the report nev
const downloads = path.join(o.publicDir!, "downloads");
mkdirSync(downloads, { recursive: true });
writeFileSync(path.join(downloads, "snapshot.json"), "yesterday's");
- const before = statSync(path.join(downloads, "snapshot.json")).mtimeMs;
- assert.equal(await publishSource(o), 1);
+ assert.equal(await publishSource({ ...o, keepScratch: true }), 1);
const report = logs.join("\n");
assert.ok(!report.includes(SECRET), report);
+ assert.ok(!report.includes("is here"), "no byte from beside the hit (review L4)");
assert.match(report, /AUDIT REFUSED: 1 hit in \d+ objects/);
- assert.match(report, /#1 \(p…, len 13\): 1 in blob/);
- assert.match(report, /blob [0-9a-f]{12} keys\.txt #1 \(p…, len 13\): the \[REDACTED\] is here\./);
- assert.match(report, /add a rule to .*source-scrub\.txt or drop the file from history, then re-run\.$/);
+ assert.match(report, /denylist line 1 \(len 13\): 1 in blob/);
+ assert.match(report, /blob [0-9a-f]{12} keys\.txt \(byte 4\): denylist line 1 \(len 13\)/);
+ assert.match(report, /add a rule to .*source-scrub\.txt or drop the file from history, then re-run\./);
assert.ok(!existsSync(path.join(o.publicDir!, "source")), "no manifest, no mirror");
- assert.equal(readFileSync(path.join(downloads, "snapshot.json"), "utf8"), "yesterday's");
- assert.equal(statSync(path.join(downloads, "snapshot.json")).mtimeMs, before);
+ assert.ok(!existsSync(path.join(downloads, "snapshot.json")), "yesterday's snapshot is withdrawn too");
+ // --keep-scratch keeps the clone for a look, never the scrub rules.
+ const kept = /scratch kept at (\S+) \(replace\.txt, the scrub rules, deleted\)/.exec(report);
+ assert.ok(kept, report);
+ assert.ok(existsSync(path.join(kept[1], "bare")));
+ assert.ok(!existsSync(path.join(kept[1], "replace.txt")));
+ rmSync(kept[1], { recursive: true, force: true });
});
test("--no-source's clear: the manifest, mirror, tree, tarball and skip key go; a linked downloads/ goes as a link", async () => {
diff --git a/common/publish/source.ts b/common/publish/source.ts
@@ -31,7 +31,7 @@
import { createHash } from "node:crypto";
import { createReadStream, existsSync } from "node:fs";
-import { cp, lstat, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
+import { cp, lstat, mkdir, mkdtemp, readdir, readFile, realpath, rm, stat, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { runChildIntoLog } from "../jobs/runChild";
@@ -53,6 +53,7 @@ import {
cleanGitEnv,
dedupeLiterals,
formatAuditReport,
+ operatorLines,
maskLiterals,
onPath,
parseDenylist,
@@ -69,6 +70,18 @@ export { SourceRefusal } from "./sourceAudit";
/** The one branch mirrored (an operator decision: no other refs, no tags). */
export const SOURCE_BRANCH = "main";
+/**
+ * The step's own version, hashed into the rules hash (the skip key and the
+ * deploy check). BUMP IT WHENEVER THE SCRUB OR THE AUDIT CHANGES — what the
+ * rules parse to, what the gate reads, what it refuses — so an unchanged
+ * `main` is re-published through the fixed step instead of skipped, and a
+ * `homepage/out` built by the old step refuses to deploy.
+ * 1 — release 12 slice R.
+ * 2 — release 12 slice R review: BOM/CRLF/trailing-slash parsing, empty
+ * left sides refused, `404.html` refused, no context bytes.
+ */
+export const SOURCE_STEP_VERSION = 2;
+
/** What `pipx run` fetches when `git filter-repo` is not installed. */
export const FILTER_REPO_PIPX_SPEC = "git-filter-repo==2.47.0";
export const FILTER_REPO_INSTALL = "pipx install git-filter-repo";
@@ -112,6 +125,10 @@ export type SourcePublishOpts = PublishOpts & {
now?: () => Date;
// The home dir the built-in rule scrubs. Default: os.homedir().
homeDir?: string;
+ // A checkout with no git repository: "refuse" (the CLI: exit 1) or "empty"
+ // (buildHomepage: the /source page's empty state, exit 0). Either way it
+ // says NO_REPOSITORY and removes an old publish.
+ noRepository?: "refuse" | "empty";
};
// ── the operator's files ────────────────────────────────────────────────────
@@ -121,37 +138,55 @@ export type ScrubRules = {
// the operator's rules in order (comments and blank lines dropped —
// filter-repo itself would treat a `#` line as a literal to replace).
lines: string[];
- // Every rule's LITERAL left side: denied, exact case, by implication.
- denied: string[];
+ // Every rule's LITERAL left side: denied, exact case, by implication, with
+ // where it was written (the report's name for it).
+ denied: Array<{ text: string; from: string }>;
};
+export const BUILT_IN_HOME_RULE = "built-in home rule";
+
/**
* The scrub file's text as rules. A line is `lhs==>rhs` (split at the LAST
* `==>`, as filter-repo splits it), `literal:lhs==>rhs`, `regex:…==>…` or
* `glob:…==>…`; a line with no `==>` is replaced by filter-repo's
* `***REMOVED***`. Lines whose first non-blank character is `#` are comments.
+ * A byte-order mark and CRLF endings are dropped first (operatorLines), and
+ * the home dir loses a trailing `/`: either would make a rule — and the
+ * denial it implies — silently match nothing. An empty left side is a
+ * refusal, not a rule filter-repo would skip.
*/
export function parseScrubRules(text: string, homeDir: string): ScrubRules {
const lines: string[] = [];
+ const denied: ScrubRules["denied"] = [];
+ const home = homeDir.replace(/\/+$/, "");
// A home dir of `/` (a container user) would scrub every slash, and one
// that IS the replacement would deny the replacement itself.
- if (homeDir.length > 1 && homeDir !== HOME_REPLACEMENT) {
- lines.push(`${homeDir}==>${HOME_REPLACEMENT}`);
+ if (home.length > 1 && home !== HOME_REPLACEMENT) {
+ lines.push(`${home}==>${HOME_REPLACEMENT}`);
+ denied.push({ text: home, from: BUILT_IN_HOME_RULE });
}
- for (const raw of text.split("\n")) {
- const line = raw.replace(/\r$/, "");
+ operatorLines(text).forEach((line, n) => {
const t = line.trim();
- if (t === "" || t.startsWith("#")) continue;
- lines.push(line);
- }
- const denied: string[] = [];
- for (const line of lines) {
+ if (t === "" || t.startsWith("#")) return;
const i = line.lastIndexOf("==>");
let lhs = i === -1 ? line : line.slice(0, i);
- if (lhs.startsWith("regex:") || lhs.startsWith("glob:")) continue;
- if (lhs.startsWith("literal:")) lhs = lhs.slice("literal:".length);
- if (lhs) denied.push(lhs);
- }
+ const from = `scrub line ${n + 1} lhs`;
+ for (const prefix of ["regex:", "glob:", "literal:"]) {
+ if (lhs.startsWith(prefix)) {
+ if (lhs.length === prefix.length) {
+ throw new SourceRefusal(`scrub line ${n + 1} has an empty left side — it would match nothing and deny nothing`);
+ }
+ if (prefix === "literal:") lhs = lhs.slice(prefix.length);
+ else lhs = "";
+ break;
+ }
+ }
+ if (i === 0) {
+ throw new SourceRefusal(`scrub line ${n + 1} has an empty left side — it would match nothing and deny nothing`);
+ }
+ lines.push(line);
+ if (lhs) denied.push({ text: lhs, from });
+ });
return { lines, denied };
}
@@ -193,12 +228,12 @@ export async function loadSourceRules(opts: {
const scrub = parseScrubRules(scrubText, opts.homeDir ?? os.homedir());
const literals = dedupeLiterals([
...parseDenylist(denyText),
- ...scrub.denied.map((d) => ({ bytes: Buffer.from(d, "utf8"), ci: false })),
+ ...scrub.denied.map((d) => ({ bytes: Buffer.from(d.text, "utf8"), ci: false, from: d.from })),
]);
const rulesHash = createHash("sha256")
.update(
JSON.stringify({
- v: SOURCE_MANIFEST_VERSION,
+ step: SOURCE_STEP_VERSION,
rules: scrub.lines,
literals: literals.map((l) => `${l.ci ? "i" : "x"}:${l.bytes.toString("hex")}`),
}),
@@ -369,12 +404,20 @@ export function sourcePublicDir(paths: Paths, override?: string): string {
}
// The skip key, kept BESIDE the public dir, never in it: it holds the rules
-// hash, and public/ is deployed.
+// hash, and public/ is deployed. It is also what `deployHomepage` checks
+// homepage/out's source against (publishedSourceProblem).
function statePath(publicDir: string): string {
return path.join(path.dirname(publicDir), ".source-publish.json");
}
-type PublishState = { sourceCommit: string; rulesHash: string };
+type PublishState = {
+ sourceCommit: string;
+ mirrorHead: string;
+ // The rules, the literals and SOURCE_STEP_VERSION.
+ rulesHash: string;
+ // "<label> <version>": an upgrade may rewrite every id, so it re-publishes.
+ filterRepo: string;
+};
async function readJson(file: string): Promise<unknown> {
try {
@@ -389,69 +432,191 @@ export async function readPublishedManifest(publicDir: string): Promise<SourceMa
return parseSourceManifest(await readJson(path.join(publicDir, "source", "manifest.json")));
}
+/** Said, and nothing mirrored, in a checkout without git history. */
+export const NO_REPOSITORY =
+ "no git repository here; nothing to mirror — the /source page will show its empty state";
+
+/**
+ * Remove a publish from `publicDir`: the manifest FIRST (the page never
+ * describes a half-removed tree), the skip key, the mirror and the tree, the
+ * tarball and snapshot.json. Link-safe like the install: a linked
+ * `source/` or `downloads/` (a worktree's, into the primary) goes as a link,
+ * its target untouched. True when there was something to remove.
+ */
+export async function removePublishedSource(publicDir: string): Promise<boolean> {
+ const pubSource = path.join(publicDir, "source");
+ const pubDownloads = path.join(publicDir, "downloads");
+ const had =
+ existsSync(path.join(pubSource, "manifest.json")) ||
+ existsSync(path.join(pubSource, MIRROR_DIR)) ||
+ existsSync(path.join(pubDownloads, TARBALL_NAME));
+ await rm(path.join(pubSource, "manifest.json"), { force: true });
+ await rm(statePath(publicDir), { force: true });
+ // fs.rm reads the path with lstat: a linked public/source goes as a link.
+ await rm(pubSource, { recursive: true, force: true });
+ const linked = await lstat(pubDownloads).then((s) => s.isSymbolicLink(), () => false);
+ if (linked) {
+ await rm(pubDownloads, { force: true });
+ } else {
+ await rm(path.join(pubDownloads, "snapshot.json"), { force: true });
+ await rm(path.join(pubDownloads, TARBALL_NAME), { force: true });
+ }
+ return had;
+}
+
+// The checkout's git common dir, or null when there is no repository here (or
+// no git at all — a docker runtime, a tarball install).
+async function commonDir(ctx: Ctx, cwd: string): Promise<string | null> {
+ const r = await run(ctx, "git", ["rev-parse", "--path-format=absolute", "--git-common-dir"], {
+ cwd,
+ timeoutMs: 30_000,
+ allowFail: true,
+ });
+ if (r.code === 0) return lastLine(r.out);
+ if (!onPath("git", ctx.env.PATH) || r.out.some((l) => /not a git repository/i.test(l))) return null;
+ const tail = r.out.slice(-2).map((l) => maskLiterals(l, ctx.literals)).join(" / ");
+ throw new SourceRefusal(`git rev-parse exited ${r.code}${tail ? `: ${tail}` : ""}`);
+}
+
+// The real path of `p`, or of its deepest existing ancestor with the rest
+// appended: a scratch root may not exist yet, and a symlink must not hide
+// where it lands.
+async function landsAt(p: string): Promise<string> {
+ const abs = path.resolve(p);
+ let head = abs;
+ const rest: string[] = [];
+ for (;;) {
+ try {
+ return path.join(await realpath(head), ...rest);
+ } catch {
+ const up = path.dirname(head);
+ if (up === head) return abs;
+ rest.unshift(path.basename(head));
+ head = up;
+ }
+ }
+}
+
+const within = (child: string, parent: string) =>
+ child === parent || child.startsWith(parent.endsWith(path.sep) ? parent : parent + path.sep);
+
+/**
+ * Why `scratchRoot` may not hold the scratch dir, or null. Inside the checkout
+ * or the public dir, a kept scratch (`--keep-scratch`) could be committed or
+ * published — and it held replace.txt, the scrub rules in plain text.
+ */
+export async function scratchRootProblem(
+ scratchRoot: string,
+ places: ReadonlyArray<[string, string]>,
+): Promise<string | null> {
+ const at = await landsAt(scratchRoot);
+ for (const [what, dir] of places) {
+ if (within(at, await landsAt(dir))) {
+ return `the scratch root ${tildify(scratchRoot)} (ARCHILYZER_SOURCE_SCRATCH) is inside ${what}, where a kept scratch dir could be committed or published — point it outside`;
+ }
+ }
+ return null;
+}
+
// ── the step ────────────────────────────────────────────────────────────────
/**
* Publish the source (see the header). Returns 0 when published, skipped or
* checked, 1 when refused or cancelled; the refusal's reason (and the audit's
- * redacted report) is in the log. Throws only on a bug or an I/O failure.
+ * report) is in the log. Throws only on a bug or an I/O failure.
+ *
+ * A REFUSAL WITHDRAWS THE PREVIOUS PUBLISH. Once the rules are loaded, any
+ * outcome but success — an audit hit, a limit, a missing tool, a cancel, a
+ * crash — removes what the last publish left (removePublishedSource): it was
+ * audited under rules that may not be today's, and a refusal is the best
+ * evidence that they are not. `--check` writes nothing, this included.
+ *
+ * `noRepository: "empty"` (buildHomepage) turns a checkout with no git
+ * repository into an empty /source page and 0; the CLI's default refuses.
*/
export async function publishSource(opts: SourcePublishOpts = {}): Promise<number> {
const paths = opts.paths ?? getPaths();
const onLog = opts.onLog ?? terminalLog;
const signal = opts.signal ?? new AbortController().signal;
const ctx: Ctx = { onLog, signal, env: cleanGitEnv(opts.env ?? process.env), literals: [] };
+ const publicDir = sourcePublicDir(paths, opts.publicDir);
+ const progress = { rulesLoaded: false };
+ const withdraw = async () => {
+ if (!progress.rulesLoaded || opts.check) return;
+ if (await removePublishedSource(publicDir)) {
+ onLog(
+ "[source] the previous publish was WITHDRAWN (mirror, tree, tarball): it was audited under rules that may not be today's. /source shows its empty state until a publish passes.",
+ );
+ }
+ };
+ let code: number;
try {
- return await publish(opts, paths, ctx);
+ code = await publish(opts, paths, ctx, publicDir, progress);
} catch (err) {
if (err instanceof Cancelled || signal.aborted) {
onLog("[source] cancelled — nothing published");
- return 1;
- }
- if (err instanceof SourceRefusal) {
+ code = 1;
+ } else if (err instanceof SourceRefusal) {
onLog(`[source] REFUSED: ${err.message}`);
- return 1;
+ code = 1;
+ } else {
+ await withdraw().catch(() => {});
+ throw err;
}
- throw err;
}
+ if (code !== 0) await withdraw();
+ return code;
}
-async function publish(opts: SourcePublishOpts, paths: Paths, ctx: Ctx): Promise<number> {
+async function publish(
+ opts: SourcePublishOpts,
+ paths: Paths,
+ ctx: Ctx,
+ publicDir: string,
+ progress: { rulesLoaded: boolean },
+): Promise<number> {
const { onLog } = ctx;
const started = Date.now();
- const publicDir = sourcePublicDir(paths, opts.publicDir);
const pubSource = path.join(publicDir, "source");
const pubDownloads = path.join(publicDir, "downloads");
- // 1. The private main.
- const sourceRepo =
- opts.sourceRepo ??
- lastLine(
- (
- await run(ctx, "git", ["rev-parse", "--path-format=absolute", "--git-common-dir"], {
- cwd: paths.monorepoRoot,
- timeoutMs: 30_000,
- })
- ).out,
- );
+ // 1. The private main — or no repository at all (L8: the docker runtime,
+ // a tarball install), where nothing can be mirrored and nothing can leak.
+ const sourceRepo = opts.sourceRepo ?? (await commonDir(ctx, paths.monorepoRoot));
+ if (sourceRepo === null) {
+ onLog(`[source] ${NO_REPOSITORY}`);
+ if (opts.check) return 1;
+ if (await removePublishedSource(publicDir)) onLog("[source] the previous publish was removed.");
+ return opts.noRepository === "empty" ? 0 : 1;
+ }
const sourceCommit = await revParse(ctx, sourceRepo, `refs/heads/${SOURCE_BRANCH}^{commit}`);
- // 2. The operator's rules.
+ // 2. The operator's rules. From here on, a failure withdraws the last publish.
const rules = await loadSourceRules({
scrubFile: opts.scrubFile ?? paths.sourceScrubFile,
denylistFile: opts.denylistFile ?? paths.sourceDenylistFile,
homeDir: opts.homeDir,
});
ctx.literals = rules.literals;
+ progress.rulesLoaded = true;
- // 3. Nothing changed: skip.
+ // 3. The tools (the filter-repo version is part of the skip key).
+ const filterRepo: FilterRepoChoice =
+ opts.filterRepo && opts.filterRepo.length > 0
+ ? { argv: opts.filterRepo, label: opts.filterRepo.join(" "), version: "(given)" }
+ : await resolveFilterRepo({ env: ctx.env, signal: ctx.signal });
+ const filterRepoId = `${filterRepo.label} ${filterRepo.version}`;
+
+ // 4. Nothing changed: skip.
if (!opts.force && !opts.check) {
const manifest = await readPublishedManifest(publicDir);
- const state = (await readJson(statePath(publicDir))) as PublishState | null;
+ const state = (await readJson(statePath(publicDir))) as Partial<PublishState> | null;
if (
manifest?.sourceCommit === sourceCommit &&
state?.sourceCommit === sourceCommit &&
state.rulesHash === rules.rulesHash &&
+ state.filterRepo === filterRepoId &&
+ state.mirrorHead === manifest.mirrorHead &&
existsSync(path.join(pubSource, MIRROR_DIR, "info", "refs")) &&
existsSync(path.join(pubDownloads, TARBALL_NAME))
) {
@@ -464,19 +629,19 @@ async function publish(opts: SourcePublishOpts, paths: Paths, ctx: Ctx): Promise
`${tildify(path.join(pubSource, "index.html"))} exists and would replace the /source/ page — remove it`,
);
}
-
- // 4. The tools.
- const filterRepo: FilterRepoChoice =
- opts.filterRepo && opts.filterRepo.length > 0
- ? { argv: opts.filterRepo, label: opts.filterRepo.join(" "), version: "(given)" }
- : await resolveFilterRepo({ env: ctx.env, signal: ctx.signal });
const gitVersion = lastLine((await run(ctx, "git", ["--version"], { cwd: os.tmpdir(), timeoutMs: 30_000 })).out)
.replace(/^git version /, "");
- onLog(`[source] main ${sourceCommit.slice(0, 12)}; git ${gitVersion}; ${filterRepo.label} ${filterRepo.version}`);
+ onLog(`[source] main ${sourceCommit.slice(0, 12)}; git ${gitVersion}; ${filterRepoId}`);
const scratchRoot = opts.scratchRoot ?? paths.sourceScratchDir;
+ const badRoot = await scratchRootProblem(scratchRoot, [
+ ["the checkout", paths.monorepoRoot],
+ ["the public dir", publicDir],
+ ]);
+ if (badRoot) throw new SourceRefusal(badRoot);
await mkdir(scratchRoot, { recursive: true });
const scratch = await mkdtemp(path.join(scratchRoot, "archilyzer-source-"));
+ const replace = path.join(scratch, "replace.txt");
try {
const bare = path.join(scratch, "bare");
@@ -491,8 +656,7 @@ async function publish(opts: SourcePublishOpts, paths: Paths, ctx: Ctx): Promise
await run(ctx, "git", ["--git-dir", bare, "remote", "remove", "origin"], { cwd: bare, timeoutMs: 30_000 });
// 6. The rewrite. --force: filter-repo wants a fresh clone with an origin.
- const replace = path.join(scratch, "replace.txt");
- await writeFile(replace, rules.scrub.lines.join("\n") + "\n");
+ await writeFile(replace, rules.scrub.lines.join("\n") + "\n", { mode: 0o600 });
onLog(`[source] rewriting history (${rules.scrub.lines.length} scrub rule${rules.scrub.lines.length === 1 ? "" : "s"})…`);
await run(
ctx,
@@ -607,7 +771,10 @@ async function publish(opts: SourcePublishOpts, paths: Paths, ctx: Ctx): Promise
}
const mirrorFiles = await walkFiles(stageMirror);
- // The manifest, staged with the rest so the file sweep reads it too.
+ // The manifest, staged with the rest so the file sweep reads it too. It
+ // carries no rules hash and no literal count: `plans/` publishes which
+ // literals the plan put in the files, so either would say whether the
+ // operator added private ones — and the hash would confirm a guess.
const staged = await walkFiles(stage);
const manifest: SourceManifest = {
version: SOURCE_MANIFEST_VERSION,
@@ -628,12 +795,11 @@ async function publish(opts: SourcePublishOpts, paths: Paths, ctx: Ctx): Promise
cloneUrl: CLONE_URL,
treeHref: TREE_HREF,
audit: {
- literals: rules.literals.length,
objects: audit.objects,
commits: audit.commits,
gitleaks: audit.gitleaks === "clean" ? "clean" : "skipped",
},
- tools: { git: gitVersion, filterRepo: `${filterRepo.label} ${filterRepo.version}` },
+ tools: { git: gitVersion, filterRepo: filterRepoId },
};
const manifestText = JSON.stringify(manifest, null, 2) + "\n";
await writeFile(path.join(stageSource, "manifest.json"), manifestText);
@@ -674,7 +840,7 @@ async function publish(opts: SourcePublishOpts, paths: Paths, ctx: Ctx): Promise
await ownDir(pubDownloads);
await copyPublicFile(tarball, path.join(pubDownloads, TARBALL_NAME));
await writePublicFile(path.join(pubDownloads, "snapshot.json"), snapshotText);
- const state: PublishState = { sourceCommit, rulesHash: rules.rulesHash };
+ const state: PublishState = { sourceCommit, mirrorHead, rulesHash: rules.rulesHash, filterRepo: filterRepoId };
await writeFile(statePath(publicDir), JSON.stringify(state, null, 2) + "\n");
await writePublicFile(path.join(pubSource, "manifest.json"), manifestText);
@@ -682,8 +848,13 @@ async function publish(opts: SourcePublishOpts, paths: Paths, ctx: Ctx): Promise
onLog(`[source] published ${summary} (${elapsed(started)})`);
return 0;
} finally {
- if (opts.keepScratch) onLog(`[source] scratch kept at ${scratch}`);
- else await rm(scratch, { recursive: true, force: true });
+ if (opts.keepScratch) {
+ // The scrub rules in plain text never outlive the run.
+ await rm(replace, { force: true });
+ onLog(`[source] scratch kept at ${scratch} (replace.txt, the scrub rules, deleted)`);
+ } else {
+ await rm(scratch, { recursive: true, force: true });
+ }
}
}
@@ -692,34 +863,16 @@ function elapsed(started: number): string {
}
/**
- * `build homepage --no-source`: remove what an earlier publish left (the
- * manifest first, so the page never describes a half-removed tree), because
+ * `build homepage --no-source`: remove what an earlier publish left, because
* it was audited against the rules of ITS day. The pages then show their
- * empty states. Link-safe like the install: a linked directory is replaced,
- * never followed.
+ * empty states.
*/
export async function clearPublishedSource(
opts: PublishOpts & { publicDir?: string } = {},
): Promise<void> {
const paths = opts.paths ?? getPaths();
const onLog = opts.onLog ?? terminalLog;
- const publicDir = sourcePublicDir(paths, opts.publicDir);
- const pubSource = path.join(publicDir, "source");
- const pubDownloads = path.join(publicDir, "downloads");
- const had = existsSync(path.join(pubSource, "manifest.json")) || existsSync(path.join(pubDownloads, TARBALL_NAME));
- await rm(path.join(pubSource, "manifest.json"), { force: true });
- await rm(statePath(publicDir), { force: true });
- // fs.rm reads the path with lstat: a linked public/source goes as a link.
- await rm(pubSource, { recursive: true, force: true });
- // A linked downloads/ (a worktree's, into the primary) is dropped as a link,
- // never reached through: its files are the other checkout's.
- const linked = await lstat(pubDownloads).then((s) => s.isSymbolicLink(), () => false);
- if (linked) {
- await rm(pubDownloads, { force: true });
- } else {
- await rm(path.join(pubDownloads, "snapshot.json"), { force: true });
- await rm(path.join(pubDownloads, TARBALL_NAME), { force: true });
- }
+ const had = await removePublishedSource(sourcePublicDir(paths, opts.publicDir));
onLog(
had
? "[notice] --no-source: the previously published source (mirror, tree, tarball) was removed — this build ships none.\n"
@@ -727,6 +880,78 @@ export async function clearPublishedSource(
);
}
+/**
+ * Why homepage/out's source may not be deployed, as one sentence — or null.
+ * `deployHomepage` asks before every deploy, production and preview alike: a
+ * deploy-only ships `out/` as the last build left it, and that build's source
+ * was audited under the rules of its day. It deploys only when the last
+ * publish (the skip key beside public/) was made under TODAY's rules and step
+ * (`rulesHash`), of TODAY's `main`, and is the one in `out/` (its mirror head,
+ * its tarball). An `out/` with no source at all — a `--no-source` build —
+ * deploys as it always has.
+ */
+export async function publishedSourceProblem(
+ paths: Paths,
+ outDir: string,
+ opts: {
+ publicDir?: string;
+ scrubFile?: string;
+ denylistFile?: string;
+ homeDir?: string;
+ sourceRepo?: string;
+ env?: NodeJS.ProcessEnv;
+ } = {},
+): Promise<string | null> {
+ const outSource = path.join(outDir, "source");
+ const outTarball = path.join(outDir, "downloads", TARBALL_NAME);
+ if (!existsSync(outSource) && !existsSync(outTarball)) return null;
+ const rebuild = "run `archilyzer build homepage` (it re-audits), then deploy";
+ const manifest = parseSourceManifest(await readJson(path.join(outSource, "manifest.json")));
+ if (!manifest) return `homepage/out holds a source publish without a valid manifest — ${rebuild}`;
+ const state = (await readJson(statePath(sourcePublicDir(paths, opts.publicDir)))) as Partial<PublishState> | null;
+ if (!state?.rulesHash || !state.mirrorHead || !state.sourceCommit) {
+ return `homepage/out's source has no record of the rules it was audited under — ${rebuild}`;
+ }
+ let rules: SourceRules;
+ try {
+ rules = await loadSourceRules({
+ scrubFile: opts.scrubFile ?? paths.sourceScrubFile,
+ denylistFile: opts.denylistFile ?? paths.sourceDenylistFile,
+ homeDir: opts.homeDir,
+ });
+ } catch (err) {
+ if (err instanceof SourceRefusal) return `${err.message}; homepage/out's source cannot be checked`;
+ throw err;
+ }
+ if (state.rulesHash !== rules.rulesHash) {
+ return `homepage/out's source was audited under other rules (or an older version of the step) — ${rebuild}`;
+ }
+ if (state.mirrorHead !== manifest.mirrorHead) {
+ return `homepage/out's source (${manifest.mirrorHead.slice(0, 12)}) is not the last publish (${state.mirrorHead.slice(0, 12)}) — ${rebuild}`;
+ }
+ const ctx: Ctx = {
+ onLog: () => {},
+ signal: new AbortController().signal,
+ env: cleanGitEnv(opts.env ?? process.env),
+ literals: rules.literals,
+ };
+ let main: string | null = null;
+ try {
+ const repo = opts.sourceRepo ?? (await commonDir(ctx, paths.monorepoRoot));
+ main = repo ? await revParse(ctx, repo, `refs/heads/${SOURCE_BRANCH}^{commit}`) : null;
+ } catch (err) {
+ if (!(err instanceof SourceRefusal)) throw err;
+ }
+ if (main === null) return `homepage/out holds a source publish, and main cannot be read here — ${rebuild}`;
+ if (main !== state.sourceCommit || main !== manifest.sourceCommit) {
+ return `homepage/out's source mirrors main at ${manifest.sourceCommit.slice(0, 12)}, and main is now ${main.slice(0, 12)} — ${rebuild}`;
+ }
+ if (existsSync(outTarball) && (await sha256File(outTarball)) !== manifest.tarball.sha256) {
+ return `homepage/out's source tarball is not the one its manifest describes — ${rebuild}`;
+ }
+ return null;
+}
+
// ── `archilyzer source audit` ───────────────────────────────────────────────
/**
