commit 30e05773e5139737ffb8dfa277c9f5e159898f40
parent 066f7e7cea04c3bd50969796a5477bbc25d449e9
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 5 Oct 2026 03:30:39 -0400
common: the cited out/ allowlist audit — builtSiteProblem and builtBundleProblem refuse a cited build holding anything but reports, moments, cited media and the shell; a full build of a site now cited is refused; build site fails on either; the hub drops reports/m/media
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
3 files changed, 215 insertions(+), 10 deletions(-)
diff --git a/common/bin/compose-hub.ts b/common/bin/compose-hub.ts
@@ -97,6 +97,10 @@ export const SITE_ONLY_PUBLIC_ENTRIES: readonly string[] = [
"digests",
"stats",
"archives",
+ // A report site's reports, moments and cited media (publish/composeReports.ts).
+ "reports",
+ "m",
+ "media",
"site.json",
"tags.json",
"duplicates.json",
diff --git a/common/lib/builtExport.ts b/common/lib/builtExport.ts
@@ -13,7 +13,7 @@
// public/ into out/. So the check is a file read, and it is cheap enough to do
// before every deploy.
-import { existsSync, readFileSync, statSync } from "node:fs";
+import { existsSync, readdirSync, readFileSync, statSync, type Dirent } from "node:fs";
import path from "node:path";
/**
@@ -64,7 +64,7 @@ export function builtSiteProblem(outDir: string, siteId: string): string | null
if (corpusSiteIdIn(outDir) !== asked) {
return `export/out holds an incomplete build of "${asked}" (its corpus.json does not name it) — build ${asked} first`;
}
- return null;
+ return citedBuildProblem(outDir);
}
/**
@@ -95,7 +95,7 @@ export function builtBundleProblem(outDir: string, siteId: string): string | nul
if (described !== asked) {
return `${outDir} describes "${described}", not "${asked}" (corpus.json)`;
}
- return null;
+ return citedBuildProblem(outDir);
}
/**
@@ -149,10 +149,172 @@ export function builtAudienceProblem(outDir: string): string | null {
* logs or throws, or null.
*/
export function deployAudienceProblem(
- site: { siteId: string; audience?: string },
+ site: { siteId: string; audience?: string; publish?: string },
+ outDir: string,
+): string | null {
+ return siteDeployProblem(site) ?? builtAudienceProblem(outDir) ?? builtScopeProblem(site, outDir);
+}
+
+// ─── A CITED build (site.json `publish: "cited"`) ───
+//
+// A cited site publishes its reports and the moments they cite, and nothing
+// else (plans/report-sites.md). export/public is shared by every site's build
+// in turn, so compose prunes everything corpus-shaped before it writes a cited
+// site — and this audit is the second line: after `next build`, a cited out/
+// may hold only what the list below names. Anything else (a stale summaries
+// tree, a transcripts shard, an archive, a service worker) refuses the build
+// and every deploy of it: builtSiteProblem and builtBundleProblem ask it, so
+// runDeployIntoLog, the container deploy phase, deploySite, the editor's
+// deploy actions and docker/build-site.sh all refuse what it refuses, and
+// `archilyzer build site` fails on it (publish/build.ts runBuildPhase).
+//
+// The bundle names its own scope — corpus.json's `site.scope: "cited"`, which
+// compose always writes for a cited site — so the audit needs no site config.
+// A site configured cited whose out/ is a FULL build is builtScopeProblem's.
+
+// What a cited out/ may hold at its top level.
+export const CITED_OUT_ALLOWED_DIRS: readonly string[] = [
+ // Next's assets, and the routes a cited build still renders (the shell's
+ // pages that say "not on this site", the not-found page).
+ "_next",
+ "_not-found",
+ "404",
+ "ask",
+ "changelog",
+ "downloads",
+ "duplicates",
+ "offline",
+ // What compose's reports stage writes: the reports and their stills, the
+ // moment pages, the cited media (with each route's placeholder, `_none`).
+ "reports",
+ "m",
+ "media",
+ "icons",
+];
+
+export const CITED_OUT_ALLOWED_FILES: readonly string[] = [
+ "site.json",
+ "corpus.json",
+ "llms.txt",
+ "robots.txt",
+ "sitemap.xml",
+ "_headers",
+ "index.html",
+ "index.txt",
+ "404.html",
+ "favicon.ico",
+ "manifest.webmanifest",
+ // export/public's checked-in assets.
+ "file.svg",
+ "globe.svg",
+ "next.svg",
+ "vercel.svg",
+ "window.svg",
+];
+
+// Next's per-segment payloads for the root route (`__next._tree.txt`, …).
+const NEXT_SEGMENT_FILE_RE = /^__next\..+\.txt$/;
+
+// What `media/` may hold: the cited clips and post captures, nothing else.
+export const CITED_MEDIA_ALLOWED_DIRS: readonly string[] = ["clips", "posts"];
+
+// Cloudflare Pages' own limits: a file of at most 25 MiB, at most 20,000 files.
+export const PAGES_MAX_FILE_BYTES = 25 * 1024 * 1024;
+export const PAGES_MAX_FILES = 20_000;
+
+// corpus.json's `site.scope`, or null.
+function builtScopeIn(outDir: string): string | null {
+ try {
+ const parsed: unknown = JSON.parse(readFileSync(path.join(outDir, "corpus.json"), "utf8"));
+ const scope = (parsed as { site?: { scope?: unknown } } | null)?.site?.scope;
+ return typeof scope === "string" ? scope : null;
+ } catch {
+ return null;
+ }
+}
+
+// Whether the build in `outDir` is a cited site's (its corpus.json says so).
+export function isCitedBuild(outDir: string): boolean {
+ return builtScopeIn(outDir) === "cited";
+}
+
+/**
+ * Why the CITED build in `outDir` may not ship, as one sentence naming what
+ * it holds that a cited site may not — or null when it may, or when the build
+ * is not a cited one.
+ */
+export function citedBuildProblem(outDir: string): string | null {
+ if (!isCitedBuild(outDir)) return null;
+ const extra: string[] = [];
+ let entries: Dirent[];
+ try {
+ entries = readdirSync(outDir, { withFileTypes: true });
+ } catch {
+ return `${outDir} is a cited build that cannot be read`;
+ }
+ for (const e of entries) {
+ if (e.isDirectory()) {
+ if (!CITED_OUT_ALLOWED_DIRS.includes(e.name)) extra.push(`${e.name}/`);
+ } else if (!CITED_OUT_ALLOWED_FILES.includes(e.name) && !NEXT_SEGMENT_FILE_RE.test(e.name)) {
+ extra.push(e.name);
+ }
+ }
+ const mediaDir = path.join(outDir, "media");
+ if (existsSync(mediaDir)) {
+ for (const e of readdirSync(mediaDir, { withFileTypes: true })) {
+ if (!e.isDirectory() || !CITED_MEDIA_ALLOWED_DIRS.includes(e.name)) {
+ extra.push(`media/${e.name}${e.isDirectory() ? "/" : ""}`);
+ }
+ }
+ }
+ if (extra.length > 0) {
+ extra.sort();
+ const shown = extra.slice(0, 12).join(", ") + (extra.length > 12 ? `, … (${extra.length} in all)` : "");
+ return (
+ `${outDir} is a cited build, which publishes only reports and their moments, ` +
+ `but it also holds ${shown} — compose the site again`
+ );
+ }
+ // The Pages limits, which a cited build is small enough to walk for.
+ let files = 0;
+ const oversize: string[] = [];
+ const walk = (dir: string, rel: string): void => {
+ for (const e of readdirSync(dir, { withFileTypes: true })) {
+ const p = path.join(dir, e.name);
+ const r = rel ? `${rel}/${e.name}` : e.name;
+ if (e.isDirectory()) walk(p, r);
+ else {
+ files++;
+ if (statSync(p).size > PAGES_MAX_FILE_BYTES) oversize.push(r);
+ }
+ }
+ };
+ walk(outDir, "");
+ if (oversize.length > 0) {
+ return `${outDir} holds ${oversize.length} file(s) over Pages' 25 MiB limit: ${oversize.slice(0, 5).join(", ")}`;
+ }
+ if (files > PAGES_MAX_FILES) {
+ return `${outDir} holds ${files} files, over Pages' limit of ${PAGES_MAX_FILES}`;
+ }
+ return null;
+}
+
+/**
+ * Why `outDir` may not be deployed as `site` because of what it PUBLISHES, as
+ * one sentence, or null: a site configured cited (`publish: "cited"`) whose
+ * build is not a cited one was built before the switch, and would ship the
+ * whole corpus. Asked beside the audience refusals (deployAudienceProblem).
+ */
+export function builtScopeProblem(
+ site: { siteId: string; publish?: string },
outDir: string,
): string | null {
- return siteDeployProblem(site) ?? builtAudienceProblem(outDir);
+ if (site.publish !== "cited" || !existsSync(path.join(outDir, "corpus.json"))) return null;
+ if (isCitedBuild(outDir)) return null;
+ return (
+ `Site "${site.siteId}" publishes only its reports (publish: cited), but ${outDir} ` +
+ `holds a full build — build ${site.siteId} again, then deploy`
+ );
}
// corpus.json's `site.id`, or null when there is no readable one.
@@ -200,7 +362,18 @@ export function builtHubProblem(outDir: string): string | null {
// The per-site data trees a hub bundle must never carry (the trees of
// compose-hub's SITE_ONLY_PUBLIC_ENTRIES).
-const HUB_FORBIDDEN_TREES = ["summaries", "transcripts", "subs", "posts", "digests", "stats", "archives"];
+const HUB_FORBIDDEN_TREES = [
+ "summaries",
+ "transcripts",
+ "subs",
+ "posts",
+ "digests",
+ "stats",
+ "archives",
+ "reports",
+ "m",
+ "media",
+];
/**
* Why `outDir` — the homepage package's `homepage/out` — may not be deployed as
diff --git a/common/publish/build.ts b/common/publish/build.ts
@@ -18,7 +18,9 @@ import {
builtAudienceProblem,
builtBundleProblem,
builtHubProblem,
+ builtScopeProblem,
builtSiteProblem,
+ citedBuildProblem,
deployAudienceProblem,
siteDeployProblem,
} from "../lib/builtExport";
@@ -81,6 +83,8 @@ export function buildSiteSteps(opts: {
paths: Paths;
skipData?: boolean;
skipArchives?: boolean;
+ // Let a report citation with no prepared media through compose.
+ allowMissingMedia?: boolean;
baseEnv?: NodeJS.ProcessEnv;
}): BuildStep[] {
const { paths } = opts;
@@ -94,6 +98,8 @@ export function buildSiteSteps(opts: {
// makes compose-site skip generation this build regardless of the
// global/site flags.
...(opts.skipArchives ? { BUILD_ARCHIVES: "0" } : {}),
+ // compose-site.ts ALLOW_MISSING_MEDIA_ENV (`--allow-missing-media`).
+ ...(opts.allowMissingMedia ? { REPORTS_ALLOW_MISSING_MEDIA: "1" } : {}),
};
const step = (args: string[]): BuildStep => ({
command: "pnpm",
@@ -134,7 +140,7 @@ export async function runBuildPhase(
signal: AbortSignal,
siteId: string,
paths: Paths,
- opts?: { skipData?: boolean; skipArchives?: boolean },
+ opts?: { skipData?: boolean; skipArchives?: boolean; allowMissingMedia?: boolean },
): Promise<number> {
// Skipping the data rebuild composes from the existing .export-index staging
// (see buildSiteSteps).
@@ -149,11 +155,29 @@ export async function runBuildPhase(
if (skipArchives) {
onLog("[notice] Skipping archive-zip generation for this build.\n");
}
- return runSteps(
+ const code = await runSteps(
onLog,
signal,
- buildSiteSteps({ siteId, paths, skipData, skipArchives }),
+ buildSiteSteps({
+ siteId,
+ paths,
+ skipData,
+ skipArchives,
+ allowMissingMedia: opts?.allowMissingMedia === true,
+ }),
);
+ if (code !== 0) return code;
+ // A CITED site's build must hold nothing but its reports, their moments and
+ // the shell (lib/builtExport.ts citedBuildProblem) — and must BE a cited
+ // build. Failing here is loud and early; every deploy path asks again.
+ const outDir = resolveOutDir(siteId, paths);
+ const scopeProblem =
+ citedBuildProblem(outDir) ?? builtScopeProblem(getSite(siteId, paths), outDir);
+ if (scopeProblem) {
+ onLog(`[build] REFUSED — ${scopeProblem}.\n`);
+ return 1;
+ }
+ return 0;
}
// Where the basic (host) compose staged this site's oversize archives for R2
@@ -750,12 +774,13 @@ function resolved(opts: PublishOpts): {
/** Build one site into export/out (basic/host mode). Returns the exit code. */
export async function buildSite(
siteId: string,
- opts: PublishOpts & { skipData?: boolean; skipArchives?: boolean } = {},
+ opts: PublishOpts & { skipData?: boolean; skipArchives?: boolean; allowMissingMedia?: boolean } = {},
): Promise<number> {
const { paths, onLog, signal } = resolved(opts);
return runBuildPhase(onLog, signal, siteId.trim(), paths, {
skipData: opts.skipData,
skipArchives: opts.skipArchives,
+ allowMissingMedia: opts.allowMissingMedia,
});
}
@@ -795,6 +820,9 @@ export async function deploySite(
// Before the R2 upload below: a bundle built private is never deployed.
const builtPrivate = builtAudienceProblem(outDir);
if (builtPrivate) throw new Error(`${builtPrivate}. Build ${site.siteId} again, then deploy.`);
+ // …nor a full build of a site that now publishes only its reports.
+ const builtScope = builtScopeProblem(site, outDir);
+ if (builtScope) throw new Error(`${builtScope}.`);
// The production path logs no banner and gains none here: its log has
// always opened on wrangler's own first line.
if (branch) {
