commit 1862ed472d8229e8fe23da582331aaad606351ee
parent 19c7b28885f2fd998510f3e53d0c7f9d2ab650d5
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 6 Oct 2026 08:20:01 -0400
common: withdrawn X posts ship tombstones, served no-store (site and hub)
While social.x.visibility is private, compose-site replaces each withheld
member's posts paths with tombstones (publish/tombstones.ts): the channel's
posts manifest at pageCount 0 and `[]` for every page the shared tree holds,
plus an empty posts/manifest.json when the index wrote none. compose-hub
writes the same for every X channel with a shared posts tree, with an empty
posts/manifest.json, after removing SITE_ONLY_PUBLIC_ENTRIES. renderHeadersFile
takes `noStore`: `Cache-Control: no-store` for /posts/manifest.json and
/posts/<slug>/* on a site, /posts/* on the hub — with CORS only where no CORS
rule already covers the path (a repeated header is appended). builtHubProblem
accepts a posts/ of tombstones only (isTombstonePostsTree).
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
10 files changed, 750 insertions(+), 11 deletions(-)
diff --git a/common/bin/compose-hub.test.ts b/common/bin/compose-hub.test.ts
@@ -16,6 +16,7 @@ import { tmpdir } from "node:os";
import path from "node:path";
import { getPaths, type Paths } from "../lib/paths";
import { main } from "./compose-hub";
+import { builtHubProblem } from "../lib/builtExport";
import { readGlobalAliases } from "../lib/aliasesStore";
// Run with:
@@ -41,6 +42,9 @@ function fixturePaths(root: string): Paths {
lmdbPath: path.join(root, "index.mdb"), // never created: no index
exportPublicDir,
exportIndexDir: path.join(root, ".export-index"),
+ // The hub's tombstones read the shared posts tree (release 18): this
+ // fixture's own, never the checkout's.
+ exportSharedPostsDir: path.join(root, ".export-index", "shared", "posts"),
};
}
@@ -263,3 +267,81 @@ test("compose-hub removes a site's data from public/, a linked entry by its link
rmSync(root, { recursive: true, force: true });
}
});
+
+// Release 18: Cloudflare's edge kept serving the hub's withdrawn X shard after
+// the deploy that removed it. While X posts are private the hub REPLACES those
+// paths — path for path, the rollout's requirement: posts/manifest.json (empty),
+// posts/thequartering-X/manifest.json (pageCount 0), posts/thequartering-X/
+// page-0000.json ([]) — and serves posts/ no-store. A Bluesky channel is not X
+// and gets nothing; with X public the hub ships no posts/ at all.
+test("while X posts are private, the hub ships tombstones for every X channel, served no-store", async () => {
+ const root = mkdtempSync(path.join(tmpdir(), "compose-hub-"));
+ const log = console.log;
+ try {
+ const paths = fixturePaths(root);
+ const pub = paths.exportPublicDir;
+ const writeJson = (file: string, value: unknown) => {
+ mkdirSync(path.dirname(file), { recursive: true });
+ writeFileSync(file, JSON.stringify(value));
+ };
+ const X = "thequartering-X";
+ const SKY = "jer-sky";
+ writeJson(path.join(paths.channelsDir, X, "config.json"), {
+ handling: "youtube",
+ url: "https://x.com/x",
+ sourceKind: "social",
+ platform: "twitter",
+ });
+ writeJson(path.join(paths.channelsDir, SKY, "config.json"), {
+ handling: "youtube",
+ url: "https://bsky.app/profile/jer.example",
+ sourceKind: "social",
+ platform: "bluesky",
+ });
+ for (const slug of [X, SKY]) {
+ writeJson(path.join(paths.exportSharedPostsDir, slug, "manifest.json"), {
+ version: 1,
+ channelSlug: slug,
+ pageCount: 1,
+ maxPageBytes: 1000,
+ generatedAt: "2026-10-01T00:00:00.000Z",
+ slugToPage: { "1": 0 },
+ });
+ writeJson(path.join(paths.exportSharedPostsDir, slug, "page-0000.json"), [{ id: "1", text: "a post" }]);
+ }
+
+ console.log = () => {};
+ await main({ paths, settings: { social: { x: { visibility: "private" } } } });
+ console.log = log;
+
+ const read = (rel: string) => JSON.parse(readFileSync(path.join(pub, rel), "utf8"));
+ assert.deepEqual(read("posts/manifest.json").channels, []);
+ assert.equal(read("posts/manifest.json").totalCount, 0);
+ assert.equal(read(`posts/${X}/manifest.json`).pageCount, 0);
+ assert.deepEqual(read(`posts/${X}/manifest.json`).slugToPage, {});
+ assert.deepEqual(read(`posts/${X}/page-0000.json`), []);
+ assert.ok(!existsSync(path.join(pub, "posts", SKY)), "a Bluesky channel is not withdrawn");
+ const headers = readFileSync(path.join(pub, "_headers"), "utf8");
+ assert.ok(
+ headers.endsWith(
+ "# Withdrawn content (tombstones): never stored at the edge.\n" +
+ "/posts/*\n Cache-Control: no-store\n Access-Control-Allow-Origin: *\n",
+ ),
+ headers,
+ );
+ // corpus.json advertises no posts; and the hub bundle (public/ stands in
+ // for out/) is still a hub: a posts/ of tombstones is its own.
+ assert.ok(!readFileSync(path.join(pub, "corpus.json"), "utf8").includes("posts"));
+ assert.equal(builtHubProblem(pub), null);
+
+ // X public again: no posts/ at all, no no-store block.
+ console.log = () => {};
+ await main({ paths, settings: {} });
+ console.log = log;
+ assert.ok(!existsSync(path.join(pub, "posts")));
+ assert.ok(!readFileSync(path.join(pub, "_headers"), "utf8").includes("no-store"));
+ } finally {
+ console.log = log;
+ rmSync(root, { recursive: true, force: true });
+ }
+});
diff --git a/common/bin/compose-hub.ts b/common/bin/compose-hub.ts
@@ -13,6 +13,11 @@
// public/_headers <- CORS for the hub's own served JSON
// public/sw.js <- the hub service worker (the hub always ships a PWA)
// public/search-aliases.json <- the global alias dictionary
+// public/posts/ <- TOMBSTONES only, while X posts are private:
+// an empty posts/manifest.json and, per X
+// channel with a shared posts tree, its
+// manifest at pageCount 0 and `[]` pages
+// (publish/tombstones.ts), served no-store
//
// and REMOVES every per-site entry a site's compose left in public/
// (SITE_ONLY_PUBLIC_ENTRIES below): the hub holds no site's data.
@@ -38,6 +43,14 @@ import { HUB_CORS_PATHS, renderHeadersFile } from "../lib/archive/headers";
import { buildPoolSummary } from "../controller/poolSummary";
import { HUB_SUMMARY_FILE, toHubSummary } from "../lib/hubSummary";
import { readGlobalAliases } from "../lib/aliasesStore";
+import { getSettings } from "../lib/settings";
+import {
+ emptyPostsManifest,
+ tombstoneNoStoreForHub,
+ withdrawnXChannels,
+ writePostsTombstones,
+ type PostsTombstone,
+} from "../publish/tombstones";
import { runIfEntryPoint } from "./_cli";
import { writePublicFile } from "./_publicFile";
@@ -110,13 +123,44 @@ export const SITE_ONLY_PUBLIC_ENTRIES: readonly string[] = [
"sitemap.xml",
];
-export async function main(opts: { paths?: Paths } = {}): Promise<void> {
+// THE HUB'S TOMBSTONES (release 18). A hub built over a site's compose once
+// shipped that site's posts (the review's HIGH 1 above), and Cloudflare's edge
+// kept serving them after the deploy that removed them. Removing is not enough
+// at the edge, so while X posts are private the hub REPLACES every path an X
+// channel's posts could have been served from: an empty posts manifest, and per
+// X channel with a shared posts tree its manifest at pageCount 0 and an empty
+// page for each page the shared tree holds now — all served no-store. With X
+// posts public, or no X channel, the hub ships no posts/ at all, as before.
+async function composeHubTombstones(
+ paths: Paths,
+ publicDir: string,
+ settings: { social?: { x?: { visibility?: unknown } } },
+): Promise<PostsTombstone[]> {
+ const slugs = await withdrawnXChannels(paths, settings);
+ if (slugs.length === 0) return [];
+ const postsDir = path.join(publicDir, "posts");
+ const tombstones = await writePostsTombstones({
+ postsDir,
+ sharedPostsDir: paths.exportSharedPostsDir,
+ slugs,
+ });
+ await writePublicFile(
+ path.join(postsDir, "manifest.json"),
+ JSON.stringify(emptyPostsManifest(new Date().toISOString())),
+ );
+ return tombstones;
+}
+
+export async function main(
+ opts: { paths?: Paths; settings?: { social?: { x?: { visibility?: unknown } } } } = {},
+): Promise<void> {
const paths = opts.paths ?? getPaths();
const publicDir = paths.exportPublicDir;
for (const entry of SITE_ONLY_PUBLIC_ENTRIES) {
await rm(path.join(publicDir, entry), { recursive: true, force: true });
}
+ const tombstones = await composeHubTombstones(paths, publicDir, opts.settings ?? getSettings());
// The hub's own alias dictionary is the global one (no site's overrides):
// what a hub reader loads for hub-wide search (lib/archive/reader-hub.ts),
// where it used to get whichever site had composed last.
@@ -181,7 +225,9 @@ export async function main(opts: { paths?: Paths } = {}): Promise<void> {
await writePublicFile(
path.join(publicDir, "_headers"),
- renderHeadersFile("compose-hub.ts", HUB_CORS_PATHS),
+ renderHeadersFile("compose-hub.ts", HUB_CORS_PATHS, {
+ noStore: tombstoneNoStoreForHub(tombstones),
+ }),
);
// The hub always ships a PWA. Copy the hub service worker into place. Until
@@ -196,8 +242,12 @@ export async function main(opts: { paths?: Paths } = {}): Promise<void> {
const summaryNote = await composeHubSummary(paths, publicDir);
+ const tombstoneNote =
+ tombstones.length > 0
+ ? `; ${tombstones.length} withdrawn X channel(s) shipped as tombstones, served no-store`
+ : "";
console.log(
- `compose-hub: ${builtins.length} built-in pool site(s) into ${publicDir}; ${summaryNote}.`,
+ `compose-hub: ${builtins.length} built-in pool site(s) into ${publicDir}; ${summaryNote}${tombstoneNote}.`,
);
}
diff --git a/common/bin/compose-site.postsVisibility.test.ts b/common/bin/compose-site.postsVisibility.test.ts
@@ -9,6 +9,12 @@
// and says `"audience": "private"` with no hubUrl. Flipping the setting back
// is a rebuild. The rule itself is lib/postsVisibility.ts (its own tests).
//
+// Release 18: the public site's withheld X channel is not merely left out — its
+// posts paths ship TOMBSTONES (publish/tombstones.ts): posts/<x>/manifest.json
+// at pageCount 0 and `[]` for every page the shared tree holds, served no-store
+// by the site's _headers. A tombstone is not a posts tree: `postTrees` below
+// lists real trees only, `tombstones` the rest.
+//
// The export e2e cannot show this: its data is route-mocked, never built by
// buildIndex and compose. export/e2e/x-posts-private.spec.ts serves the two
// posts manifests this file pins and checks what a visitor sees.
@@ -17,7 +23,15 @@
import { after, test } from "node:test";
import assert from "node:assert/strict";
-import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import {
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ readdirSync,
+ rmSync,
+ writeFileSync,
+} from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
@@ -160,7 +174,10 @@ async function index() {
// What one site's compose put in public/.
type Composed = {
postsManifest: { channels: { slug: string; platform: string; postCount: number }[]; totalCount: number };
+ // Real posts trees (a channel manifest with pages), and tombstones (pageCount 0).
postTrees: string[];
+ tombstones: string[];
+ headers: string;
transcriptTrees: string[];
siteJson: { channels: { slug: string }[]; hubUrl?: string };
corpus: {
@@ -181,9 +198,14 @@ async function compose(siteId: string): Promise<Composed> {
const pub = paths.exportPublicDir;
const trees = (dir: string) =>
[VIDEOS, X, SKY].filter((slug) => existsSync(path.join(dir, slug)));
+ const isTombstone = (slug: string) =>
+ readJson<{ pageCount: number }>(path.join(pub, "posts", slug, "manifest.json")).pageCount === 0;
+ const posts = trees(path.join(pub, "posts"));
return {
postsManifest: readJson(path.join(pub, "posts", "manifest.json")),
- postTrees: trees(path.join(pub, "posts")),
+ postTrees: posts.filter((slug) => !isTombstone(slug)),
+ tombstones: posts.filter(isTombstone),
+ headers: readFileSync(path.join(pub, "_headers"), "utf8"),
transcriptTrees: trees(path.join(pub, "transcripts")),
siteJson: readJson(path.join(pub, "site.json")),
corpus: readJson(path.join(pub, "corpus.json")),
@@ -212,6 +234,32 @@ test("X private: a public site carries no X channel; a private site carries all
assert.deepEqual(slugs(pub.postsManifest.channels), [SKY]);
assert.equal(pub.postsManifest.totalCount, 1);
assert.deepEqual(pub.postTrees, [SKY]);
+ // The withheld X channel's paths ship tombstones (release 18), path for path:
+ // its manifest at pageCount 0 and an empty page for the shared tree's one page.
+ assert.deepEqual(pub.tombstones, [X]);
+ const xDir = path.join(paths.exportPublicDir, "posts", X);
+ const sharedPages = readJson<{ pageCount: number }>(
+ path.join(paths.exportSharedPostsDir, X, "manifest.json"),
+ ).pageCount;
+ assert.equal(sharedPages, 1);
+ assert.deepEqual(readdirSync(xDir).sort(), ["manifest.json", "page-0000.json"]);
+ assert.deepEqual(readJson(path.join(xDir, "page-0000.json")), []);
+ const tomb = readJson<{ channelSlug: string; pageCount: number; slugToPage: object }>(
+ path.join(xDir, "manifest.json"),
+ );
+ assert.equal(tomb.channelSlug, X);
+ assert.equal(tomb.pageCount, 0);
+ assert.deepEqual(tomb.slugToPage, {});
+ assert.ok(!readFileSync(path.join(xDir, "page-0000.json"), "utf8").includes("x post"));
+ // …served no-store, after the CORS lines, with no second CORS header.
+ assert.ok(
+ pub.headers.endsWith(
+ "# Withdrawn content (tombstones): never stored at the edge.\n" +
+ "/posts/manifest.json\n Cache-Control: no-store\n" +
+ `/posts/${X}/*\n Cache-Control: no-store\n`,
+ ),
+ pub.headers,
+ );
assert.deepEqual(pub.transcriptTrees, [VIDEOS, SKY]);
assert.deepEqual(slugs(pub.siteJson.channels), [SKY, VIDEOS]);
assert.deepEqual(slugs(pub.corpus.channels), [SKY, VIDEOS]);
@@ -225,7 +273,11 @@ test("X private: a public site carries no X channel; a private site carries all
const priv = await compose("priv");
assert.deepEqual(slugs(priv.postsManifest.channels), [SKY, X]);
assert.equal(priv.postsManifest.totalCount, 3);
+ // The private site carries the real tree where the public one had the
+ // tombstone, and withholds nothing: no tombstone, no no-store block.
assert.deepEqual(priv.postTrees, [X, SKY]);
+ assert.deepEqual(priv.tombstones, []);
+ assert.ok(!priv.headers.includes("no-store"));
assert.deepEqual(slugs(priv.corpus.channels), [SKY, X, VIDEOS]);
assert.equal(priv.corpus.channels.find((c) => c.slug === X)?.postCount, 2);
assert.ok(priv.corpus.channels.find((c) => c.slug === X)?.manifests.posts);
@@ -249,6 +301,9 @@ test("X private: a public site carries no X channel; a private site carries all
// private site's channel list).
const again = await compose("pub");
assert.deepEqual(again.postTrees, [SKY]);
+ // The private site's real X tree is REPLACED by the tombstone, never left.
+ assert.deepEqual(again.tombstones, [X]);
+ assert.deepEqual(readJson(path.join(paths.exportPublicDir, "posts", X, "page-0000.json")), []);
assert.deepEqual(again.transcriptTrees, [VIDEOS, SKY]);
assert.deepEqual(slugs(again.siteJson.channels), [SKY, VIDEOS]);
assert.deepEqual(slugs(again.corpus.channels), [SKY, VIDEOS]);
@@ -272,6 +327,9 @@ test("a config compose cannot read does not ship the posts tree the index build
assert.deepEqual(pub.postTrees, [SKY]);
assert.deepEqual(slugs(pub.postsManifest.channels), [SKY]);
assert.deepEqual(slugs(pub.corpus.channels), [SKY, VIDEOS]);
+ // compose reads X as visible here (no config): no tombstone for it either,
+ // and no no-store block — the index build's word kept the tree out.
+ assert.deepEqual(pub.tombstones, []);
} finally {
writeFileSync(cfg, saved);
}
@@ -284,6 +342,7 @@ test("a compose over an index built before the setting flipped lists no X channe
writeSettings("private");
const pub = await compose("pub");
assert.deepEqual(pub.postTrees, [SKY]);
+ assert.deepEqual(pub.tombstones, [X]);
assert.deepEqual(slugs(pub.postsManifest.channels), [SKY]);
assert.equal(pub.postsManifest.totalCount, 1);
assert.equal(pub.corpus.channels.find((c) => c.slug === X)?.postCount, undefined);
@@ -298,7 +357,10 @@ test("X public again: the next build puts X back on the public site", async () =
await index();
const pub = await compose("pub");
assert.deepEqual(slugs(pub.postsManifest.channels), [SKY, X]);
+ // The tombstone is replaced by the real tree, and the no-store block goes.
assert.deepEqual(pub.postTrees, [X, SKY]);
+ assert.deepEqual(pub.tombstones, []);
+ assert.ok(!pub.headers.includes("no-store"));
assert.deepEqual(slugs(pub.corpus.channels), [SKY, X, VIDEOS]);
// No setting at all is public too.
@@ -326,6 +388,7 @@ test("a public site whose only posts were X posts ships an empty posts manifest
// toggle on this site, with nothing special-cased.
assert.deepEqual(xonly.postsManifest.channels, []);
assert.deepEqual(xonly.postTrees, []);
+ assert.deepEqual(xonly.tombstones, [X]);
assert.equal(xonly.corpus.postScheme, undefined);
assert.deepEqual(slugs(xonly.corpus.channels), [VIDEOS]);
});
diff --git a/common/bin/compose-site.ts b/common/bin/compose-site.ts
@@ -41,7 +41,7 @@ import type { PostsManifest } from "../lib/posts";
import type { DigestsManifest } from "../lib/digests";
import { buildSiteDescriptor, type PublicSiteDescriptor } from "../lib/siteDescriptor";
import { shipsPwa } from "../lib/archive/contract";
-import { renderHeadersFile } from "../lib/archive/headers";
+import { SITE_CORS_PATHS, renderHeadersFile } from "../lib/archive/headers";
import { effectiveSiteAliases } from "../lib/aliasesStore";
import { effectiveSiteTags } from "../lib/curatedTagsStore";
import { TAGS_FILENAME } from "../lib/curatedTags";
@@ -78,6 +78,11 @@ import {
reportRoutes,
type ComposedReports,
} from "../publish/composeReports";
+import {
+ emptyPostsManifest,
+ tombstoneNoStoreForSite,
+ writePostsTombstones,
+} from "../publish/tombstones";
import { runIfEntryPoint } from "./_cli";
import { copyPublicFile, ownDir, writePublicFile } from "./_publicFile";
@@ -93,13 +98,17 @@ import { copyPublicFile, ownDir, writePublicFile } from "./_publicFile";
// Exported, with emitAiFiles, for the cited fixture site's e2e staging
// (export/e2e-report/stage.ts), which writes a cited site's contract around
// fixture report views exactly as this compose would.
+//
+// `noStore` names the paths _headers serves uncached: the tombstones of X
+// channels this site withheld (publish/tombstones.ts).
export async function emitFederationFiles(
site: Site,
paths: ReturnType<typeof getPaths>,
+ opts: { noStore?: readonly string[] } = {},
): Promise<void> {
await writePublicFile(
path.join(paths.exportPublicDir, "_headers"),
- renderHeadersFile("compose-site.ts"),
+ renderHeadersFile("compose-site.ts", SITE_CORS_PATHS, { noStore: opts.noStore }),
);
// A cited site has no summaries: its descriptor names no channel, and it is
@@ -995,6 +1004,33 @@ export async function main(
}),
);
}
+ // --- tombstones for the X channels this site withheld (release 18) ---
+ // A withheld channel's posts were served from this site's paths before (or
+ // may still be cached at the edge from a build when X was public): every
+ // such path is REPLACED with an empty object of the same shape and served
+ // no-store, never left out (publish/tombstones.ts). The site posts manifest
+ // above already lists no withheld channel; a site whose only posts were X
+ // posts, with no manifest from the index, still ships an empty one.
+ const memberSet = new Set(memberSlugs);
+ const tombstones = await writePostsTombstones({
+ postsDir: paths.exportPostsDir,
+ sharedPostsDir: paths.exportSharedPostsDir,
+ slugs: site.channels.map((c) => c.slug).filter((slug) => !memberSet.has(slug)),
+ });
+ if (tombstones.length > 0) {
+ const postsManifest = path.join(paths.exportPostsDir, "manifest.json");
+ if (!(await exists(postsManifest))) {
+ await writePublicFile(
+ postsManifest,
+ JSON.stringify(emptyPostsManifest(new Date().toISOString(), siteId)),
+ );
+ }
+ console.log(
+ `[compose] posts: ${tombstones.length} withheld X channel(s) shipped as tombstones ` +
+ `(${tombstones.reduce((n, t) => n + t.pages, 0)} empty page(s)), served no-store.`,
+ );
+ }
+
// Same for the per-site digests manifest (which channels carry digests).
const digestsManifestSrc = path.join(
paths.exportSitesIndexDir,
@@ -1163,7 +1199,7 @@ export async function main(
const composed = await composeReports({ paths, site, allowMissingMedia, log: console.log });
// --- federation contract: /site.json descriptor + CORS _headers ---
- await emitFederationFiles(site, paths);
+ await emitFederationFiles(site, paths, { noStore: tombstoneNoStoreForSite(tombstones) });
// A site's bundle is not a hub's. export/public is shared with the hub build,
// whose compose writes hub-sites.json; left in place it ships in this site's
// out/ and makes the bundle ambiguous to builtHubProblem (and to a site's
diff --git a/common/lib/archive/headers.test.ts b/common/lib/archive/headers.test.ts
@@ -77,6 +77,63 @@ test("renderHeadersFile: the hub block, in full", () => {
assert.equal(renderHeadersFile("compose-hub.ts", HUB_CORS_PATHS), HUB_HEADERS);
});
+// The tombstone blocks (release 18): withdrawn X posts are served no-store. A
+// site's /posts/* CORS rule already covers them, so its no-store rules carry no
+// second CORS header (a repeated header is APPENDED: "*, *"); the hub lists no
+// posts tree, so its /posts/* rule carries both.
+const SITE_TOMBSTONE_BLOCK = `# Withdrawn content (tombstones): never stored at the edge.
+/posts/manifest.json
+ Cache-Control: no-store
+/posts/jer-x/*
+ Cache-Control: no-store
+`;
+
+const HUB_TOMBSTONE_BLOCK = `# Withdrawn content (tombstones): never stored at the edge.
+/posts/*
+ Cache-Control: no-store
+ Access-Control-Allow-Origin: *
+`;
+
+test("renderHeadersFile: a site's no-store block follows its CORS lines, with no second CORS header", () => {
+ assert.equal(
+ renderHeadersFile("compose-site.ts", SITE_CORS_PATHS, {
+ noStore: ["/posts/manifest.json", "/posts/jer-x/*"],
+ }),
+ SITE_HEADERS + SITE_TOMBSTONE_BLOCK,
+ );
+ // No paths, no block: the file is byte-identical to the one without opts.
+ assert.equal(renderHeadersFile("compose-site.ts", SITE_CORS_PATHS, { noStore: [] }), SITE_HEADERS);
+});
+
+test("renderHeadersFile: the hub's /posts/* no-store rule carries its own CORS", () => {
+ assert.equal(
+ renderHeadersFile("compose-hub.ts", HUB_CORS_PATHS, { noStore: ["/posts/*"] }),
+ HUB_HEADERS + HUB_TOMBSTONE_BLOCK,
+ );
+});
+
+test("no rendered file sets a header twice for one path", () => {
+ // Every pair of rules that both match a path must not both set the same
+ // header: wrangler appends the second value.
+ const files = [
+ renderHeadersFile("s", SITE_CORS_PATHS, { noStore: ["/posts/manifest.json", "/posts/a/*"] }),
+ renderHeadersFile("h", HUB_CORS_PATHS, { noStore: ["/posts/*"] }),
+ ];
+ for (const file of files) {
+ const rules: { path: string; headers: string[] }[] = [];
+ for (const line of file.split("\n")) {
+ if (line.startsWith("/")) rules.push({ path: line, headers: [] });
+ else if (line.startsWith(" ")) rules[rules.length - 1].headers.push(line.trim().split(":")[0]);
+ }
+ const covers = (rule: string, p: string) =>
+ rule.endsWith("*") ? p.startsWith(rule.slice(0, -1)) : rule === p;
+ for (const probe of ["/posts/manifest.json", "/posts/a/page-0000.json", "/corpus.json"]) {
+ const set = rules.filter((r) => covers(r.path, probe)).flatMap((r) => r.headers);
+ assert.equal(new Set(set).size, set.length, `${probe}: ${set.join(", ")}`);
+ }
+ }
+});
+
test("the two paths that used to be served without CORS are declared", () => {
// The wire change. A cross-origin viewer could read every other tree and got
// a CORS failure on exactly these two.
diff --git a/common/lib/archive/headers.ts b/common/lib/archive/headers.ts
@@ -71,17 +71,49 @@ export const HUB_CORS_PATHS: readonly string[] = [
"/robots.txt",
];
+// What a WITHDRAWN path is served with (release 18): never stored at the edge.
+// Cloudflare Pages keeps serving a cached object after a deploy that changed or
+// removed it (the hub served a withdrawn posts shard from a 7-day edge cache), so
+// the tombstones that replace withdrawn content — and the manifests that list it
+// — are marked uncacheable. publish/tombstones.ts names the paths.
+const NO_STORE_HEADER = "Cache-Control: no-store";
+
+// Whether a `_headers` path rule (a literal path, or one ending in a `*` splat)
+// matches `p`, itself a literal path or a splat rule. A splat rule covers every
+// path under its prefix.
+function ruleCovers(rule: string, p: string): boolean {
+ if (rule.endsWith("*")) return p.startsWith(rule.slice(0, -1));
+ return rule === p;
+}
+
// Render a `_headers` file: a banner naming the generator, then one path /
// indented-header pair per served surface. `generator` is the script name so a
// reader of a deploy artifact knows what to edit instead of the file.
+//
+// `noStore` adds, after the CORS lines, one rule per path marked
+// `Cache-Control: no-store`. It carries the CORS header too — but only where no
+// CORS rule above already covers the path: every matching rule applies, and a
+// header a later rule sets again is APPENDED (wrangler's attachHeaders), so a
+// second CORS line would serve `Access-Control-Allow-Origin: *, *`, which no
+// browser accepts. A site's `/posts/*` CORS rule covers its posts tombstones; the
+// hub, which lists no posts tree, gets its CORS from the no-store rule itself.
export function renderHeadersFile(
generator: string,
paths: readonly string[] = SITE_CORS_PATHS,
+ opts: { noStore?: readonly string[] } = {},
): string {
const out = [`# Generated by ${generator} — do not edit by hand.`];
for (const p of paths) {
out.push(p, ` ${CORS_HEADER}`);
}
+ const noStore = opts.noStore ?? [];
+ if (noStore.length > 0) {
+ out.push("# Withdrawn content (tombstones): never stored at the edge.");
+ for (const p of noStore) {
+ out.push(p, ` ${NO_STORE_HEADER}`);
+ if (!paths.some((rule) => ruleCovers(rule, p))) out.push(` ${CORS_HEADER}`);
+ }
+ }
return `${out.join("\n")}\n`;
}
diff --git a/common/lib/builtExport.test.ts b/common/lib/builtExport.test.ts
@@ -18,6 +18,7 @@ import {
PUBLISH_MAX_FILE_BYTES,
publishFileSizeProblem,
reportHistoryProblem,
+ isTombstonePostsTree,
} from "./builtExport";
function tempOut(siteJson?: string): { dir: string; cleanup: () => void } {
@@ -125,6 +126,28 @@ test("builtHubProblem accepts only a hub bundle", () => {
"export/out holds no hub build — build the hub first",
);
+ // Release 18: a posts/ of TOMBSTONES only is the hub's own (compose-hub
+ // writes them for withdrawn X posts) — and one real post in it is not.
+ mkdirSync(path.join(hub.dir, "posts", "jer-x"), { recursive: true });
+ writeFileSync(path.join(hub.dir, "posts", "manifest.json"), '{"channels":[],"totalCount":0}');
+ writeFileSync(
+ path.join(hub.dir, "posts", "jer-x", "manifest.json"),
+ '{"channelSlug":"jer-x","pageCount":0,"slugToPage":{}}',
+ );
+ writeFileSync(path.join(hub.dir, "posts", "jer-x", "page-0000.json"), "[]");
+ assert.equal(isTombstonePostsTree(path.join(hub.dir, "posts")), true);
+ assert.equal(builtHubProblem(hub.dir), null);
+ writeFileSync(path.join(hub.dir, "posts", "jer-x", "page-0000.json"), '[{"id":"1"}]');
+ assert.equal(isTombstonePostsTree(path.join(hub.dir, "posts")), false);
+ assert.equal(
+ builtHubProblem(hub.dir),
+ "export/out holds a hub build that still carries a site's data (posts) — build the hub again",
+ );
+ writeFileSync(path.join(hub.dir, "posts", "jer-x", "page-0000.json"), "[]");
+ writeFileSync(path.join(hub.dir, "posts", "manifest.json"), '{"channels":[{"slug":"jer-x"}]}');
+ assert.equal(isTombstonePostsTree(path.join(hub.dir, "posts")), false);
+ rmSync(path.join(hub.dir, "posts"), { recursive: true });
+
// Release 17 XP: a hub bundle composed over a site's data is refused.
mkdirSync(path.join(hub.dir, "posts", "jer-x"), { recursive: true });
mkdirSync(path.join(hub.dir, "summaries"));
diff --git a/common/lib/builtExport.ts b/common/lib/builtExport.ts
@@ -421,8 +421,14 @@ export function builtHubProblem(outDir: string): string | null {
}
// The hub holds no site's data (compose-hub removes it): a hub bundle that
// still carries a site's data trees was composed over one, and could ship
- // that site's posts — a private site's included.
- const carried = HUB_FORBIDDEN_TREES.filter((tree) => existsSync(path.join(outDir, tree)));
+ // that site's posts — a private site's included. Its one posts tree is the
+ // tombstones compose-hub writes for withdrawn X posts (release 18), and a
+ // tree holding anything but tombstones is a site's.
+ const carried = HUB_FORBIDDEN_TREES.filter(
+ (tree) =>
+ existsSync(path.join(outDir, tree)) &&
+ !(tree === "posts" && isTombstonePostsTree(path.join(outDir, tree))),
+ );
if (carried.length > 0) {
return (
`export/out holds a hub build that still carries a site's data (${carried.join(", ")}) — ` +
@@ -432,8 +438,58 @@ export function builtHubProblem(outDir: string): string | null {
return null;
}
+/**
+ * Whether `postsDir` (a bundle's `posts/`) holds TOMBSTONES and nothing else
+ * (publish/tombstones.ts): a site posts manifest listing no channel, and per
+ * channel dir a posts manifest at pageCount 0 and only `[]` pages. Anything
+ * else — a post, a listed channel, an unreadable file, a stray entry — is not.
+ */
+export function isTombstonePostsTree(postsDir: string): boolean {
+ const readJson = (file: string): unknown => {
+ try {
+ return JSON.parse(readFileSync(file, "utf8"));
+ } catch {
+ return undefined;
+ }
+ };
+ const manifest = readJson(path.join(postsDir, "manifest.json")) as
+ | { channels?: unknown }
+ | undefined;
+ if (!manifest || !Array.isArray(manifest.channels) || manifest.channels.length > 0) return false;
+ let entries: Dirent[];
+ try {
+ entries = readdirSync(postsDir, { withFileTypes: true });
+ } catch {
+ return false;
+ }
+ for (const e of entries) {
+ if (e.name === "manifest.json" && e.isFile()) continue;
+ if (!e.isDirectory()) return false;
+ const dir = path.join(postsDir, e.name);
+ const channel = readJson(path.join(dir, "manifest.json")) as
+ | { pageCount?: unknown; slugToPage?: unknown }
+ | undefined;
+ if (!channel || channel.pageCount !== 0) return false;
+ if (channel.slugToPage && Object.keys(channel.slugToPage as object).length > 0) return false;
+ let files: Dirent[];
+ try {
+ files = readdirSync(dir, { withFileTypes: true });
+ } catch {
+ return false;
+ }
+ for (const f of files) {
+ if (f.name === "manifest.json") continue;
+ if (!f.isFile() || !/^page-\d+\.json$/.test(f.name)) return false;
+ const page = readJson(path.join(dir, f.name));
+ if (!Array.isArray(page) || page.length > 0) return false;
+ }
+ }
+ return true;
+}
+
// The per-site data trees a hub bundle must never carry (the trees of
-// compose-hub's SITE_ONLY_PUBLIC_ENTRIES).
+// compose-hub's SITE_ONLY_PUBLIC_ENTRIES). A `posts/` of tombstones only is
+// the hub's own (isTombstonePostsTree).
const HUB_FORBIDDEN_TREES = [
"summaries",
"transcripts",
diff --git a/common/publish/tombstones.test.ts b/common/publish/tombstones.test.ts
@@ -0,0 +1,163 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ rmSync,
+ symlinkSync,
+ writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import {
+ emptyPostsManifest,
+ tombstoneChannelManifest,
+ tombstoneNoStoreForHub,
+ tombstoneNoStoreForSite,
+ tombstonePaths,
+ withdrawnXChannels,
+ writePostsTombstones,
+} from "./tombstones";
+
+// Run with: pnpm --filter yt-dlp-transcript-common test
+
+const writeJson = (file: string, value: unknown) => {
+ mkdirSync(path.dirname(file), { recursive: true });
+ writeFileSync(file, JSON.stringify(value));
+};
+const readJson = (file: string) => JSON.parse(readFileSync(file, "utf8"));
+
+function sharedTree(shared: string, slug: string, pages: number) {
+ writeJson(path.join(shared, slug, "manifest.json"), {
+ version: 1,
+ channelSlug: slug,
+ pageCount: pages,
+ maxPageBytes: 4096,
+ generatedAt: "2026-10-01T00:00:00.000Z",
+ slugToPage: { a: 0 },
+ });
+ for (let i = 0; i < pages; i++) {
+ writeJson(path.join(shared, slug, `page-${String(i).padStart(4, "0")}.json`), [{ id: `p${i}` }]);
+ }
+}
+
+test("the tombstone shapes: a channel manifest with no pages, a site manifest with no channels", () => {
+ assert.deepEqual(tombstoneChannelManifest("x", "T", 9), {
+ version: 1,
+ channelSlug: "x",
+ pageCount: 0,
+ maxPageBytes: 9,
+ generatedAt: "T",
+ slugToPage: {},
+ });
+ assert.deepEqual(emptyPostsManifest("T", "pub"), {
+ version: 1,
+ channels: [],
+ totalCount: 0,
+ generatedAt: "T",
+ siteId: "pub",
+ });
+ assert.equal("siteId" in emptyPostsManifest("T"), false);
+});
+
+test("writePostsTombstones: one empty page per shared page, replacing a real tree, never through a link", async () => {
+ const root = mkdtempSync(path.join(tmpdir(), "tombstones-"));
+ try {
+ const shared = path.join(root, "shared", "posts");
+ sharedTree(shared, "big-x", 3);
+ // A real tree an earlier public build shipped: replaced.
+ const posts = path.join(root, "public", "posts");
+ writeJson(path.join(posts, "big-x", "page-0000.json"), [{ id: "p0", text: "a post" }]);
+ writeJson(path.join(posts, "big-x", "page-0007.json"), [{ id: "p7" }]);
+ // A linked tree (a worktree's public/ entries link into the primary's):
+ // the link goes, its target is untouched.
+ const primary = path.join(root, "primary", "posts", "linked-x");
+ writeJson(path.join(primary, "page-0000.json"), [{ id: "keep" }]);
+ symlinkSync(primary, path.join(posts, "linked-x"));
+
+ const written = await writePostsTombstones({
+ postsDir: posts,
+ sharedPostsDir: shared,
+ slugs: ["big-x", "linked-x"],
+ generatedAt: "T",
+ });
+ assert.deepEqual(written, [
+ { slug: "big-x", pages: 3 },
+ { slug: "linked-x", pages: 0 },
+ ]);
+ assert.deepEqual(readdirSync(path.join(posts, "big-x")).sort(), [
+ "manifest.json",
+ "page-0000.json",
+ "page-0001.json",
+ "page-0002.json",
+ ]);
+ for (const page of ["page-0000.json", "page-0001.json", "page-0002.json"]) {
+ assert.deepEqual(readJson(path.join(posts, "big-x", page)), []);
+ }
+ assert.deepEqual(readJson(path.join(posts, "big-x", "manifest.json")), tombstoneChannelManifest("big-x", "T", 4096));
+ assert.deepEqual(readdirSync(path.join(posts, "linked-x")), ["manifest.json"]);
+ assert.deepEqual(readJson(path.join(primary, "page-0000.json")), [{ id: "keep" }]);
+
+ assert.deepEqual(tombstonePaths(written, { withManifest: true }), [
+ "posts/manifest.json",
+ "posts/big-x/manifest.json",
+ "posts/big-x/page-0000.json",
+ "posts/big-x/page-0001.json",
+ "posts/big-x/page-0002.json",
+ "posts/linked-x/manifest.json",
+ ]);
+ assert.deepEqual(tombstonePaths([]), []);
+ // Nothing to withdraw writes nothing — not even posts/.
+ const none = path.join(root, "none", "posts");
+ assert.deepEqual(await writePostsTombstones({ postsDir: none, sharedPostsDir: shared, slugs: [] }), []);
+ assert.equal(existsSync(none), false);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("the no-store paths: a site names its manifest and each tombstone; the hub its whole posts tree", () => {
+ const t = [
+ { slug: "a-x", pages: 1 },
+ { slug: "b-x", pages: 0 },
+ ];
+ assert.deepEqual(tombstoneNoStoreForSite(t), ["/posts/manifest.json", "/posts/a-x/*", "/posts/b-x/*"]);
+ assert.deepEqual(tombstoneNoStoreForHub(t), ["/posts/*"]);
+ assert.deepEqual(tombstoneNoStoreForSite([]), []);
+ assert.deepEqual(tombstoneNoStoreForHub([]), []);
+});
+
+test("withdrawnXChannels: the X channels with a shared posts tree, only while X posts are private", async () => {
+ const root = mkdtempSync(path.join(tmpdir(), "tombstones-"));
+ try {
+ const paths = {
+ channelsDir: path.join(root, "channels"),
+ exportSharedPostsDir: path.join(root, "shared", "posts"),
+ } as Paths;
+ const config = (slug: string, platform: string) =>
+ writeJson(path.join(paths.channelsDir, slug, "config.json"), {
+ handling: "youtube",
+ url: `https://example.test/${slug}`,
+ sourceKind: "social",
+ platform,
+ });
+ config("z-x", "twitter");
+ config("a-x", "twitter");
+ config("sky", "bluesky");
+ config("no-tree-x", "twitter");
+ for (const slug of ["z-x", "a-x", "sky", "no-config"]) sharedTree(paths.exportSharedPostsDir, slug, 1);
+
+ const priv = { social: { x: { visibility: "private" } } };
+ assert.deepEqual(await withdrawnXChannels(paths, priv), ["a-x", "z-x"]);
+ assert.deepEqual(await withdrawnXChannels(paths, {}), []);
+ assert.deepEqual(await withdrawnXChannels(paths, { social: { x: { visibility: "public" } } }), []);
+ const empty = { ...paths, exportSharedPostsDir: path.join(root, "absent") } as Paths;
+ assert.deepEqual(await withdrawnXChannels(empty, priv), []);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
diff --git a/common/publish/tombstones.ts b/common/publish/tombstones.ts
@@ -0,0 +1,177 @@
+// TOMBSTONES FOR WITHDRAWN X POSTS (release 18).
+//
+// While `social.x.visibility` is "private", a public site leaves every X channel
+// out whole (lib/postsVisibility.ts) and the hub carries no posts at all
+// (compose-hub's SITE_ONLY_PUBLIC_ENTRIES). Leaving a path OUT of a deploy does
+// not take it off Cloudflare's edge: Pages keeps serving a cached object until
+// its TTL runs out, whatever the new deployment holds (the hub served a
+// withdrawn X shard from a 7-day cache after the deploy that removed it). So
+// withdrawn content is REPLACED, never deleted: at every path it was served
+// from, the deploy ships an empty object of the same shape —
+//
+// posts/manifest.json a site posts manifest listing no channel
+// (only when the site lists none at all)
+// posts/<slug>/manifest.json the channel's posts manifest, pageCount 0
+// posts/<slug>/page-NNNN.json `[]`, for every N below the pageCount the
+// shared posts tree has now
+//
+// — and lib/archive/headers.ts serves those paths `Cache-Control: no-store`
+// (the paths are `tombstoneNoStore*` below). Nothing reads a tombstone: the
+// site's posts manifest does not list the channel, so no reader asks for its
+// tree, and corpus.json advertises no posts for it. A visitor holding a stale
+// link gets an empty page instead of the post.
+//
+// The writers go through bin/_publicFile.ts: in a worktree, public/'s entries
+// are links into the primary checkout, and nothing here writes through one.
+
+import path from "node:path";
+import { readdir, readFile, rm } from "node:fs/promises";
+import type { Paths } from "../lib/paths";
+import {
+ POSTS_MANIFEST_VERSION,
+ SITE_POSTS_MANIFEST_VERSION,
+ postsPageFileName,
+ type ChannelPostsManifest,
+ type PostsManifest,
+} from "../lib/posts";
+import { isXPostsChannel, xPostsVisibility } from "../lib/postsVisibility";
+import { readChannelConfig } from "../controller/channels";
+import { ownDir, writePublicFile } from "../bin/_publicFile";
+
+// One channel's tombstone: its slug and how many empty pages stand in for it.
+export type PostsTombstone = { slug: string; pages: number };
+
+// The channel posts manifest a tombstone ships: no pages, no posts.
+export function tombstoneChannelManifest(
+ slug: string,
+ generatedAt: string,
+ maxPageBytes = 0,
+): ChannelPostsManifest {
+ return {
+ version: POSTS_MANIFEST_VERSION,
+ channelSlug: slug,
+ pageCount: 0,
+ maxPageBytes,
+ generatedAt,
+ slugToPage: {},
+ };
+}
+
+// The site posts manifest of a site that lists no posts channel.
+export function emptyPostsManifest(generatedAt: string, siteId?: string): PostsManifest {
+ return {
+ version: SITE_POSTS_MANIFEST_VERSION,
+ channels: [],
+ totalCount: 0,
+ generatedAt,
+ ...(siteId ? { siteId } : {}),
+ };
+}
+
+// The pageCount (and page cap) of a channel's SHARED posts tree, or zeros when
+// it has none or it cannot be read.
+async function sharedPostsShape(
+ sharedPostsDir: string,
+ slug: string,
+): Promise<{ pageCount: number; maxPageBytes: number }> {
+ try {
+ const m = JSON.parse(
+ await readFile(path.join(sharedPostsDir, slug, "manifest.json"), "utf8"),
+ ) as Partial<ChannelPostsManifest>;
+ const pageCount = Number.isInteger(m.pageCount) && (m.pageCount as number) > 0 ? (m.pageCount as number) : 0;
+ const maxPageBytes = typeof m.maxPageBytes === "number" ? m.maxPageBytes : 0;
+ return { pageCount, maxPageBytes };
+ } catch {
+ return { pageCount: 0, maxPageBytes: 0 };
+ }
+}
+
+/**
+ * Write a tombstone tree for each of `slugs` under `postsDir` (a served
+ * `posts/`): the channel dir is replaced by its manifest at pageCount 0 and one
+ * `[]` page for every page the shared tree holds now. Returns what was written,
+ * in `slugs` order.
+ */
+export async function writePostsTombstones(opts: {
+ postsDir: string;
+ sharedPostsDir: string;
+ slugs: readonly string[];
+ generatedAt?: string;
+}): Promise<PostsTombstone[]> {
+ const generatedAt = opts.generatedAt ?? new Date().toISOString();
+ const written: PostsTombstone[] = [];
+ if (opts.slugs.length === 0) return written;
+ await ownDir(opts.postsDir);
+ for (const slug of opts.slugs) {
+ const dir = path.join(opts.postsDir, slug);
+ // Whatever was there (a real tree a public build shipped before, or a
+ // linked one in a worktree) goes; rm removes a link, never its target.
+ await rm(dir, { recursive: true, force: true });
+ await ownDir(dir);
+ const { pageCount, maxPageBytes } = await sharedPostsShape(opts.sharedPostsDir, slug);
+ await writePublicFile(
+ path.join(dir, "manifest.json"),
+ JSON.stringify(tombstoneChannelManifest(slug, generatedAt, maxPageBytes)),
+ );
+ for (let i = 0; i < pageCount; i++) {
+ await writePublicFile(path.join(dir, postsPageFileName(i)), "[]");
+ }
+ written.push({ slug, pages: pageCount });
+ }
+ return written;
+}
+
+/**
+ * The served paths a set of tombstones occupies, root-relative without a
+ * leading slash (`posts/<slug>/manifest.json`, `posts/<slug>/page-0000.json`,
+ * …), with `posts/manifest.json` first when `withManifest`. What a live check
+ * probes, and what a test pins.
+ */
+export function tombstonePaths(
+ tombstones: readonly PostsTombstone[],
+ opts: { withManifest?: boolean } = {},
+): string[] {
+ const out = opts.withManifest ? ["posts/manifest.json"] : [];
+ for (const t of tombstones) {
+ out.push(`posts/${t.slug}/manifest.json`);
+ for (let i = 0; i < t.pages; i++) out.push(`posts/${t.slug}/${postsPageFileName(i)}`);
+ }
+ return out;
+}
+
+/** A site's no-store paths: its posts manifest and each tombstoned tree. */
+export function tombstoneNoStoreForSite(tombstones: readonly PostsTombstone[]): string[] {
+ if (tombstones.length === 0) return [];
+ return ["/posts/manifest.json", ...tombstones.map((t) => `/posts/${t.slug}/*`)];
+}
+
+/** The hub's no-store paths: its whole posts tree, which holds only tombstones. */
+export function tombstoneNoStoreForHub(tombstones: readonly PostsTombstone[]): string[] {
+ return tombstones.length === 0 ? [] : ["/posts/*"];
+}
+
+/**
+ * Every X channel that has a SHARED posts tree, while X posts are private —
+ * the channels a hub tombstones (it never carries posts, and once carried a
+ * site's). Empty while X posts are public.
+ */
+export async function withdrawnXChannels(
+ paths: Paths,
+ settings: { social?: { x?: { visibility?: unknown } } },
+): Promise<string[]> {
+ if (xPostsVisibility(settings) !== "private") return [];
+ let entries: string[];
+ try {
+ entries = (await readdir(paths.exportSharedPostsDir, { withFileTypes: true }))
+ .filter((e) => e.isDirectory())
+ .map((e) => e.name)
+ .sort();
+ } catch {
+ return [];
+ }
+ const out: string[] = [];
+ for (const slug of entries) {
+ if (isXPostsChannel(await readChannelConfig(paths, slug))) out.push(slug);
+ }
+ return out;
+}
