commit 870d26ff9fa9941e203b2a51b7ebea58d5e05798
parent 6ccf2d78fb2d05c46893f6dc9703cf29bcb4cda8
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 1 Oct 2026 23:18:56 -0400
Merge main into r17/umtool-deliverables
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
37 files changed, 1843 insertions(+), 40 deletions(-)
diff --git a/SETTINGS.md b/SETTINGS.md
@@ -160,7 +160,7 @@ Default: `"when-required"`
## `social`
-Per-platform settings of the social-post fetchers. Today one key: where the X fetchers' login comes from (`social.x.cookieSource`, chosen in the X account session section of /settings). See common/social/xCookieSource.ts.
+Per-platform settings of the social posts. Today two keys, both X's, both chosen in the X account session section of /settings: where the X fetchers' login comes from (`social.x.cookieSource`) and where X posts may appear (`social.x.visibility`). See common/social/xCookieSource.ts.
#### `social`
@@ -173,6 +173,7 @@ Per-platform settings of the social-post fetchers. Today one key: where the X fe
| Key | Default | Description |
|---|---|---|
| `cookieSource` | absent | Where the X fetchers' login comes from. `"browser"`: the operator's everyday browser, named by `cookiesFromBrowser` — gallery-dl is handed `--cookies-from-browser <spec>` and reads it on every run, and the Playwright fallback reads the same store (Firefox only; common/social/xBrowserLogin.ts), so the login lasts as long as the browser's. `"profile"`: the session broker's persistent profile ("Connect X account" on /settings) and the cookie jar it exports. ABSENT (the default) is resolved at read time, never stored: `"browser"` when `cookiesFromBrowser` is set and no profile is connected (no exported jar carrying an auth_token), else `"profile"`. The source, not `cookieMode`, governs the X fetchers. |
+| `visibility` | absent | Where X posts may appear. `"public"` (the default; absent): an X channel's posts are built into every site that has the channel. `"private"`: every X channel's posts (a channel with `sourceKind: "social"` and `platform: "twitter"`) are left out of every PUBLIC site build — the channel with them, since posts are all an X channel holds — and built only into PRIVATE sites (`site.json` `audience`). Nothing on disk changes and fetching does not; a site already published changes on its next build and deploy, and flipping back is a rebuild. Chosen in the X account session section of /settings; the rule is common/lib/postsVisibility.ts. |
Default:
diff --git a/SITE.md b/SITE.md
@@ -24,6 +24,7 @@ Regenerate this file with `pnpm --filter yt-dlp-transcript-common exec tsx bin/f
| [`accent`](#accent) | absent |
| [`siteUrl`](#siteurl) | absent |
| [`listed`](#listed) | `true` |
+| [`audience`](#audience) | absent |
| [`relatedSites`](#relatedsites) | `[]` |
| [`pwa`](#pwa) | `false` |
| [`archives`](#archives) | `true` |
@@ -164,6 +165,12 @@ Whether the family lists this site. Opt-OUT: absent/true = listed, only an expli
Default: `true`
+## `audience`
+
+Who this site is built for. `"public"` (the default; absent) or `"private"`: the operator's own reading copy, built on this machine and never deployed — every deploy path (Build & deploy, Deploy, `archilyzer deploy site`, Build & deploy all, docker/publish-site.sh) refuses it before any upload, while a build without a deploy still works. A private site is never listed (as `listed: false`, whatever `listed` says), publishes no `hubUrl`, and its `/corpus.json` says `"audience": "private"`. Content kept from the public — X posts while `social.x.visibility` is `"private"` — is built only into private sites. Only `"private"` is written.
+
+Default: absent
+
## `relatedSites`
Pulls specific siblings to the front of the footer's cross-site list, in named groups. Siblings not named here fall into a trailing "Other sites" group. Absent/empty = one flat list of every sibling.
diff --git a/common/bin/compose-hub.test.ts b/common/bin/compose-hub.test.ts
@@ -16,6 +16,7 @@ import { tmpdir } from "node:os";
import path from "node:path";
import { getPaths, type Paths } from "../lib/paths";
import { main } from "./compose-hub";
+import { readGlobalAliases } from "../lib/aliasesStore";
// Run with:
// pnpm --filter yt-dlp-transcript-common test
@@ -170,3 +171,54 @@ test("an unlisted site is in none of the hub's files; a listed one is in each",
rmSync(root, { recursive: true, force: true });
}
});
+
+// Release 17 slice XP (the review's HIGH 1): a site's compose leaves its data
+// in public/ — a private site's X posts included — and the hub builds from
+// public/ next. compose-hub removes every per-site entry, through a link only
+// the link, and ships the global alias dictionary as its own.
+test("compose-hub removes a site's data from public/, a linked entry by its link only", async () => {
+ const root = mkdtempSync(path.join(tmpdir(), "compose-hub-"));
+ const log = console.log;
+ try {
+ const paths = fixturePaths(root);
+ const pub = paths.exportPublicDir;
+ // What a private site's compose leaves.
+ for (const tree of ["summaries", "transcripts", "subs", "digests", "stats", "archives"]) {
+ mkdirSync(path.join(pub, tree, "x"), { recursive: true });
+ writeFileSync(path.join(pub, tree, "x", "page-0000.json"), "[]");
+ }
+ for (const f of ["site.json", "tags.json", "duplicates.json", "chart-templates.json", "sitemap.xml"]) {
+ writeFileSync(path.join(pub, f), "{}");
+ }
+ writeFileSync(path.join(pub, "search-aliases.json"), JSON.stringify({ aliases: [{ site: 1 }] }));
+ // posts/ as a worktree has it: a link into the primary checkout.
+ const primaryPosts = path.join(root, "primary-public", "posts");
+ mkdirSync(path.join(primaryPosts, "jer-x"), { recursive: true });
+ writeFileSync(path.join(primaryPosts, "manifest.json"), '{"channels":[{"slug":"jer-x"}]}');
+ symlinkSync(primaryPosts, path.join(pub, "posts"));
+ // A checked-in static asset stays.
+ writeFileSync(path.join(pub, "globe.svg"), "<svg/>");
+ // No global dictionary file: the seeded defaults are the global one.
+ const hubPaths = { ...paths, globalAliasesFile: path.join(root, "no-aliases.json") };
+
+ console.log = () => {};
+ await main({ paths: hubPaths });
+ console.log = log;
+
+ for (const gone of [
+ "summaries", "transcripts", "subs", "posts", "digests", "stats", "archives",
+ "site.json", "tags.json", "duplicates.json", "chart-templates.json", "sitemap.xml",
+ ]) {
+ assert.ok(!existsSync(path.join(pub, gone)), `${gone} was removed`);
+ }
+ assert.ok(existsSync(path.join(primaryPosts, "manifest.json")), "the link's target is untouched");
+ assert.ok(existsSync(path.join(pub, "globe.svg")));
+ assert.ok(existsSync(path.join(pub, "hub-sites.json")));
+ const aliases = JSON.parse(readFileSync(path.join(pub, "search-aliases.json"), "utf8"));
+ assert.deepEqual(aliases, { aliases: readGlobalAliases(hubPaths).aliases });
+ assert.ok(!JSON.stringify(aliases).includes('"site"'), "not the site's aliases");
+ } finally {
+ console.log = log;
+ rmSync(root, { recursive: true, force: true });
+ }
+});
diff --git a/common/bin/compose-hub.ts b/common/bin/compose-hub.ts
@@ -11,6 +11,10 @@
// there is no index to walk
// public/_headers <- CORS for the hub's own served JSON
// public/sw.js <- the hub service worker (the hub always ships a PWA)
+// public/search-aliases.json <- the global alias dictionary
+//
+// and REMOVES every per-site entry a site's compose left in public/
+// (SITE_ONLY_PUBLIC_ENTRIES below): the hub holds no site's data.
//
// The hub's branding ("Archilyzer") is resolved at build/render time from the
// HomepageConfig (see export/app/lib/site.ts hubSite()), not composed here.
@@ -32,6 +36,7 @@ import {
import { HUB_CORS_PATHS, renderHeadersFile } from "../lib/archive/headers";
import { buildPoolSummary } from "../controller/poolSummary";
import { HUB_SUMMARY_FILE, toHubSummary } from "../lib/hubSummary";
+import { readGlobalAliases } from "../lib/aliasesStore";
import { runIfEntryPoint } from "./_cli";
import { writePublicFile } from "./_publicFile";
@@ -76,10 +81,45 @@ async function composeHubSummary(
}
}
+// THE HUB CARRIES NO SITE'S DATA (release 17 slice XP, the review's HIGH 1).
+// public/ is the one directory every site composes into in turn, and the hub
+// builds from it next: whatever the last site's compose left there — its data
+// trees and its per-site files — `next build` copied into the hub's out/, and
+// the hub deploy shipped it. The live hub served jeralyzer's posts manifest;
+// after a PRIVATE site's build it would have served every X post. So the hub's
+// compose removes every per-site entry first. A worktree's public/ entries are
+// links into the primary checkout: rm removes the link, never its target.
+export const SITE_ONLY_PUBLIC_ENTRIES: readonly string[] = [
+ "summaries",
+ "transcripts",
+ "subs",
+ "posts",
+ "digests",
+ "stats",
+ "archives",
+ "site.json",
+ "tags.json",
+ "duplicates.json",
+ "search-aliases.json",
+ "chart-templates.json",
+ "sitemap.xml",
+];
+
export async function main(opts: { paths?: Paths } = {}): Promise<void> {
const paths = opts.paths ?? getPaths();
const publicDir = paths.exportPublicDir;
+ for (const entry of SITE_ONLY_PUBLIC_ENTRIES) {
+ await rm(path.join(publicDir, entry), { recursive: true, force: true });
+ }
+ // The hub's own alias dictionary is the global one (no site's overrides):
+ // what a hub reader loads for hub-wide search (lib/archive/reader-hub.ts),
+ // where it used to get whichever site had composed last.
+ await writePublicFile(
+ path.join(publicDir, "search-aliases.json"),
+ JSON.stringify({ aliases: readGlobalAliases(paths).aliases }),
+ );
+
// Built-in pool: every configured site that publishes a public URL and is
// listed. An unlisted site (`listed: false`) still builds and deploys, but the
// hub does not list it: not a member, not in federated search, not in the
diff --git a/common/bin/compose-site.postsVisibility.test.ts b/common/bin/compose-site.postsVisibility.test.ts
@@ -0,0 +1,331 @@
+// Integration: X posts are private (release 17 slice XP), through the REAL
+// index build and the REAL site compose, over a temp corpus.
+//
+// One video channel, one X channel and one Bluesky channel, on two sites that
+// both have all three: `pub` (public) and `priv` (`audience: "private"`). With
+// `social.x.visibility` "private", the public site's build carries no X channel
+// at all — no posts manifest entry, no posts tree, no channel in its channel
+// list, site.json or corpus.json — while the private one carries everything
+// and says `"audience": "private"` with no hubUrl. Flipping the setting back
+// is a rebuild. The rule itself is lib/postsVisibility.ts (its own tests).
+//
+// The export e2e cannot show this: its data is route-mocked, never built by
+// buildIndex and compose. export/e2e/x-posts-private.spec.ts serves the two
+// posts manifests this file pins and checks what a visitor sees.
+//
+// Run with: node_modules/.bin/tsx --test common/bin/compose-site.postsVisibility.test.ts
+
+import { after, test } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
+// Every path getPaths() can resolve to a place this file's code may write is
+// pinned under ROOT before anything calls it (buildIndex.test.ts's list).
+const ROOT = mkdtempSync(path.join(tmpdir(), "posts-visibility-"));
+const PINNED: Record<string, string> = {
+ TRANSCRIPTS_DIR: path.join(ROOT, "transcripts"),
+ SAVED_VIDEOS_DIR: path.join(ROOT, "saved-videos"),
+ SITES_DIR: path.join(ROOT, "transcripts", "sites"),
+ SETTINGS_FILE: path.join(ROOT, "settings.json"),
+ EXPORT_PUBLIC_DIR: path.join(ROOT, "public"),
+ EXPORT_INDEX_DIR: path.join(ROOT, ".export-index"),
+ EXPORT_BUILDS_DIR: path.join(ROOT, ".export-builds"),
+ EDITOR_CHANGELOG_FILE: path.join(ROOT, "editor-CHANGELOG.md"),
+ EXPORT_CHANGELOG_FILE: path.join(ROOT, "export-CHANGELOG.md"),
+ CHARTS_CONFIG_FILE: path.join(ROOT, "chart-templates.json"),
+ SEARCH_ALIASES_FILE: path.join(ROOT, "transcripts", "search-aliases.json"),
+ CURATED_TAGS_FILE: path.join(ROOT, "transcripts", "tags.json"),
+ ARCHILYZER_CONFIG_DIR: path.join(ROOT, "config"),
+ ARCHILYZER_SOURCE_SCRATCH: path.join(ROOT, "source-scratch"),
+};
+Object.assign(process.env, PINNED);
+after(() => rmSync(ROOT, { recursive: true, force: true }));
+
+const { getPaths } = await import("../lib/paths");
+const { buildIndex } = await import("../controller/buildIndex");
+const { writePosts } = await import("../lib/posts-server");
+const { main: composeSite } = await import("./compose-site");
+const { builtAudienceProblem, deployAudienceProblem } = await import("../lib/builtExport");
+
+const paths = getPaths();
+const VIDEOS = "vids";
+const X = "jer-x";
+const SKY = "jer-sky";
+const HUB = "https://hub.example.test";
+
+const writeJson = (file: string, value: unknown) => {
+ mkdirSync(path.dirname(file), { recursive: true });
+ writeFileSync(file, JSON.stringify(value, null, 2));
+};
+const readJson = <T>(file: string): T => JSON.parse(readFileSync(file, "utf8")) as T;
+
+// YouTube's rolling-caption shape (parseVtt keeps lines with inline timing).
+const VTT =
+ "WEBVTT\nKind: captions\nLanguage: en\n\n" +
+ "00:00:00.000 --> 00:00:05.000 align:start position:0%\n" +
+ "First<00:00:01.000><c> caption</c><00:00:02.000><c> line.</c>\n";
+
+function post(slug: string, id: string, platform: "twitter" | "bluesky", text: string) {
+ return {
+ id,
+ slug: `${slug}/${id}`,
+ channelSlug: slug,
+ author: slug,
+ createdAt: "2026-09-01T12:00:00.000Z",
+ uploadDate: "20260901",
+ text,
+ url: `https://example.test/${slug}/${id}`,
+ platform,
+ isReply: false,
+ isRepost: false,
+ links: [],
+ };
+}
+
+function writeSettings(visibility?: "public" | "private") {
+ writeJson(paths.settingsFile, {
+ homepageUrl: HUB,
+ ...(visibility ? { social: { x: { visibility } } } : {}),
+ });
+}
+
+async function seedCorpus() {
+ writeJson(path.join(paths.channelsDir, VIDEOS, "config.json"), {
+ handling: "youtube",
+ name: "Videos",
+ url: "https://www.youtube.com/@vids/videos",
+ });
+ const dir = path.join(paths.channelsDir, VIDEOS, "data", "v1");
+ writeJson(path.join(dir, "metadata.info.json"), {
+ id: "v1",
+ title: "Video one",
+ channel: VIDEOS,
+ upload_date: "20260601",
+ duration: 120,
+ webpage_url: "https://www.youtube.com/watch?v=v1",
+ extractor_key: "Youtube",
+ });
+ writeFileSync(path.join(dir, "transcript.en.vtt"), VTT);
+
+ writeJson(path.join(paths.channelsDir, X, "config.json"), {
+ handling: "youtube",
+ name: "Jer on X",
+ url: "https://x.com/jer",
+ sourceKind: "social",
+ platform: "twitter",
+ socialHandle: "jer",
+ });
+ await writePosts(path.join(paths.channelsDir, X), [
+ post(X, "1001", "twitter", "an x post"),
+ post(X, "1002", "twitter", "another x post"),
+ ]);
+ writeJson(path.join(paths.channelsDir, SKY, "config.json"), {
+ handling: "youtube",
+ name: "Jer on Bluesky",
+ url: "https://bsky.app/profile/jer.example",
+ sourceKind: "social",
+ platform: "bluesky",
+ socialHandle: "jer.example",
+ });
+ await writePosts(path.join(paths.channelsDir, SKY), [
+ post(SKY, "3kabc", "bluesky", "a bluesky post"),
+ ]);
+
+ const site = (siteId: string, extra: Record<string, unknown> = {}) =>
+ writeJson(path.join(paths.sitesDir, siteId, "site.json"), {
+ siteId,
+ siteTitle: siteId,
+ siteDescription: "fixture",
+ headerTitle: siteId,
+ homeTagline: "",
+ socialLinks: [],
+ groups: [{ id: "default", name: "All channels", selectedByDefault: true }],
+ defaultGroupId: "default",
+ channels: [VIDEOS, X, SKY].map((slug) => ({ slug, groupId: "default" })),
+ siteUrl: `https://${siteId}.example.test`,
+ archives: false,
+ ...extra,
+ });
+ site("pub");
+ site("priv", { audience: "private" });
+}
+
+const quiet = () => {};
+async function index() {
+ await buildIndex({ paths, onLog: quiet });
+}
+
+// What one site's compose put in public/.
+type Composed = {
+ postsManifest: { channels: { slug: string; platform: string; postCount: number }[]; totalCount: number };
+ postTrees: string[];
+ transcriptTrees: string[];
+ siteJson: { channels: { slug: string }[]; hubUrl?: string };
+ corpus: {
+ site: { id: string; hubUrl?: string; audience?: string };
+ channels: { slug: string; postCount?: number; manifests: { posts?: string } }[];
+ postScheme?: unknown;
+ totals: { channels: number };
+ };
+};
+async function compose(siteId: string): Promise<Composed> {
+ const log = console.log;
+ console.log = quiet;
+ try {
+ await composeSite({ siteId, paths });
+ } finally {
+ console.log = log;
+ }
+ const pub = paths.exportPublicDir;
+ const trees = (dir: string) =>
+ [VIDEOS, X, SKY].filter((slug) => existsSync(path.join(dir, slug)));
+ return {
+ postsManifest: readJson(path.join(pub, "posts", "manifest.json")),
+ postTrees: trees(path.join(pub, "posts")),
+ transcriptTrees: trees(path.join(pub, "transcripts")),
+ siteJson: readJson(path.join(pub, "site.json")),
+ corpus: readJson(path.join(pub, "corpus.json")),
+ };
+}
+
+const slugs = (xs: { slug: string }[]) => xs.map((c) => c.slug).sort();
+
+test("X private: a public site carries no X channel; a private site carries all of it and says so", async () => {
+ await seedCorpus();
+ writeSettings("private");
+ await index();
+
+ // The index build's per-site manifests, before any compose.
+ const sitePosts = (id: string) =>
+ readJson<Composed["postsManifest"]>(
+ path.join(paths.exportSitesIndexDir, id, "posts", "manifest.json"),
+ );
+ assert.deepEqual(slugs(sitePosts("pub").channels), [SKY]);
+ assert.deepEqual(slugs(sitePosts("priv").channels), [SKY, X]);
+ // The shared posts tree is corpus-wide and keeps X: the private site and
+ // the MCP over its build read it.
+ assert.ok(existsSync(path.join(paths.exportSharedPostsDir, X, "manifest.json")));
+
+ const pub = await compose("pub");
+ assert.deepEqual(slugs(pub.postsManifest.channels), [SKY]);
+ assert.equal(pub.postsManifest.totalCount, 1);
+ assert.deepEqual(pub.postTrees, [SKY]);
+ assert.deepEqual(pub.transcriptTrees, [VIDEOS, SKY]);
+ assert.deepEqual(slugs(pub.siteJson.channels), [SKY, VIDEOS]);
+ assert.deepEqual(slugs(pub.corpus.channels), [SKY, VIDEOS]);
+ assert.equal(pub.corpus.totals.channels, 2);
+ assert.equal(pub.corpus.site.audience, undefined);
+ assert.equal(pub.corpus.site.hubUrl, HUB);
+ assert.equal(pub.siteJson.hubUrl, HUB);
+ assert.equal(builtAudienceProblem(paths.exportPublicDir), null);
+ assert.equal(deployAudienceProblem({ siteId: "pub" }, paths.exportPublicDir), null);
+
+ const priv = await compose("priv");
+ assert.deepEqual(slugs(priv.postsManifest.channels), [SKY, X]);
+ assert.equal(priv.postsManifest.totalCount, 3);
+ assert.deepEqual(priv.postTrees, [X, SKY]);
+ assert.deepEqual(slugs(priv.corpus.channels), [SKY, X, VIDEOS]);
+ assert.equal(priv.corpus.channels.find((c) => c.slug === X)?.postCount, 2);
+ assert.ok(priv.corpus.channels.find((c) => c.slug === X)?.manifests.posts);
+ assert.equal(priv.corpus.site.audience, "private");
+ // A private site belongs under no hub.
+ assert.equal(priv.corpus.site.hubUrl, undefined);
+ assert.equal(priv.siteJson.hubUrl, undefined);
+ // And its bundle refuses to deploy — under its own id, and under another
+ // site's (a public site's identity on a private build is still refused).
+ assert.match(builtAudienceProblem(paths.exportPublicDir) ?? "", /private build of "priv"/);
+ assert.match(
+ deployAudienceProblem({ siteId: "priv", audience: "private" }, paths.exportPublicDir) ?? "",
+ /^Site "priv" is private \(audience: private\)/,
+ );
+
+ // Compose the public site again over the private one's public/, with
+ // nothing changed since its last compose: the X channel the private site
+ // carried is pruned from every tree, and the summaries are the public
+ // site's again — its compose cache is not trusted over another site's
+ // compose (the summaries used to be skipped as "unchanged" and shipped the
+ // private site's channel list).
+ const again = await compose("pub");
+ assert.deepEqual(again.postTrees, [SKY]);
+ assert.deepEqual(again.transcriptTrees, [VIDEOS, SKY]);
+ assert.deepEqual(slugs(again.siteJson.channels), [SKY, VIDEOS]);
+ assert.deepEqual(slugs(again.corpus.channels), [SKY, VIDEOS]);
+ assert.equal(again.corpus.site.audience, undefined);
+ // And over its own last compose the cache is trusted as before.
+ const third = await compose("pub");
+ assert.deepEqual(slugs(third.corpus.channels), [SKY, VIDEOS]);
+});
+
+test("a config compose cannot read does not ship the posts tree the index build withheld", async () => {
+ // The index build (X private) withheld X from pub's posts manifest; compose
+ // then fails to read X's config, so its own rule reads X as visible. The
+ // site posts manifest is the index build's word: no X posts tree ships.
+ writeSettings("private");
+ await index();
+ const cfg = path.join(paths.channelsDir, X, "config.json");
+ const saved = readFileSync(cfg, "utf8");
+ rmSync(cfg);
+ try {
+ const pub = await compose("pub");
+ assert.deepEqual(pub.postTrees, [SKY]);
+ assert.deepEqual(slugs(pub.postsManifest.channels), [SKY]);
+ assert.deepEqual(slugs(pub.corpus.channels), [SKY, VIDEOS]);
+ } finally {
+ writeFileSync(cfg, saved);
+ }
+});
+
+test("a compose over an index built before the setting flipped lists no X channel anywhere", async () => {
+ // Index with X public, then flip to private and compose WITHOUT indexing.
+ writeSettings("public");
+ await index();
+ writeSettings("private");
+ const pub = await compose("pub");
+ assert.deepEqual(pub.postTrees, [SKY]);
+ assert.deepEqual(slugs(pub.postsManifest.channels), [SKY]);
+ assert.equal(pub.postsManifest.totalCount, 1);
+ assert.equal(pub.corpus.channels.find((c) => c.slug === X)?.postCount, undefined);
+ assert.equal(pub.corpus.postScheme !== undefined, true, "the Bluesky posts are still advertised");
+ // The channel list too: no X channel in site.json or corpus.json.
+ assert.deepEqual(slugs(pub.siteJson.channels), [SKY, VIDEOS]);
+ assert.deepEqual(slugs(pub.corpus.channels), [SKY, VIDEOS]);
+});
+
+test("X public again: the next build puts X back on the public site", async () => {
+ writeSettings("public");
+ await index();
+ const pub = await compose("pub");
+ assert.deepEqual(slugs(pub.postsManifest.channels), [SKY, X]);
+ assert.deepEqual(pub.postTrees, [X, SKY]);
+ assert.deepEqual(slugs(pub.corpus.channels), [SKY, X, VIDEOS]);
+
+ // No setting at all is public too.
+ writeSettings();
+ await index();
+ assert.deepEqual(slugs((await compose("pub")).postsManifest.channels), [SKY, X]);
+});
+
+test("a public site whose only posts were X posts ships an empty posts manifest and no post scheme", async () => {
+ writeSettings("private");
+ writeJson(path.join(paths.sitesDir, "xonly", "site.json"), {
+ siteId: "xonly",
+ siteTitle: "xonly",
+ siteDescription: "fixture",
+ headerTitle: "xonly",
+ homeTagline: "",
+ groups: [{ id: "default", name: "All channels", selectedByDefault: true }],
+ defaultGroupId: "default",
+ channels: [VIDEOS, X].map((slug) => ({ slug, groupId: "default" })),
+ archives: false,
+ });
+ await index();
+ const xonly = await compose("xonly");
+ // SearchSessionContext's hasPostsCorpus is `channels.length > 0`: no Posts
+ // toggle on this site, with nothing special-cased.
+ assert.deepEqual(xonly.postsManifest.channels, []);
+ assert.deepEqual(xonly.postTrees, []);
+ assert.equal(xonly.corpus.postScheme, undefined);
+ assert.deepEqual(slugs(xonly.corpus.channels), [VIDEOS]);
+});
diff --git a/common/bin/compose-site.ts b/common/bin/compose-site.ts
@@ -60,6 +60,10 @@ import {
type ArchiveManifest,
type ArchiveManifestEntry,
} from "../lib/archiveOptions";
+import { readChannelConfig } from "../controller/channels";
+import { builtSiteIdIn } from "../lib/builtExport";
+import { publishedMemberSlugs } from "../lib/postsVisibility";
+import { isPrivateSite } from "../lib/siteSchema";
import { runIfEntryPoint } from "./_cli";
import { copyPublicFile, ownDir, writePublicFile } from "./_publicFile";
@@ -98,7 +102,10 @@ async function emitFederationFiles(
// navigate the already-served paginated shards; they never enumerate per-video
// files, so the count is constant regardless of corpus size. Runs after
// site.json and the archives are composed (both feed into these files).
-async function emitAiFiles(paths: ReturnType<typeof getPaths>): Promise<void> {
+async function emitAiFiles(
+ site: Site,
+ paths: ReturnType<typeof getPaths>,
+): Promise<void> {
const sitePath = path.join(paths.exportPublicDir, "site.json");
if (!(await exists(sitePath))) return; // no composed data → nothing to describe
const descriptor = JSON.parse(
@@ -161,6 +168,9 @@ async function emitAiFiles(paths: ReturnType<typeof getPaths>): Promise<void> {
postCounts,
digestCounts,
hasTags,
+ // A private site says so in its own corpus.json — the bundle's word that
+ // the deploy guard (lib/builtExport.ts builtAudienceProblem) reads.
+ private: isPrivateSite(site),
});
await writePublicFile(
path.join(paths.exportPublicDir, "corpus.json"),
@@ -485,6 +495,34 @@ async function replaceDir(src: string, dest: string): Promise<void> {
}
}
+// Rewrite a served manifest whose `channels` name a slug outside `members`,
+// keeping the rest of it. A missing or unreadable manifest is left alone.
+// Answers whether it rewrote the file.
+async function narrowManifestChannels(file: string, members: Set<string>): Promise<boolean> {
+ let m: { channels?: { slug?: string }[] };
+ try {
+ m = JSON.parse(await readFile(file, "utf8"));
+ } catch {
+ return false;
+ }
+ const channels = m.channels ?? [];
+ const kept = channels.filter((c) => typeof c.slug !== "string" || members.has(c.slug));
+ if (kept.length === channels.length) return false;
+ await writePublicFile(file, JSON.stringify({ ...m, channels: kept }));
+ return true;
+}
+
+// The channel slugs a site posts manifest lists, or null when there is no
+// readable manifest.
+async function readPostsManifestSlugs(file: string): Promise<Set<string> | null> {
+ try {
+ const pm = JSON.parse(await readFile(file, "utf8")) as PostsManifest;
+ return new Set((pm.channels ?? []).map((c) => c.slug));
+ } catch {
+ return null;
+ }
+}
+
// --- Incremental compose cache ------------------------------------------------
// Per-site record of what we last materialized into public/, keyed by a cheap
// content signature of each source. When the signature is unchanged and the
@@ -665,11 +703,58 @@ export async function main(
}
const paths = opts.paths ?? getPaths();
const site = getSite(siteId, paths);
- const memberSlugs = site.channels.map((c) => c.slug);
+ // The members this site may publish (lib/postsVisibility.ts): every member,
+ // less an X channel while `social.x.visibility` is "private" and the site is
+ // public. The index build's per-site manifests are narrowed by the same rule;
+ // narrowing the trees here prunes an X channel a public site shipped before.
+ const configs = new Map(
+ await Promise.all(
+ site.channels.map(
+ async (c) => [c.slug, await readChannelConfig(paths, c.slug).catch(() => null)] as const,
+ ),
+ ),
+ );
+ const memberSlugs = publishedMemberSlugs(
+ site,
+ (slug) => configs.get(slug),
+ getSettings(),
+ );
+ const withheld = site.channels.length - memberSlugs.length;
+ if (withheld > 0) {
+ console.log(
+ `[compose] ${withheld} X channel(s) left out: X posts are private (social.x.visibility) and this site is public.`,
+ );
+ }
// Incremental compose: skip stages whose source is unchanged since last build.
+ //
+ // ONLY OVER THIS SITE'S OWN LAST COMPOSE. The cache is per site but public/
+ // is one directory every site composes into in turn (the basic build), so
+ // after another site's compose a skipped stage would ship THAT site's files —
+ // its summaries, its whole channel list — under this site's name: composing
+ // a private site and then a public one shipped the private site's summaries
+ // as the public site's. public/site.json names the site composed into it
+ // last (it is written below, every compose); another name, or none, and the
+ // site's own stages (summaries, stats, duplicates) are composed afresh.
+ //
+ // The per-channel tree signatures stay trusted: those trees are copies of
+ // the SHARED trees, the same bytes whichever site copied them, and a
+ // channel another site pruned is re-copied because its directory is gone.
const cachePath = composeCachePath(paths, siteId);
- const cache = await readComposeCache(cachePath);
+ const lastComposed = builtSiteIdIn(paths.exportPublicDir);
+ const cached = await readComposeCache(cachePath);
+ const cache: ComposeCache =
+ lastComposed === siteId
+ ? cached
+ : { ...cached, summaries: undefined, stats: undefined, duplicates: undefined };
+ if (lastComposed !== siteId && lastComposed !== null) {
+ console.log(
+ `[compose] public/ was last composed for "${lastComposed}": composing ${siteId}'s summaries, stats and duplicates afresh.`,
+ );
+ }
+ // Until this compose writes its own, public/ names no site: a compose cut
+ // short part-way leaves the next one nothing to trust.
+ await rm(path.join(paths.exportPublicDir, "site.json"), { force: true });
// --- per-site aggregates (whole-dir swaps), gated on the source signature ---
const summariesSrc = path.join(paths.exportSitesIndexDir, siteId, "summaries");
@@ -683,6 +768,19 @@ export async function main(
} else {
console.log("[compose] summaries: unchanged.");
}
+ // The served summaries manifest lists only the members this compose
+ // publishes (lib/postsVisibility.ts), even over an index built before
+ // `social.x.visibility` flipped: site.json and corpus.json are built from it.
+ // A narrowed copy is no longer the source's copy: the next compose copies
+ // the summaries again rather than trusting it.
+ if (
+ await narrowManifestChannels(
+ path.join(paths.exportSummariesDir, "manifest.json"),
+ new Set(memberSlugs),
+ )
+ ) {
+ cache.summaries = undefined;
+ }
const statsSrc = path.join(paths.exportSitesIndexDir, siteId, "stats");
const statsSig = await dirSignature(statsSrc);
if (cache.stats !== statsSig || !(await exists(paths.exportStatsDir))) {
@@ -713,11 +811,20 @@ export async function main(
// The social-post corpus: same shared-tree shape, same incremental reconcile.
// Only social member channels have a source dir; reconcileChannelTree treats a
// missing one as "nothing to copy", so passing every member slug is correct.
+ //
+ // NEVER A TREE THE SITE'S POSTS MANIFEST DOES NOT LIST. The index build wrote
+ // that manifest by the same visibility rule as memberSlugs above, so the two
+ // agree — unless a channel's config could not be read here (it then reads as
+ // visible): the manifest is the index build's word, and a tree it withheld is
+ // not shipped on a failed read. A site with no manifest yet keeps the old rule.
+ const postsListed = await readPostsManifestSlugs(
+ path.join(paths.exportSitesIndexDir, siteId, "posts", "manifest.json"),
+ );
cache.posts = await reconcileChannelTree(
"posts",
paths.exportSharedPostsDir,
paths.exportPostsDir,
- memberSlugs,
+ postsListed ? memberSlugs.filter((slug) => postsListed.has(slug)) : memberSlugs,
cache.posts ?? {},
console.log,
);
@@ -752,7 +859,21 @@ export async function main(
);
if (await exists(postsManifestSrc)) {
await ownDir(paths.exportPostsDir);
- await copyPublicFile(postsManifestSrc, path.join(paths.exportPostsDir, "manifest.json"));
+ // Narrowed to the members this compose publishes, so a compose run over an
+ // index built before `social.x.visibility` flipped (`archilyzer compose
+ // site` alone, `build site --nodata`) does not list a withheld X channel's
+ // name and count beside the pruned tree.
+ const pm = JSON.parse(await readFile(postsManifestSrc, "utf8")) as PostsManifest;
+ const members = new Set(memberSlugs);
+ const channels = (pm.channels ?? []).filter((c) => members.has(c.slug));
+ await writePublicFile(
+ path.join(paths.exportPostsDir, "manifest.json"),
+ JSON.stringify({
+ ...pm,
+ channels,
+ totalCount: channels.reduce((n, c) => n + (c.postCount ?? 0), 0),
+ }),
+ );
}
// Same for the per-site digests manifest (which channels carry digests).
const digestsManifestSrc = path.join(
@@ -934,7 +1055,7 @@ export async function main(
// --- AI discovery: llms.txt / corpus.json / robots.txt / sitemap.xml ---
// (after site.json + archives — both feed into these fixed-count files)
- await emitAiFiles(paths);
+ await emitAiFiles(site, paths);
// Persist the incremental-compose signatures for the next build.
await writeComposeCache(cachePath, cache);
diff --git a/common/controller/buildIndex.ts b/common/controller/buildIndex.ts
@@ -141,6 +141,7 @@ import {
postsAvailabilityPath,
} from "../lib/posts-server";
import { isSocialChannel } from "../lib/channelConfig";
+import { publishedMemberSlugs } from "../lib/postsVisibility";
import {
DIGESTS_MANIFEST_VERSION,
SITE_DIGESTS_MANIFEST_VERSION,
@@ -1940,8 +1941,21 @@ export async function buildIndex({
let aggregateSummaries = 0;
let representativeChannelCount = 0;
+ // Which members a site may publish: an X channel is built only into private
+ // sites while `social.x.visibility` is "private" (lib/postsVisibility.ts —
+ // compose-site narrows its trees by the same rule).
+ const visibilitySettings = getSettings();
for (const site of sites) {
- const memberSlugs = site.channels.map((c) => c.slug);
+ const memberSlugs = publishedMemberSlugs(
+ site,
+ (slug) => channelConfigs.get(slug),
+ visibilitySettings,
+ );
+ // Members the rule leaves out of THIS build, named in the fingerprint below
+ // so flipping the setting (or the site's audience) rebuilds the site.
+ const withheld = site.channels
+ .map((c) => c.slug)
+ .filter((slug) => !memberSlugs.includes(slug));
const slugSet = new Set(memberSlugs);
const slugGroup = new Map<string, string>();
for (const m of site.channels) {
@@ -1972,6 +1986,9 @@ export async function buildIndex({
// yesterday's counts.
curatedRules: curated.rulesHash,
curatedAssign: curated.assignHash,
+ // Only when the visibility rule withholds a member, so a site it does
+ // not touch keeps the fingerprint it had.
+ ...(withheld.length > 0 ? { withheld } : {}),
});
const fpKey = `siteFp:${site.siteId}`;
if (
diff --git a/common/controller/poolSummary.test.ts b/common/controller/poolSummary.test.ts
@@ -26,3 +26,26 @@ test("channel-sites.json names listed sites only; a channel only an unlisted sit
});
assert.ok(!JSON.stringify(channelSitesOf(sites)).includes("fixture-unlisted"));
});
+
+// Release 17 slice XP: a PRIVATE site is never listed, so it is in neither; and
+// with X posts private an X channel a public site leaves out of its build is
+// not mapped to that site (buildPoolSummary passes the narrowing).
+test("channel-sites.json leaves out a private site, and an X channel its public site withholds", async () => {
+ const { publishedMemberSlugs } = await import("../lib/postsVisibility");
+ const sites = [
+ parseSite("fixture-a", { channels: [{ slug: "vids" }, { slug: "jer-x" }] }),
+ parseSite("fixture-private", {
+ audience: "private",
+ channels: [{ slug: "vids" }, { slug: "jer-x" }, { slug: "own" }],
+ }),
+ ];
+ const configs: Record<string, { sourceKind?: "social"; platform?: "twitter" }> = {
+ "jer-x": { sourceKind: "social", platform: "twitter" },
+ };
+ const privateX = { social: { x: { visibility: "private" } } };
+ assert.deepEqual(
+ channelSitesOf(sites, (site) => publishedMemberSlugs(site, (slug) => configs[slug], privateX)),
+ { vids: ["fixture-a"] },
+ );
+ assert.deepEqual(channelSitesOf(sites), { vids: ["fixture-a"], "jer-x": ["fixture-a"] });
+});
diff --git a/common/controller/poolSummary.ts b/common/controller/poolSummary.ts
@@ -12,6 +12,9 @@ import { mkdir, readFile } from "node:fs/promises";
import type { Paths } from "../lib/paths";
import { buildStats } from "./buildStats";
import { isListedSite, listSites, type Site } from "../lib/site";
+import { getSettings } from "../lib/settings";
+import { publishedMemberSlugs } from "../lib/postsVisibility";
+import { readChannelConfig } from "./channels";
import {
statsPageFileName,
type StatsManifest,
@@ -49,12 +52,21 @@ export async function readStatsPages(statsDir: string): Promise<VideoStat[]> {
// published `channel-sites.json`. A channel on multiple sites maps to all of
// them; a pool-only channel is simply absent, and so is an unlisted site
// (site.json `listed: false`) and a channel only unlisted sites expose.
-export function channelSitesOf(sites: readonly Site[]): ChannelSitesMap {
+//
+// `membersOf` answers the members a site's build publishes; absent, every
+// member. buildPoolSummary passes lib/postsVisibility.ts's narrowing, so an X
+// channel a public site leaves out while X posts are private is not mapped to
+// that site here either.
+export function channelSitesOf(
+ sites: readonly Site[],
+ membersOf: (site: Site) => readonly string[] = (site) =>
+ site.channels.map((c) => c.slug),
+): ChannelSitesMap {
const channelSites: ChannelSitesMap = {};
for (const site of sites) {
if (!isListedSite(site)) continue;
- for (const c of site.channels) {
- (channelSites[c.slug] ??= []).push(site.siteId);
+ for (const slug of membersOf(site)) {
+ (channelSites[slug] ??= []).push(site.siteId);
}
}
return channelSites;
@@ -79,7 +91,15 @@ export async function buildPoolSummary(opts: {
// side effect, which is harmless.
await buildStats({ paths, wholePoolStatsDir: statsDir });
const sites = listSites(paths);
- const channelSites = channelSitesOf(sites);
+ // The members each site's build publishes (lib/postsVisibility.ts).
+ const configs = new Map<string, Awaited<ReturnType<typeof readChannelConfig>>>();
+ for (const slug of new Set(sites.flatMap((s) => s.channels.map((c) => c.slug)))) {
+ configs.set(slug, await readChannelConfig(paths, slug).catch(() => null));
+ }
+ const settings = getSettings();
+ const channelSites = channelSitesOf(sites, (site) =>
+ publishedMemberSlugs(site, (slug) => configs.get(slug), settings),
+ );
const stats = await readStatsPages(statsDir);
const summary = buildHomepageSummary(
stats,
diff --git a/common/lib/builtExport.test.ts b/common/lib/builtExport.test.ts
@@ -116,6 +116,14 @@ test("builtHubProblem accepts only a hub bundle", () => {
builtHubProblem(none.dir),
"export/out holds no hub build — build the hub first",
);
+
+ // Release 17 XP: a hub bundle composed over a site's data is refused.
+ mkdirSync(path.join(hub.dir, "posts", "jer-x"), { recursive: true });
+ mkdirSync(path.join(hub.dir, "summaries"));
+ assert.equal(
+ builtHubProblem(hub.dir),
+ "export/out holds a hub build that still carries a site's data (summaries, posts) — build the hub again",
+ );
} finally {
hub.cleanup();
site.cleanup();
diff --git a/common/lib/builtExport.ts b/common/lib/builtExport.ts
@@ -98,6 +98,63 @@ export function builtBundleProblem(outDir: string, siteId: string): string | nul
return null;
}
+/**
+ * Why `site` may not be deployed because of who it is built for, as one
+ * sentence — or null when it may (release 17 slice XP).
+ *
+ * A PRIVATE site (`site.json` `audience: "private"`) is the operator's own
+ * reading copy: it may carry what the public may not (X posts while
+ * `social.x.visibility` is "private"), so no deploy path ships it. Every
+ * deploy path asks this BEFORE ANY UPLOAD — the R2 archive push included — in
+ * the place it asks builtBundleProblem: runDeployIntoLog, the container deploy
+ * phase, deploySite, and the editor's Build & deploy, Deploy and Build &
+ * deploy all. A build without a deploy is untouched.
+ */
+export function siteDeployProblem(site: {
+ siteId: string;
+ audience?: string;
+}): string | null {
+ if (site.audience !== "private") return null;
+ return (
+ `Site "${site.siteId}" is private (audience: private): it is built for reading ` +
+ `on this machine and is never deployed. Build it without deploying, or set its ` +
+ `audience to public on its Settings tab`
+ );
+}
+
+/**
+ * Why the bundle in `outDir` may not be deployed because it was built PRIVATE
+ * — its corpus.json says `"audience": "private"` (compose-site writes it for a
+ * private site) — as one sentence, or null. Asked beside siteDeployProblem, so
+ * a site switched to public after a private build still cannot ship that
+ * build: it is rebuilt first.
+ */
+export function builtAudienceProblem(outDir: string): string | null {
+ try {
+ const parsed: unknown = JSON.parse(readFileSync(path.join(outDir, "corpus.json"), "utf8"));
+ const site = (parsed as { site?: { audience?: unknown; id?: unknown } } | null)?.site;
+ if (site?.audience !== "private") return null;
+ const id = typeof site.id === "string" ? ` of "${site.id}"` : "";
+ return (
+ `${outDir} holds a private build${id} (its corpus.json says "audience": "private"), ` +
+ `which is never deployed`
+ );
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * Both audience refusals, the site's first: the one sentence a deploy path
+ * logs or throws, or null.
+ */
+export function deployAudienceProblem(
+ site: { siteId: string; audience?: string },
+ outDir: string,
+): string | null {
+ return siteDeployProblem(site) ?? builtAudienceProblem(outDir);
+}
+
// corpus.json's `site.id`, or null when there is no readable one.
function corpusSiteIdIn(outDir: string): string | null {
try {
@@ -128,9 +185,23 @@ export function builtHubProblem(outDir: string): string | null {
if (!existsSync(path.join(outDir, "hub-sites.json"))) {
return "export/out holds no hub build — build the hub first";
}
+ // The hub holds no site's data (compose-hub removes it): a hub bundle that
+ // still carries a site's data trees was composed over one, and could ship
+ // that site's posts — a private site's included.
+ const carried = HUB_FORBIDDEN_TREES.filter((tree) => existsSync(path.join(outDir, tree)));
+ if (carried.length > 0) {
+ return (
+ `export/out holds a hub build that still carries a site's data (${carried.join(", ")}) — ` +
+ `build the hub again`
+ );
+ }
return null;
}
+// The per-site data trees a hub bundle must never carry (the trees of
+// compose-hub's SITE_ONLY_PUBLIC_ENTRIES).
+const HUB_FORBIDDEN_TREES = ["summaries", "transcripts", "subs", "posts", "digests", "stats", "archives"];
+
/**
* Why `outDir` — the homepage package's `homepage/out` — may not be deployed as
* the homepage, as one sentence, or null when it holds a build.
diff --git a/common/lib/corpus.ts b/common/lib/corpus.ts
@@ -175,6 +175,9 @@ export type SiteCorpus = {
description: string;
url?: string;
hubUrl?: string;
+ // Present only on a PRIVATE site's build (site.json `audience`, release 17
+ // slice XP): the operator's own reading copy, which no deploy path ships.
+ audience?: "private";
};
totals: { channels: number; videos: number };
channels: CorpusChannel[];
@@ -229,6 +232,9 @@ export function buildSiteCorpus(
// visible tag has a non-zero count here). Absent/false leaves corpus.json
// shaped as before apart from the spec bump.
hasTags?: boolean;
+ // A private site's build (site.json `audience: "private"`): corpus.json's
+ // `site.audience` says so. Absent/false leaves corpus.json as before.
+ private?: boolean;
},
): SiteCorpus {
const base = descriptor.siteUrl;
@@ -268,6 +274,7 @@ export function buildSiteCorpus(
description: descriptor.siteDescription,
...(descriptor.siteUrl ? { url: descriptor.siteUrl } : {}),
...(descriptor.hubUrl ? { hubUrl: descriptor.hubUrl } : {}),
+ ...(opts.private ? { audience: "private" as const } : {}),
},
totals: { channels: channels.length, videos },
channels,
diff --git a/common/lib/postsVisibility.test.ts b/common/lib/postsVisibility.test.ts
@@ -0,0 +1,99 @@
+// The one rule for which sites an X channel's posts are built into (release 17
+// slice XP). The build that applies it is pinned by
+// bin/compose-site.postsVisibility.test.ts.
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ isXPostsChannel,
+ postsVisibleTo,
+ publishedMemberSlugs,
+ xPostsVisibility,
+} from "./postsVisibility";
+import { isListedSite, parseSite, siteToDisk } from "./siteSchema";
+import { sanitizeSocial } from "../social/xCookieSource";
+
+const X = { sourceKind: "social" as const, platform: "twitter" as const };
+const BLUESKY = { sourceKind: "social" as const, platform: "bluesky" as const };
+const VIDEOS = { platform: "youtube" as const };
+const PRIVATE_X = { social: { x: { visibility: "private" } } };
+const PUBLIC_X = { social: { x: { visibility: "public" } } };
+const publicSite = { audience: undefined };
+const privateSite = { audience: "private" as const };
+
+test("an X channel is a social channel on platform twitter, and nothing else is", () => {
+ assert.equal(isXPostsChannel(X), true);
+ assert.equal(isXPostsChannel(BLUESKY), false);
+ assert.equal(isXPostsChannel(VIDEOS), false);
+ // A video channel on X (were there one) is not a posts channel.
+ assert.equal(isXPostsChannel({ platform: "twitter" }), false);
+ assert.equal(isXPostsChannel(null), false);
+ assert.equal(isXPostsChannel(undefined), false);
+});
+
+test("social.x.visibility: absent and unknown read as public", () => {
+ assert.equal(xPostsVisibility({}), "public");
+ assert.equal(xPostsVisibility({ social: { x: {} } }), "public");
+ assert.equal(xPostsVisibility({ social: { x: { visibility: "hidden" } } }), "public");
+ assert.equal(xPostsVisibility(PUBLIC_X), "public");
+ assert.equal(xPostsVisibility(PRIVATE_X), "private");
+});
+
+test("public: X posts go wherever the channel is a member", () => {
+ for (const settings of [{}, PUBLIC_X]) {
+ assert.equal(postsVisibleTo(publicSite, X, settings), true);
+ assert.equal(postsVisibleTo(privateSite, X, settings), true);
+ }
+});
+
+test("private: X posts go to private sites only; every other channel is untouched", () => {
+ assert.equal(postsVisibleTo(publicSite, X, PRIVATE_X), false);
+ assert.equal(postsVisibleTo({}, X, PRIVATE_X), false);
+ assert.equal(postsVisibleTo(privateSite, X, PRIVATE_X), true);
+ for (const site of [publicSite, privateSite]) {
+ assert.equal(postsVisibleTo(site, BLUESKY, PRIVATE_X), true);
+ assert.equal(postsVisibleTo(site, VIDEOS, PRIVATE_X), true);
+ assert.equal(postsVisibleTo(site, null, PRIVATE_X), true);
+ }
+});
+
+test("publishedMemberSlugs narrows the membership, in its order", () => {
+ const configs: Record<string, object> = { vids: VIDEOS, x: X, sky: BLUESKY };
+ const channels = [{ slug: "x" }, { slug: "vids" }, { slug: "sky" }, { slug: "gone" }];
+ const configOf = (slug: string) => configs[slug] as typeof X | undefined;
+ assert.deepEqual(
+ publishedMemberSlugs({ channels }, configOf, PRIVATE_X),
+ ["vids", "sky", "gone"],
+ );
+ assert.deepEqual(
+ publishedMemberSlugs({ channels, audience: "private" }, configOf, PRIVATE_X),
+ ["x", "vids", "sky", "gone"],
+ );
+ assert.deepEqual(
+ publishedMemberSlugs({ channels }, configOf, {}),
+ ["x", "vids", "sky", "gone"],
+ );
+});
+
+test("site.json audience: only private is kept and written; a private site is never listed", () => {
+ assert.equal(parseSite("a", {}).audience, undefined);
+ assert.equal(parseSite("a", { audience: "public" }).audience, undefined);
+ assert.equal(parseSite("a", { audience: "secret" }).audience, undefined);
+ const priv = parseSite("a", { audience: "private" });
+ assert.equal(priv.audience, "private");
+ assert.equal(siteToDisk(priv).audience, "private");
+ assert.equal("audience" in siteToDisk(parseSite("a", {})), false);
+ assert.equal(isListedSite(priv), false);
+ assert.equal(isListedSite({ ...priv, listed: true }), false);
+ assert.equal(isListedSite(parseSite("a", {})), true);
+});
+
+test("sanitizeSocial keeps visibility beside cookieSource and drops an unknown one", () => {
+ assert.deepEqual(sanitizeSocial({ x: { visibility: "private" } }), {
+ x: { visibility: "private" },
+ });
+ assert.deepEqual(
+ sanitizeSocial({ x: { cookieSource: "browser", visibility: "public" } }),
+ { x: { cookieSource: "browser", visibility: "public" } },
+ );
+ assert.deepEqual(sanitizeSocial({ x: { visibility: "nobody" } }), { x: {} });
+});
diff --git a/common/lib/postsVisibility.ts b/common/lib/postsVisibility.ts
@@ -0,0 +1,75 @@
+// WHICH SITES A CHANNEL'S POSTS ARE BUILT INTO (release 17 slice XP).
+//
+// THE ONE RULE, asked by both places a site's posts are decided:
+// - the index build's per-site loop (controller/buildIndex.ts), which writes
+// each site's summaries, subs, posts and digests manifests from the site's
+// member channels;
+// - the site compose (bin/compose-site.ts), which copies the shared
+// per-channel trees (transcripts, subs, posts, digests) of those members
+// into the served public dir and writes /site.json and /corpus.json.
+// Both narrow the site's members through `publishedMemberSlugs`, so the
+// manifests and the trees can never disagree about a channel.
+//
+// The rule: with `social.x.visibility` "private", an X channel (a social
+// channel on platform "twitter") is built only into a PRIVATE site
+// (`site.json` `audience: "private"`). Posts are all a social channel holds — it
+// has no videos (buildIndex's scan skips it) — so a public site leaves the
+// channel out whole: no posts tree, no posts-manifest entry, no empty channel
+// in its channel list or its corpus.json. Every other channel, and every
+// channel on a private site, is unaffected. Nothing on disk changes and
+// fetching does not: the shared posts tree (exportSharedPostsDir) and the LMDB
+// posts sub-DB are corpus-wide and keep every channel, which is what a private
+// site and the MCP over a private build read.
+//
+// The Search in row's Posts toggle needs no rule of its own: it shows only
+// when the site's posts manifest lists a channel (SearchSessionContext
+// hasPostsCorpus), so a public site whose only posts were X posts ships an
+// empty posts manifest and no Posts toggle.
+//
+// Pure: no fs, no settings read. The callers pass the settings and the configs.
+
+import { isSocialChannel, type ChannelConfig } from "./channelConfig";
+import { isPrivateSite, type Site } from "./siteSchema";
+import type { XPostsVisibility } from "../social/xCookieSource";
+
+// An X channel: the posts of a social channel whose platform is X.
+export function isXPostsChannel(
+ config: Pick<ChannelConfig, "sourceKind" | "platform"> | null | undefined,
+): boolean {
+ return isSocialChannel(config) && config?.platform === "twitter";
+}
+
+// `social.x.visibility`, resolved: absent (or anything unknown) is "public".
+export function xPostsVisibility(settings: {
+ social?: { x?: { visibility?: unknown } };
+}): XPostsVisibility {
+ return settings.social?.x?.visibility === "private" ? "private" : "public";
+}
+
+// Whether `site` may carry the posts of the channel `config` describes. A
+// channel with no readable config is not an X channel as far as this rule
+// knows, and is left to whatever already decides its fate.
+export function postsVisibleTo(
+ site: Pick<Site, "audience">,
+ config: Pick<ChannelConfig, "sourceKind" | "platform"> | null | undefined,
+ settings: { social?: { x?: { visibility?: unknown } } },
+): boolean {
+ if (!isXPostsChannel(config)) return true;
+ if (xPostsVisibility(settings) === "public") return true;
+ return isPrivateSite(site);
+}
+
+// The site's member slugs, in membership order, less the channels whose posts
+// it may not carry (an X channel holds nothing else). `configOf` answers a
+// slug's channel config, or null/undefined when it has none.
+export function publishedMemberSlugs(
+ site: Pick<Site, "audience" | "channels">,
+ configOf: (
+ slug: string,
+ ) => Pick<ChannelConfig, "sourceKind" | "platform"> | null | undefined,
+ settings: { social?: { x?: { visibility?: unknown } } },
+): string[] {
+ return site.channels
+ .map((c) => c.slug)
+ .filter((slug) => postsVisibleTo(site, configOf(slug), settings));
+}
diff --git a/common/lib/settingsSchema.ts b/common/lib/settingsSchema.ts
@@ -116,6 +116,15 @@ export const X_SOCIAL_SETTINGS_FIELD_DOCS: FieldDocs<XSocialSettings> = {
"stored: `\"browser\"` when `cookiesFromBrowser` is set and no profile is connected (no " +
"exported jar carrying an auth_token), else `\"profile\"`. The source, not `cookieMode`, " +
"governs the X fetchers.",
+ visibility:
+ "Where X posts may appear. `\"public\"` (the default; absent): an X channel's posts are " +
+ "built into every site that has the channel. `\"private\"`: every X channel's posts (a " +
+ "channel with `sourceKind: \"social\"` and `platform: \"twitter\"`) are left out of every " +
+ "PUBLIC site build — the channel with them, since posts are all an X channel holds — and " +
+ "built only into PRIVATE sites (`site.json` `audience`). Nothing on disk changes and " +
+ "fetching does not; a site already published changes on its next build and deploy, and " +
+ "flipping back is a rebuild. Chosen in the X account session section of /settings; the " +
+ "rule is common/lib/postsVisibility.ts.",
};
export type { AutoQueueSettings } from "./autoQueueTypes";
export type { ChannelPriority } from "./channelPriority";
@@ -1477,7 +1486,7 @@ export const siteSettingsSchema = z.object({
"How yt-dlp invocations use the configured cookies (see common/lib/cookiePolicy.ts): \"always\" passes them on every invocation, \"when-required\" (default; the historical behavior) only to retry an auth/age failure, \"defer\" never in normal runs — auth-gated videos are excluded from batches and collected into the per-channel \"Needs cookies\" bucket for a manual cookie run. Per-channel override available (ChannelConfig.cookieMode).",
),
social: settingsField((v): SocialSettings => sanitizeSocial(v)).describe(
- "Per-platform settings of the social-post fetchers. Today one key: where the X fetchers' login comes from (`social.x.cookieSource`, chosen in the X account session section of /settings). See common/social/xCookieSource.ts.",
+ "Per-platform settings of the social posts. Today two keys, both X's, both chosen in the X account session section of /settings: where the X fetchers' login comes from (`social.x.cookieSource`) and where X posts may appear (`social.x.visibility`). See common/social/xCookieSource.ts.",
),
sleepBetweenDownloadsSeconds: settingsField((v): number => clampSleepBetweenDownloadsSeconds(v)).describe(
"Pause (seconds) inserted between per-video yt-dlp invocations in managed batch downloads. yt-dlp's own `-t sleep` only paces requests within one invocation, so without this the managed loop hammers the source IP back-to-back. 0 disables. Per-channel override available.",
diff --git a/common/lib/site.ts b/common/lib/site.ts
@@ -14,6 +14,7 @@ import { socialLinksForSave } from "./socialLinks";
import { readJsonFileSync, writeJsonAtomic } from "./jsonFile-server";
import {
isListedSite,
+ isPrivateSite,
isValidSiteId,
parseSite,
parseSiteUrl,
@@ -87,11 +88,14 @@ export function siteStatsDir(paths: Paths, siteId: string): string {
}
// The hub URL this site points visitors toward: its own override, else the
-// family default (SiteSettings.homepageUrl). Undefined when neither is set.
+// family default (SiteSettings.homepageUrl). Undefined when neither is set —
+// and always for a PRIVATE site (`audience: "private"`), which belongs under no
+// hub: a hub tells its members by the hubUrl they publish.
export function resolveHubUrl(
site: Site,
settings: SiteSettings = getSettings(),
): string | undefined {
+ if (isPrivateSite(site)) return undefined;
return site.hubUrl ?? parseSiteUrl(settings.homepageUrl);
}
diff --git a/common/lib/siteSchema.ts b/common/lib/siteSchema.ts
@@ -68,6 +68,27 @@ export const RELATED_SITE_GROUP_FIELD_DOCS: FieldDocs<RelatedSiteGroup> = {
"Sibling site ids, in display order. Invalid and repeated ids are dropped, and a group left with none is dropped. Ids are resolved against the live pool at render time, so an id for a site that does not exist (yet) is harmless — it is skipped.",
};
+// Who a site is built for (release 17 slice XP). "public" (the default, never
+// written) is every site there has ever been. "private" is the operator's own
+// reading copy: never deployed (publish/build.ts asks siteDeployProblem in
+// lib/builtExport.ts before any upload), never listed (isListedSite below),
+// publishing no hubUrl (lib/site.ts resolveHubUrl), and the only kind of site
+// that content kept from the public (X posts while `social.x.visibility` is
+// "private", lib/postsVisibility.ts) is built into.
+export type SiteAudience = "public" | "private";
+
+export const SITE_AUDIENCES: readonly SiteAudience[] = ["public", "private"];
+
+export function isSiteAudience(v: unknown): v is SiteAudience {
+ return v === "public" || v === "private";
+}
+
+// THE ONE PREDICATE for a private site. Absent or anything but "private" reads
+// as public.
+export function isPrivateSite(site: Pick<Site, "audience">): boolean {
+ return site.audience === "private";
+}
+
// A Site is a selection + presentation layer over the single global channel
// pool. Each field is documented in SITE_FIELD_DOCS below.
export type Site = {
@@ -85,6 +106,7 @@ export type Site = {
accent?: string;
siteUrl?: string;
listed?: boolean;
+ audience?: SiteAudience;
relatedSites?: RelatedSiteGroup[];
pwa?: boolean;
archives?: boolean;
@@ -119,6 +141,8 @@ export const SITE_FIELD_DOCS: FieldDocs<Site> = {
"Absolute public URL of this site's deployment, e.g. `https://jeralyzer.pages.dev` (trimmed, trailing slashes removed; anything not absolute http(s) is dropped). Drives the cross-site footer: a site with no siteUrl is omitted from every other site's list.",
listed:
"Whether the family lists this site. Opt-OUT: absent/true = listed, only an explicit `false` is written. An unlisted site still builds and deploys as before, and its own pages are unchanged; it is left out of the homepage (cards, chart, `/stats`), the hub (members, federated search, `/corpus.json`, `/llms.txt`), every other site's footer, and the published `channel-sites.json` and pooled `stats/`. A channel only unlisted sites expose is in none of the family's public totals; a channel a listed site also exposes is credited to the listed one.",
+ audience:
+ 'Who this site is built for. `"public"` (the default; absent) or `"private"`: the operator\'s own reading copy, built on this machine and never deployed — every deploy path (Build & deploy, Deploy, `archilyzer deploy site`, Build & deploy all, docker/publish-site.sh) refuses it before any upload, while a build without a deploy still works. A private site is never listed (as `listed: false`, whatever `listed` says), publishes no `hubUrl`, and its `/corpus.json` says `"audience": "private"`. Content kept from the public — X posts while `social.x.visibility` is `"private"` — is built only into private sites. Only `"private"` is written.',
relatedSites:
"Pulls specific siblings to the front of the footer's cross-site list, in named groups. Siblings not named here fall into a trailing \"Other sites\" group. Absent/empty = one flat list of every sibling.",
pwa:
@@ -150,8 +174,11 @@ export function isValidSiteId(id: unknown): id is string {
// (lib/site.ts resolveRelatedSites). The editor's own pages list every site.
// Here, beside the key, and exported from lib/site like isValidSiteId, so the
// pure summary builder can use it without importing file I/O.
-export function isListedSite(site: Pick<Site, "listed">): boolean {
- return site.listed !== false;
+//
+// A PRIVATE site (`audience: "private"`) is never listed, whatever `listed`
+// says: it is never deployed, so there is nothing at its URL to list.
+export function isListedSite(site: Pick<Site, "listed" | "audience">): boolean {
+ return site.listed !== false && !isPrivateSite(site);
}
// The channels whose content belongs to unlisted sites alone: exposed by at
@@ -160,7 +187,7 @@ export function isListedSite(site: Pick<Site, "listed">): boolean {
// and a channel no site exposes (pool-only) is not here either — the family's
// instance-wide totals have always counted it.
export function channelsOnlyOnUnlistedSites(
- sites: readonly Pick<Site, "listed" | "channels">[],
+ sites: readonly Pick<Site, "listed" | "audience" | "channels">[],
): Set<string> {
const onListed = new Set<string>();
const onUnlisted = new Set<string>();
@@ -281,6 +308,10 @@ export const siteFieldsSchema = z.object({
siteUrl: settingsField(parseSiteUrl).describe(d.siteUrl),
// Opt-out: only an explicit false unlists. Absent/true stays listed.
listed: settingsField((v): boolean => v !== false).describe(d.listed),
+ // Only "private" is kept; absent (and anything else) is the public default.
+ audience: settingsField((v): SiteAudience | undefined =>
+ v === "private" ? "private" : undefined,
+ ).describe(d.audience),
relatedSites: settingsField(parseRelatedSites).describe(d.relatedSites),
pwa: settingsField((v): boolean => v === true).describe(d.pwa),
// Opt-out: only an explicit false disables. Absent/true stays on.
@@ -372,6 +403,8 @@ export function siteToDisk(site: Site): Site {
...(siteUrl ? { siteUrl } : {}),
// Listed is the default: only the opt-out is persisted.
...(site.listed === false ? { listed: false } : {}),
+ // Public is the default: only the private audience is persisted.
+ ...(isPrivateSite(site) ? { audience: "private" as const } : {}),
...(relatedSites.length > 0 ? { relatedSites } : {}),
...(site.pwa ? { pwa: true } : {}),
// Persist only the non-default: archives is on unless explicitly disabled.
diff --git a/common/publish/build.test.ts b/common/publish/build.test.ts
@@ -23,6 +23,7 @@ import {
homepageDeployArgs,
homepageOutDir,
deployHomepage,
+ deploySite,
dockerSiteOutDir,
dockerSiteStagingDir,
resolveOutDir,
@@ -393,3 +394,120 @@ test("runDeployIntoLog refuses a bundle that is not the site's own before wrangl
rmSync(root, { recursive: true, force: true });
}
});
+
+// A PRIVATE site (site.json `audience: "private"`, release 17 slice XP) is never
+// deployed, and neither is a bundle built private (its corpus.json says so):
+// refused at the same door as the wrong-site bundle, before wrangler, in words
+// naming the audience. The fake `pnpm` is the test above's.
+function writePrivateBundle(dir: string, siteId: string): void {
+ writeBundle(dir, siteId, siteId);
+ writeFileSync(
+ path.join(dir, "corpus.json"),
+ JSON.stringify({ site: { id: siteId, audience: "private" } }),
+ );
+}
+
+test("runDeployIntoLog refuses a private site and a private build before wrangler", async () => {
+ const root = mkdtempSync(path.join(os.tmpdir(), "deploy-private-"));
+ const bin = path.join(root, "bin");
+ const argvFile = path.join(root, "pnpm-argv");
+ mkdirSync(bin);
+ writeFileSync(path.join(bin, "pnpm"), `#!/bin/sh\nprintf '%s\\n' "$@" >> '${argvFile}'\n`);
+ chmodSync(path.join(bin, "pnpm"), 0o755);
+ const savedPath = process.env.PATH;
+ process.env.PATH = bin;
+ const signal = new AbortController().signal;
+ const testPaths = { ...paths, exportDir: root } as Paths;
+ try {
+ await runChildIntoLog(() => {}, signal, { command: "pnpm", args: ["--fake?"], cwd: root, env: { ...process.env } });
+ assert.equal(readFileSync(argvFile, "utf8"), "--fake?\n");
+ rmSync(argvFile);
+
+ // The site is private: its own, well-formed bundle is still refused.
+ const own = path.join(root, "own", "out");
+ writeBundle(own, "mine", "mine");
+ const priv = { siteId: "mine", cloudflareProject: "w3c-never-real", audience: "private" } as Site;
+ let log: string[] = [];
+ assert.equal(await runDeployIntoLog((l) => log.push(l), signal, priv, own, testPaths), 1);
+ assert.equal(log.length, 1, log.join(""));
+ assert.match(
+ log[0],
+ /^\[deploy\] REFUSED — Site "mine" is private \(audience: private\): it is built for reading on this machine and is never deployed\./,
+ );
+ assert.match(log[0], /Nothing was sent to Cloudflare Pages\.\n$/);
+
+ // The site is public now, but the bundle was built private.
+ const built = path.join(root, "built", "out");
+ writePrivateBundle(built, "mine");
+ log = [];
+ const pub = { siteId: "mine", cloudflareProject: "w3c-never-real" } as Site;
+ assert.equal(await runDeployIntoLog((l) => log.push(l), signal, pub, built, testPaths), 1);
+ assert.match(log[0], /holds a private build of "mine" \(its corpus\.json says "audience": "private"\)/);
+ assert.equal(existsSync(argvFile), false, "pnpm was spawned");
+ } finally {
+ process.env.PATH = savedPath;
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("runDockerDeployAllPhase skips a private site before the upload, in the audience's words", async () => {
+ const root = mkdtempSync(path.join(os.tmpdir(), "deploy-all-private-"));
+ try {
+ const outFor = (id: string) => path.join(root, id, "out");
+ writeBundle(outFor("mine"), "mine", "mine");
+ writePrivateBundle(outFor("built"), "built");
+ const log: string[] = [];
+ // A Cloudflare project on each: without the audience check the run would
+ // reach the upload, which the log would show.
+ const sites = [
+ { siteId: "mine", audience: "private", cloudflareProject: "w3c-never-real" },
+ { siteId: "built", cloudflareProject: "w3c-never-real" },
+ ] as Site[];
+ const outcomes = await runDockerDeployAllPhase(
+ (l) => log.push(l),
+ new AbortController().signal,
+ sites,
+ new Set(["mine", "built"]),
+ { ...paths, exportBuildsDir: root } as Paths,
+ outFor,
+ );
+ assert.deepEqual(outcomes.map((o) => [o.siteId, o.status]), [["mine", "skipped"], ["built", "skipped"]]);
+ assert.match(outcomes[0].reason!, /^Site "mine" is private \(audience: private\)/);
+ assert.match(outcomes[1].reason!, /private build of "built"/);
+ assert.ok(log.some((l) => l.startsWith("[mine] deploy skipped — Site \"mine\" is private")), log.join("\n"));
+ assert.ok(!log.some((l) => l.startsWith("=== Deploy")), "nothing reached the deploy");
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("deploySite (archilyzer deploy site) refuses a private site before anything, and a private build before the upload", async () => {
+ const root = mkdtempSync(path.join(os.tmpdir(), "deploy-site-private-"));
+ try {
+ const sitesDir = path.join(root, "sites");
+ const site = (id: string, extra: Record<string, unknown> = {}) => {
+ mkdirSync(path.join(sitesDir, id), { recursive: true });
+ writeFileSync(
+ path.join(sitesDir, id, "site.json"),
+ JSON.stringify({ siteId: id, cloudflareProject: "w3c-never-real", ...extra }),
+ );
+ };
+ site("mine", { audience: "private" });
+ site("other");
+ const testPaths = { ...paths, exportDir: root, sitesDir } as Paths;
+ const log: string[] = [];
+ await assert.rejects(
+ deploySite("mine", { paths: testPaths, onLog: (l) => log.push(l) }),
+ /^Error: Site "mine" is private \(audience: private\): it is built for reading on this machine and is never deployed\. Build it without deploying, or set its audience to public on its Settings tab\.$/,
+ );
+ // Public, but export/out holds a private build of it.
+ writePrivateBundle(path.join(root, "out"), "other");
+ await assert.rejects(
+ deploySite("other", { paths: testPaths, onLog: (l) => log.push(l) }),
+ /private build of "other".*Build other again, then deploy\.$/,
+ );
+ assert.deepEqual(log, [], "nothing was logged: no upload, no deploy");
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
diff --git a/common/publish/build.ts b/common/publish/build.ts
@@ -14,7 +14,14 @@ import { createReadStream, existsSync } from "node:fs";
import { S3Client, HeadObjectCommand } from "@aws-sdk/client-s3";
import { Upload } from "@aws-sdk/lib-storage";
import { runChildIntoLog } from "../jobs/runChild";
-import { builtBundleProblem, builtHubProblem, builtSiteProblem } from "../lib/builtExport";
+import {
+ builtAudienceProblem,
+ builtBundleProblem,
+ builtHubProblem,
+ builtSiteProblem,
+ deployAudienceProblem,
+ siteDeployProblem,
+} from "../lib/builtExport";
import { getHomepageConfig } from "../lib/homepage";
import {
deploymentUrlIn,
@@ -340,6 +347,16 @@ export async function runDeployIntoLog(
// job (a build of another site, the hub) can rewrite export/out. Nothing runs
// between this check and the spawn. The hub and the homepage deploy through
// runPagesDeployIntoLog and never come here.
+ //
+ // A PRIVATE site, or a bundle built private, is refused first: it is never
+ // deployed, whatever the bundle's identity (deployAudienceProblem).
+ const audienceProblem = deployAudienceProblem(site, outDir);
+ if (audienceProblem) {
+ onLog(
+ `[deploy] REFUSED — ${audienceProblem}. Nothing was sent to Cloudflare Pages.\n`,
+ );
+ return 1;
+ }
const bundleProblem = builtBundleProblem(outDir, site.siteId);
if (bundleProblem) {
onLog(
@@ -604,10 +621,21 @@ export async function runDockerDeployAllPhase(
outcomes.push({ siteId: site.siteId, status: "skipped", reason: "build failed" });
continue;
}
- // The bundle must be this site's own before anything else is asked of it —
- // the check build-site.sh makes before it hands the bundle back, made again
- // over whatever the per-site dir holds now. First, so that nothing past it
- // (the R2 upload, the Pages deploy) is ever reached with another site's data.
+ // A private site (or a private build) is not deployed at all: skipped, in
+ // its own words, so a family with one private site does not fail every run.
+ // Asked first; a per-site dir holding another site's bundle built private
+ // therefore reads "skipped" rather than "REFUSED" — never shipped either way.
+ //
+ // Then the bundle must be this site's own — the check build-site.sh makes
+ // before it hands the bundle back, made again over whatever the per-site dir
+ // holds now — before anything past it (the R2 upload, the Pages deploy) is
+ // reached with another site's data.
+ const audienceProblem = deployAudienceProblem(site, outDirFor(site.siteId));
+ if (audienceProblem) {
+ onLog(`[${site.siteId}] deploy skipped — ${audienceProblem}`);
+ outcomes.push({ siteId: site.siteId, status: "skipped", reason: audienceProblem });
+ continue;
+ }
const bundleProblem = builtBundleProblem(outDirFor(site.siteId), site.siteId);
if (bundleProblem) {
onLog(`[${site.siteId}] deploy REFUSED — ${bundleProblem}`);
@@ -753,6 +781,9 @@ export async function deploySite(
}
const branch = opts.previewBranch?.trim() || undefined;
const site = getSite(siteId.trim(), paths);
+ // Before anything else is asked of a private site: it is never deployed.
+ const privateProblem = siteDeployProblem(site);
+ if (privateProblem) throw new Error(`${privateProblem}.`);
if (!site.cloudflareProject) {
throw new Error(
`Site "${site.siteId}" has no Cloudflare Pages project configured.`,
@@ -761,6 +792,9 @@ export async function deploySite(
const outDir = resolveOutDir(site.siteId, paths);
const builtProblem = builtSiteProblem(outDir, site.siteId);
if (builtProblem) throw new Error(builtProblem);
+ // Before the R2 upload below: a bundle built private is never deployed.
+ const builtPrivate = builtAudienceProblem(outDir);
+ if (builtPrivate) throw new Error(`${builtPrivate}. Build ${site.siteId} again, then deploy.`);
// The production path logs no banner and gains none here: its log has
// always opened on wrangler's own first line.
if (branch) {
@@ -892,7 +926,9 @@ export async function composeHub(opts: PublishOpts = {}): Promise<number> {
/**
* Build the hub into export/out. Removes public/site.json first — a site's
* compose left it there, and a hub bundle carrying one would read as that
- * site's (builtExport.ts). Returns the exit code.
+ * site's (builtExport.ts). compose-hub then removes every other per-site entry
+ * a site's compose left in public/ (SITE_ONLY_PUBLIC_ENTRIES), so the hub
+ * never ships a site's data. Returns the exit code.
*/
export async function buildHub(opts: PublishOpts = {}): Promise<number> {
const { paths, onLog, signal } = resolved(opts);
diff --git a/common/social/xCookieSource.ts b/common/social/xCookieSource.ts
@@ -27,11 +27,28 @@ export function isXCookieSource(v: unknown): v is XCookieSource {
return v === "browser" || v === "profile";
}
-// The `social` block of settings.json. Only X has a login to choose today; the
-// block is per platform so a second one does not need a second top-level key.
+// WHERE X POSTS MAY APPEAR — `social.x.visibility` (release 17 slice XP).
+// "public" — the default (absent): an X channel's posts are built into every
+// site that has the channel, as every other channel's are.
+// "private" — an X channel's posts are left out of every PUBLIC site build and
+// built only into PRIVATE sites (`site.json` `audience`). Nothing
+// on disk changes, and fetching does not; flipping back is a
+// rebuild. The rule itself is lib/postsVisibility.ts.
+export type XPostsVisibility = "public" | "private";
+
+export const X_POSTS_VISIBILITIES: readonly XPostsVisibility[] = ["public", "private"];
+
+export function isXPostsVisibility(v: unknown): v is XPostsVisibility {
+ return v === "public" || v === "private";
+}
+
+// The `social` block of settings.json. Only X has settings today; the block is
+// per platform so a second one does not need a second top-level key.
export type XSocialSettings = {
// Absent = the read-time default above.
cookieSource?: XCookieSource;
+ // Absent = "public".
+ visibility?: XPostsVisibility;
};
export type SocialSettings = {
@@ -39,7 +56,8 @@ export type SocialSettings = {
};
// Total over `unknown`, as every settings coercion is: anything that is not a
-// known source reads as absent (the default), and unknown keys are dropped.
+// known source or visibility reads as absent (the default), and unknown keys
+// are dropped.
export function sanitizeSocial(value: unknown): SocialSettings {
const r = (value && typeof value === "object" && !Array.isArray(value)
? value
@@ -48,7 +66,10 @@ export function sanitizeSocial(value: unknown): SocialSettings {
? r.x
: {}) as Record<string, unknown>;
return {
- x: isXCookieSource(x.cookieSource) ? { cookieSource: x.cookieSource } : {},
+ x: {
+ ...(isXCookieSource(x.cookieSource) ? { cookieSource: x.cookieSource } : {}),
+ ...(isXPostsVisibility(x.visibility) ? { visibility: x.visibility } : {}),
+ },
};
}
diff --git a/docker/publish-site.sh b/docker/publish-site.sh
@@ -29,6 +29,20 @@ if [ -z "${SITE_ID}" ]; then
fi
export SITE_ID
+
+# A PRIVATE site (site.json `audience: "private"`) is never deployed, and the
+# volume this script fills is what the `site` service serves: refused before
+# the build, and again over the built corpus.json below (lib/builtExport.ts).
+# Building it without publishing is `archilyzer build site <id>`.
+SITE_JSON="${SITES_DIR:-${TRANSCRIPTS_DIR:-/data/transcripts}/sites}/${SITE_ID}/site.json"
+refuse_private() {
+ echo "[publish-site] REFUSED — site '${SITE_ID}' is private (audience: private): it is built for reading on this machine and is never deployed. Nothing was published to ${SITE_OUT}. Build it without publishing: pnpm --filter yt-dlp-transcript-common exec tsx bin/archilyzer.ts build site ${SITE_ID}" >&2
+ exit 1
+}
+if grep -Eq '"audience"[[:space:]]*:[[:space:]]*"private"' "${SITE_JSON}" 2>/dev/null; then
+ refuse_private
+fi
+
cd /repo
echo "[publish-site] building '${SITE_ID}'"
@@ -37,6 +51,9 @@ echo "[publish-site] building '${SITE_ID}'"
pnpm --filter yt-dlp-transcript-common exec tsx bin/archilyzer.ts build site "${SITE_ID}"
[ -d /repo/export/out ] || { echo "[publish-site] no export/out after build" >&2; exit 1; }
+if grep -Eq '"audience"[[:space:]]*:[[:space:]]*"private"' /repo/export/out/corpus.json 2>/dev/null; then
+ refuse_private
+fi
echo "[publish-site] publishing -> ${SITE_OUT}"
mkdir -p "${SITE_OUT}"
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -23,6 +23,10 @@
- **A form whose save is refused keeps what you typed.** Every editor form put its plain fields back to the stored values when its save was refused — a site's ID rejected, a page size out of range, a slug already taken — so everything typed had to be typed again. A refused save now leaves every field as you left it, beside the reason: **Settings**; a site's form (new and existing); the hub's config on `/sites`; **Cut release**; a channel's form (new and **Configure**), **Rename** and **Delete**; a video's **Delete directory**; **Drive health timing** on `/storage`; the backup config on `/saved-videos`; the sync operation's controls; the **Digest**, **Diarization**, **Speaker attribution** and **Speaker work lane** settings; and the worker list on `/workers`. A save that succeeds behaves as before, with one difference you may notice: a drop-down, and a checkbox or choice that the page tracks as you change it (a cadence, a worker's **Enabled**, a social link's **Keep in header**, a site membership, a site's accent), now shows what was saved. A form's own drop-downs used to go back to what the page had loaded with until a reload, and a second save from the same page sent that old choice again; the others went back until the page next refreshed itself (every 5 seconds by default).
- **A media move no longer starts over a job that is writing into the channel, holds the channel's writers while it runs, and makes its copy match the source before it verifies — so a transcription or a download during a move cannot fail it.** A move that has waited its turn behind other moves now checks again when it starts: if a job is running on the channel, or an auto-queue lane is working on one of its videos, it stops at once and says which ("a transcription of abc123 is running (Transcribe all, job …) — wait for it or cancel it"), with nothing copied — and a job you have just cancelled counts until it has actually stopped ("is stopping … — wait for it to stop"); **Preview** says the same, and the Storage panel's blocked message now names the job too. While a move's marker stands, the channel is held: every lane skips it, and every job that reads or writes its media (single-video transcriptions, downloads and transcodes and the availability checks now included) refuses to start, including one that was already queued when the move began. The rack shows a **media held** chip in the channel's Tier cell and the Storage panel says "Held: its media is moving"; both go when the move finishes or its marker is cleared. The copy is now followed by a pass that makes the destination copy match the source — files the source no longer has are removed from the copy, never from the source — so a file written or deleted during the copy (a transcriber's scratch folder, say) no longer fails the check, and **Resume move** finishes a move whose copy holds such leftovers. Every file removed from a copy is listed in the move's log, and **Preview** says so when a copy from an earlier attempt is already there. If the source keeps changing, the move stops and lists what differs: extra on the destination, missing there, or changed. A new **Reconcile and resume** button beside **Resume move** lists those differences, makes the copy match and finishes the move, so no file has to be deleted by hand. The saved-video store's move does the same matching and the same check before it starts. Needs a rebuild and restart of the editor.
- **Connecting an X account opens your own browser, and the X fetchers can use your everyday browser's X login instead.** **Settings → X account session → Connect X account** used to open Playwright's bundled Chromium with its automation signals on (the "controlled by automated test software" bar, `navigator.webdriver`): Google's sign-in refused it and X's own login form stalled in it. It now opens your Chromium or Chrome when one is installed (`ARCHILYZER_X_BROWSER` names another; Playwright's bundled Chromium otherwise), without those signals. Google's sign-in may still refuse an embedded browser; X's password login is the reliable path. A new **Login source** choice (`social.x.cookieSource` in `settings.json`) says where the X fetchers' login comes from: **Browser login** hands gallery-dl `--cookies-from-browser` with your `cookiesFromBrowser` on every fetch, so the login lasts as long as you stay logged in to x.com in that browser and no window is needed; **Connected profile** is the session broker, as before. Left on **Automatic**, it is the browser login when `cookiesFromBrowser` is set and no profile is connected, and the profile otherwise. **Check** says which source is in use, whether an X login is visible in it and when it was last used (the browser's cookies are read from a private copy, never written; this reads Firefox's, and gallery-dl reads Chromium's itself). Needs a rebuild and restart of the editor.
+- **X posts can be kept off every public site.** **Settings → X account session** has a new choice, **Where X posts appear** (`social.x.visibility` in `settings.json`): **Public**, the default, builds an X channel's posts into every site that has the channel, as before; **Private** leaves every X channel out of every public site's build — its posts, its posts manifest entry and its place in the channel list, `site.json` and `corpus.json` — and builds it only into private sites (below). Nothing on disk changes and fetching goes on. A site already published changes on its next build and deploy, and a public site whose only posts were X posts loses its **Posts** box under **Search in**. Choosing the X login source no longer forgets this choice, and choosing this one keeps the login source. Needs a rebuild and restart of the editor, then a rebuild and deploy of every site and the hub.
+- **A site can be private: built for reading on this machine, never deployed and never listed.** A site's settings have a new **Audience** choice (`audience` in `site.json`; only `"private"` is written). A private site is refused by every deploy — **Build & deploy**, **Deploy**, `archilyzer deploy site`, `pnpm ops build-deploy` and `deploy-site`, **Build & deploy all** (which builds it and skips its deploy) and `docker/publish-site.sh` — before anything is uploaded, in a sentence naming the audience; **Build** still builds it. It is left off the homepage, the hub and every other site's footer whatever **List on the Archilyzer homepage and hub** says, it publishes no hub URL, and its `corpus.json` says `"audience": "private"`, so a build of it is refused too if the site is switched back to public before it is rebuilt. Point the MCP at a private site's build to ask about what only it holds. Needs a rebuild and restart of the editor.
+- **A site built on the host right after another site no longer ships that site's channel list.** A site's **Build** (and Build all without containers) composes every site into the same folder, and the step that copies a site's summaries, stats and duplicates was skipped when that site's own data had not changed, even though another site had composed there since. The site then went out with the other site's channel list and stats. Those steps now run again whenever another site composed last. Needs a rebuild and restart of the editor.
+- **The hub no longer ships the data of the last site built before it.** **Build hub** built from the folder a site's build had just filled, so the hub carried that site's summaries, transcripts, posts and other data, and served them. The hub's build now clears every site's data first and uses the global search aliases, and **Deploy hub** refuses a hub build that still carries a site's data. Needs a rebuild and restart of the editor, then a rebuild and deploy of the hub.
## [0.11.0] - 2026-09-30
- **Transcripts that arrived after a video was first seen are counted.** The stats behind the homepage, the hub and every site's charts were cached per video and refreshed only when the video's metadata changed, so a transcript that came later — a Whisper run days after the download, or a video downloaded after the last index build — never reached them, and a video with YouTube captions alone had no transcription date. Counts and charts were low; the homepage could show a site with 0 transcripts, 0 channels and 0 hours while it served its videos. A stat is now also redone whenever the index re-reads the video, every transcript has a date, and a captioned video is dated by when its captions arrived rather than by a later Normalize run, so its place on "Transcribed over time" can move. **After updating, rebuild and restart the editor before anything else:** until then, **Build stats dataset** runs the old code and would undo the new stats, while a site, hub or homepage build already runs the new code — and the first stats build of any kind re-reads every video once (about 10–30 minutes on a large archive; it can be stopped and picks up where it stopped). Then build the index, the stats, the homepage, the hub, and the sites.
diff --git a/editor/app/settings/components/XPostsVisibilityControl.tsx b/editor/app/settings/components/XPostsVisibilityControl.tsx
@@ -0,0 +1,74 @@
+"use client";
+
+// Where X posts appear — `social.x.visibility` (release 17 slice XP), inside the
+// X account session section. "Public" builds an X channel's posts into every
+// site that has the channel; "Private" keeps them out of every public site and
+// builds them only into private sites (site.json `audience`). The rule is
+// common/lib/postsVisibility.ts; nothing on disk changes and fetching does not.
+
+import { useEffect, useState, useTransition } from "react";
+import { setXPostsVisibilityAction } from "../xSessionActions";
+import type { XPostsVisibility } from "yt-dlp-transcript-common/social/xCookieSource";
+
+export function XPostsVisibilityControl({
+ initial,
+}: {
+ initial: XPostsVisibility;
+}) {
+ const [visibility, setVisibility] = useState<XPostsVisibility>(initial);
+ const [error, setError] = useState<string | null>(null);
+ const [note, setNote] = useState<string | null>(null);
+ const [saving, startSaving] = useTransition();
+
+ // Another tab's choice re-renders the page with a new value; take it.
+ useEffect(() => setVisibility(initial), [initial]);
+
+ const choose = (choice: string) =>
+ startSaving(async () => {
+ setError(null);
+ setNote(null);
+ const res = await setXPostsVisibilityAction(choice);
+ if (res.ok) {
+ setVisibility(res.visibility);
+ setNote("Saved. Published sites change on their next build and deploy.");
+ } else {
+ setError(res.error);
+ }
+ });
+
+ return (
+ <div data-x-posts-visibility="" className="flex flex-col gap-1">
+ <div className="flex flex-wrap items-center gap-2">
+ <label htmlFor="x-posts-visibility" className="text-sm">
+ Where X posts appear
+ </label>
+ <select
+ id="x-posts-visibility"
+ value={visibility}
+ onChange={(e) => choose(e.target.value)}
+ disabled={saving}
+ className="rounded-md border border-border bg-background px-2 py-1 text-sm disabled:opacity-50"
+ >
+ <option value="public">Public — every site that has the channel</option>
+ <option value="private">Private — private sites only</option>
+ </select>
+ </div>
+ <p className="text-xs text-muted-foreground">
+ Private leaves every X channel and its posts out of every public site
+ and builds them only into sites whose audience is private, which are
+ never deployed; nothing is deleted and fetching goes on. Sites already
+ published change on their next build and deploy.
+ </p>
+ {note && (
+ <p role="status" aria-label="x posts visibility saved" className="text-xs text-success">
+ {note}
+ </p>
+ )}
+ {error && (
+ <p role="alert" className="text-xs text-destructive">
+ {error}
+ </p>
+ )}
+ </div>
+ );
+}
diff --git a/editor/app/settings/components/XSessionSection.tsx b/editor/app/settings/components/XSessionSection.tsx
@@ -26,14 +26,19 @@ import {
resolveXCookieSource,
xCookieSourceLabel,
type XCookieSourceView,
+ type XPostsVisibility,
} from "yt-dlp-transcript-common/social/xCookieSource";
+import { XPostsVisibilityControl } from "./XPostsVisibilityControl";
export function XSessionSection({
initial,
initialSource,
+ initialVisibility,
}: {
initial: XSessionStatus;
initialSource: XCookieSourceView;
+ // `social.x.visibility`, resolved (absent = "public"); release 17 slice XP.
+ initialVisibility: XPostsVisibility;
}) {
const [status, setStatus] = useState<XSessionStatus>(initial);
const [source, setSource] = useState<XCookieSourceView>(initialSource);
@@ -250,6 +255,10 @@ export function XSessionSection({
{error}
</p>
)}
+
+ <div className="border-t border-border pt-3">
+ <XPostsVisibilityControl initial={initialVisibility} />
+ </div>
</section>
);
}
diff --git a/editor/app/settings/page.tsx b/editor/app/settings/page.tsx
@@ -5,6 +5,7 @@ import { getSettings } from "yt-dlp-transcript-common/lib/settings";
import { readXSessionStatus } from "yt-dlp-transcript-common/social/xSessionBroker";
import { resolveXCookieSourceFor } from "yt-dlp-transcript-common/social/xBrowserLogin";
import { xCookieSourceView } from "yt-dlp-transcript-common/social/xCookieSource";
+import { xPostsVisibility } from "yt-dlp-transcript-common/lib/postsVisibility";
import { SettingsForm } from "./components/SettingsForm";
import { XSessionSection } from "./components/XSessionSection";
@@ -93,7 +94,11 @@ export default async function SettingsPage() {
</section>
<section className="flex flex-col gap-3 border-t border-border pt-6">
- <XSessionSection initial={xSession} initialSource={xSource} />
+ <XSessionSection
+ initial={xSession}
+ initialSource={xSource}
+ initialVisibility={xPostsVisibility(settings)}
+ />
</section>
<section className="flex flex-col gap-3 border-t border-border pt-6">
diff --git a/editor/app/settings/xSessionActions.ts b/editor/app/settings/xSessionActions.ts
@@ -30,11 +30,25 @@ import {
} from "yt-dlp-transcript-common/social/xBrowserLogin";
import {
isXCookieSource,
+ isXPostsVisibility,
xCookieSourceView,
type XCookieSourceView,
+ type XPostsVisibility,
+ type XSocialSettings,
} from "yt-dlp-transcript-common/social/xCookieSource";
+import { xPostsVisibility } from "yt-dlp-transcript-common/lib/postsVisibility";
import { saveSettings } from "./saveSettings";
+// `social.x` is ONE value to saveSettings, whose merge is one level deep: a
+// patch naming `social: { x }` replaces the whole X block. So each X choice is
+// written over the block as it is now, with only its own key changed — the
+// login source keeps the visibility, and the visibility keeps the source.
+function socialXPatch(
+ change: (x: XSocialSettings) => XSocialSettings,
+): { social: { x: XSocialSettings } } {
+ return { social: { x: change({ ...getSettings().social.x }) } };
+}
+
// Every session action returns the source in use beside the profile's status:
// connecting or forgetting a profile can move the read-time default.
export type XSessionActionResult =
@@ -124,12 +138,42 @@ export async function setXCookieSourceAction(
return { ok: false, error: `Unknown X login source "${choice}".` };
}
try {
- await saveSettings({
- social: { x: choice === "auto" ? {} : { cookieSource: choice } },
- });
+ await saveSettings(
+ socialXPatch(({ cookieSource: _was, ...rest }) =>
+ choice === "auto" ? rest : { ...rest, cookieSource: choice },
+ ),
+ );
} catch (e) {
return { ok: false, error: (e as Error).message };
}
revalidatePath("/settings");
return { ok: true, source: await sourceNow() };
}
+
+// WHERE X POSTS APPEAR — `social.x.visibility` (release 17 slice XP). "public"
+// is the default and is written as no key; "private" keeps every X channel's
+// posts out of every public site build (common/lib/postsVisibility.ts). Nothing
+// on disk changes and fetching does not: a site already published changes on
+// its next build and deploy.
+export type XPostsVisibilityResult =
+ | { ok: true; visibility: XPostsVisibility }
+ | { ok: false; error: string };
+
+export async function setXPostsVisibilityAction(
+ choice: string,
+): Promise<XPostsVisibilityResult> {
+ if (!isXPostsVisibility(choice)) {
+ return { ok: false, error: `Unknown X post visibility "${choice}".` };
+ }
+ try {
+ await saveSettings(
+ socialXPatch(({ visibility: _was, ...rest }) =>
+ choice === "public" ? rest : { ...rest, visibility: choice },
+ ),
+ );
+ } catch (e) {
+ return { ok: false, error: (e as Error).message };
+ }
+ revalidatePath("/settings");
+ return { ok: true, visibility: xPostsVisibility(getSettings()) };
+}
diff --git a/editor/app/sites/actions.ts b/editor/app/sites/actions.ts
@@ -111,6 +111,12 @@ export async function saveSiteAction(
// Listed on the homepage and hub by default: the same opt-out idiom as
// archives below (an unchecked box sends no key → persisted as false).
const listed = formData.get("listed") === "on";
+ // Who the site is built for (release 17 slice XP): only "private" is kept.
+ const audienceRaw = String(formData.get("audience") ?? "public");
+ if (audienceRaw !== "public" && audienceRaw !== "private") {
+ return { ok: false, error: `Unknown audience "${audienceRaw}".`, values };
+ }
+ const isPrivate = audienceRaw === "private";
// Hub parent (per-site override of the family default) + PWA opt-in.
const hubUrlRaw = String(formData.get("hubUrl") ?? "").trim();
@@ -248,6 +254,7 @@ export async function saveSiteAction(
...(siteUrl ? { siteUrl } : {}),
// The Site is rebuilt from the form: a key missing here is dropped on save.
...(listed ? {} : { listed: false }),
+ ...(isPrivate ? { audience: "private" as const } : {}),
...(hubUrl ? { hubUrl } : {}),
...(pwa ? { pwa: true } : {}),
...(archives ? {} : { archives: false }),
diff --git a/editor/app/sites/components/SiteForm.tsx b/editor/app/sites/components/SiteForm.tsx
@@ -16,7 +16,7 @@ import {
toSocialRow,
type SocialRow,
} from "../../components/SocialLinksField";
-import { Field } from "../../components/forms/Field";
+import { Field, SeededSelect } from "../../components/forms/Field";
import {
ControlledCheck,
ControlledSelect,
@@ -341,6 +341,25 @@ export function SiteForm({ initial, channels, allSites, isNew }: Props) {
defaultValue={initial.siteUrl ?? ""}
hint="Absolute URL this site is served at (e.g. https://jeralyzer.com). Used so other sites can link to it in their footer. Leave blank to omit this site from cross-site lists."
/>
+ <label className="flex flex-col gap-1 text-sm">
+ <span className="font-medium">Audience</span>
+ <SeededSelect
+ state={state}
+ name="audience"
+ aria-label="Audience"
+ initial={initial.audience === "private" ? "private" : "public"}
+ className="w-fit rounded border border-border bg-card px-2 py-1 text-sm"
+ >
+ <option value="public">Public — deployed and listed as configured</option>
+ <option value="private">Private — built for reading on this machine, never deployed</option>
+ </SeededSelect>
+ <span className="text-xs text-muted-foreground">
+ A private site is never deployed (Build & deploy and Deploy refuse
+ it; Build still builds it) and never listed on the homepage or the hub,
+ and publishes no hub URL. It is the one kind of site X posts are built
+ into while Settings keeps X posts private.
+ </span>
+ </label>
<label className="flex items-center gap-2 text-sm">
<input
type="checkbox"
diff --git a/editor/app/sites/lib/buildAction.ts b/editor/app/sites/lib/buildAction.ts
@@ -16,6 +16,10 @@ import {
} from "yt-dlp-transcript-common/controller/archiveLiveChat";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import { previewBranchProblem } from "yt-dlp-transcript-common/lib/pagesDeploy";
+import {
+ deployAudienceProblem,
+ siteDeployProblem,
+} from "yt-dlp-transcript-common/lib/builtExport";
import { getSite, listSites, type Site } from "yt-dlp-transcript-common/lib/site";
import {
runManagedFunction,
@@ -132,6 +136,10 @@ export async function buildAndDeployAction(
}
const branch = previewBranch?.trim();
const site = getSite(id, paths);
+ // A private site is never deployed (site.json `audience`), so Build & deploy
+ // refuses before the build; its Build button still builds it.
+ const privateProblem = siteDeployProblem(site);
+ if (privateProblem) return { ok: false, error: `${privateProblem}.` };
if (!site.cloudflareProject) {
return {
ok: false,
@@ -155,6 +163,14 @@ export async function buildAndDeployAction(
if (buildCode !== 0) {
throw new Error(`Build failed (exit ${buildCode}) — not deploying.`);
}
+ // Asked again before the upload, over the site as it is now and the
+ // bundle just built: an audience switched to private while this job
+ // waited on the queue is not deployed.
+ const audienceProblem = deployAudienceProblem(
+ getSite(id, paths),
+ resolveOutDir(id, paths),
+ );
+ if (audienceProblem) throw new Error(`${audienceProblem} — not deploying.`);
onLog(branch ? `\n=== Deploy (preview "${branch}") ===\n` : "\n=== Deploy ===\n");
if (branch) onLog(PREVIEW_SHARES_ARCHIVES_NOTICE);
// Push oversize archives to R2 before the Pages deploy (no-op when R2
@@ -401,6 +417,13 @@ async function basicBuildAndDeployAll(
});
continue;
}
+ // A private site is built and never deployed — skipped before the upload.
+ const audienceProblem = deployAudienceProblem(site, basicOut);
+ if (audienceProblem) {
+ onLog(`[${site.siteId}] deploy skipped — ${audienceProblem}`);
+ deploys.push({ siteId: site.siteId, status: "skipped", reason: audienceProblem });
+ continue;
+ }
if (!site.cloudflareProject) {
onLog(`[${site.siteId}] deploy skipped — no Cloudflare project configured`);
deploys.push({
diff --git a/editor/app/sites/lib/deployAction.ts b/editor/app/sites/lib/deployAction.ts
@@ -1,6 +1,10 @@
"use server";
-import { builtSiteProblem } from "yt-dlp-transcript-common/lib/builtExport";
+import {
+ builtAudienceProblem,
+ builtSiteProblem,
+ siteDeployProblem,
+} from "yt-dlp-transcript-common/lib/builtExport";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import { previewBranchProblem } from "yt-dlp-transcript-common/lib/pagesDeploy";
import { getSite } from "yt-dlp-transcript-common/lib/site";
@@ -38,6 +42,9 @@ export async function deployExportAction(
}
const branch = previewBranch?.trim();
const site = getSite(siteId.trim(), paths);
+ // A private site is never deployed (site.json `audience`): the first answer.
+ const privateProblem = siteDeployProblem(site);
+ if (privateProblem) return { ok: false, error: `${privateProblem}.` };
if (!site.cloudflareProject) {
return {
ok: false,
@@ -58,6 +65,10 @@ export async function deployExportAction(
const outDir = resolveOutDir(site.siteId, paths);
const builtProblem = builtSiteProblem(outDir, site.siteId);
if (builtProblem) return { ok: false, error: builtProblem };
+ const builtPrivate = builtAudienceProblem(outDir);
+ if (builtPrivate) {
+ return { ok: false, error: `${builtPrivate}. Build ${site.siteId} again, then deploy.` };
+ }
return runManagedFunction({
kind: "deploy-export",
queueKey: DEPLOY_QUEUE,
diff --git a/editor/e2e/sites-crud.spec.ts b/editor/e2e/sites-crud.spec.ts
@@ -528,3 +528,51 @@ test("brand accent radio group + wordmark lead round-trip to site.json", async (
expect("wordmarkLead" in site).toBe(false);
}).toPass({ timeout: 10_000 });
});
+
+// Who a site is built for — `site.json` `audience` (release 17 slice XP). A
+// private site is the operator's own reading copy: saved from the form, and
+// never deployed — the deploy actions refuse it before a job exists, in words
+// naming the audience. (The ops routes call the same actions the Publish tab's
+// buttons do.)
+test("a private site saves its audience, and every deploy refuses it before any job", async ({
+ page,
+ request,
+}) => {
+ await resetData("empty");
+ await writeSite("privsite", { siteTitle: "Private Site", cloudflareProject: "never-real" });
+
+ await page.goto("/sites/privsite");
+ const audience = page.getByLabel("Audience", { exact: true });
+ await expect(audience).toHaveValue("public");
+ await audience.selectOption("private");
+ await page.getByRole("button", { name: /save site/i }).click();
+ await expect(page.getByRole("status").filter({ hasText: "Saved" })).toBeVisible();
+ await expect(async () => {
+ const site = await readJson<{ audience?: string; cloudflareProject?: string }>(
+ "test-transcripts/sites/privsite/site.json",
+ );
+ expect(site.audience).toBe("private");
+ expect(site.cloudflareProject).toBe("never-real");
+ }).toPass({ timeout: 10_000 });
+ await page.reload();
+ await expect(audience).toHaveValue("private");
+
+ const refusal =
+ 'Site "privsite" is private (audience: private): it is built for reading on this machine and is never deployed. Build it without deploying, or set its audience to public on its Settings tab.';
+ for (const action of ["build-deploy", "deploy-site"]) {
+ const res = await request.post(`/api/ops/${action}`, {
+ headers: { authorization: "Bearer test-worker-token" },
+ data: { siteId: "privsite" },
+ });
+ expect(res.status(), action).toBe(400);
+ expect(((await res.json()) as { error?: string }).error, action).toBe(refusal);
+ }
+
+ // Back to public: the key is gone from the file (public is the default).
+ await audience.selectOption("public");
+ await page.getByRole("button", { name: /save site/i }).click();
+ await expect(async () => {
+ const site = await readJson<Record<string, unknown>>("test-transcripts/sites/privsite/site.json");
+ expect("audience" in site).toBe(false);
+ }).toPass({ timeout: 10_000 });
+});
diff --git a/editor/e2e/x-session.spec.ts b/editor/e2e/x-session.spec.ts
@@ -76,3 +76,44 @@ test("the login source select persists, and Check shows a status line", async ({
await expect(inUse).toHaveText(`In use: Browser login (${spec}) (automatic)`);
expect((await readJson<{ social?: unknown }>("test-settings.json")).social).toEqual({ x: {} });
});
+
+// Where X posts appear — `social.x.visibility` (release 17 slice XP). The two X
+// choices share one settings block, and saveSettings replaces a nested block
+// whole, so each is written over the other: choosing one keeps the other.
+test("where X posts appear persists, beside the login source and without it", async ({ page }) => {
+ await resetData("empty");
+ await page.goto("/settings");
+ const visibility = page.getByLabel("Where X posts appear");
+ const source = page.getByLabel("x cookie source", { exact: true });
+ const social = async () =>
+ (await readJson<{ social?: unknown }>("test-settings.json")).social;
+
+ await expect(visibility).toHaveValue("public");
+ await expect(page.locator("[data-x-posts-visibility]")).toContainText(
+ "Sites already published change on their next build and deploy.",
+ );
+
+ await source.selectOption("profile");
+ await expect(page.getByLabel("x cookie source in use")).toHaveText("In use: Connected profile");
+
+ await visibility.selectOption("private");
+ await expect(page.getByLabel("x posts visibility saved")).toHaveText(
+ "Saved. Published sites change on their next build and deploy.",
+ );
+ expect(await social()).toEqual({ x: { cookieSource: "profile", visibility: "private" } });
+
+ await page.reload();
+ await expect(visibility).toHaveValue("private");
+
+ // The login source back to automatic keeps the visibility…
+ await source.selectOption("auto");
+ await expect(page.getByLabel("x cookie source in use")).toHaveText(
+ "In use: Connected profile (automatic)",
+ );
+ expect(await social()).toEqual({ x: { visibility: "private" } });
+
+ // …and public is the default, written as no key.
+ await visibility.selectOption("public");
+ await expect(page.getByLabel("x posts visibility saved")).toBeVisible();
+ expect(await social()).toEqual({ x: {} });
+});
diff --git a/export/CHANGELOG.md b/export/CHANGELOG.md
@@ -1,6 +1,6 @@
# Changelog
-## [Unreleased]
+## [0.11.1] - 2026-10-01
- **Use with AI goes to the Archilyzer site's AI and MCP doc; the page on each site is gone.** The header's, the slide-out menu's, the footer's and Ask AI's **Use with AI** keep their label and open https://archilyzer.pages.dev/docs/ai-and-mcp/ in the same tab, on every site and the hub, where one block says how to run Claude Code against any archive (the source, `pnpm install`, `claude mcp add archilyzer`, `/ask`). `/use-with-ai/` is no longer built. `corpus.json`'s `useWithAi` names the doc; `llms.txt`'s Ask AI section lists the site's `/ask/` chat and the doc; the sitemap drops `/use-with-ai`. Needs a rebuild and deploy of each site and the hub.
- **A search with a layer that has nothing to read finishes.** A "Posts" layer under a tag chip, or a "Live chat" layer where no video in the selection has live chat, read "searched N/M…" for ever and never said "No matching videos."; it now finishes at once, having matched nothing. Needs a rebuild and deploy of each site and the hub.
- **A search reads what the visitor ticks under "Search in": Transcripts, Posts and Live chat.** The Filters panel has a new row, **Search in**, beside Type. **Transcripts** and **Posts** are ticked by default and **Live chat** is not; Posts is offered only on a site that has posts, and Live chat only on a site with live chat. The row decides what a plain query reads: with Posts ticked, a plain query now finds posts as well as videos (before, a post was found only by a layer whose scope was "Posts"); with Live chat ticked, it finds live-chat messages too, shown in the same video's card beside the transcript hits, each marked "live chat"; with Transcripts unticked it reads no transcripts. A layer whose scope is picked by name in the query builder ("Live chat", "Posts", "Title / channel", …) reads what it names, whatever the row says. An empty query still lists every video the Type row keeps. With nothing ticked, Search and Apply filters are disabled and the row says "Search in: pick at least one". The Posts box moved here from the Type row, and unticking it no longer empties a layer whose scope is "Posts". Under a tag chip a plain query reads no posts, since a post carries no tags. The row is remembered, and saved with a profile; a shared link does not carry it, so it opens with the reader's own row. A live-chat hit now wears its "live chat" badge wherever it is shown, and the hint under the search bar says to tick Live chat under Search in. Posts unticked is now also remembered after a reload and restored with a profile, which it was not. Needs a rebuild and deploy of each site and the hub.
diff --git a/export/app/offline/page.tsx b/export/app/offline/page.tsx
@@ -1,6 +1,8 @@
import type { Metadata } from "next";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels";
+import { getSettings } from "yt-dlp-transcript-common/lib/settings";
+import { postsVisibleTo } from "yt-dlp-transcript-common/lib/postsVisibility";
import { currentSite } from "../lib/site";
import { OfflineManager, type OfflineChannel } from "../components/OfflineManager";
@@ -15,12 +17,19 @@ export const metadata: Metadata = {
export default async function OfflinePage() {
const site = currentSite();
const paths = getPaths();
- const channels: OfflineChannel[] = await Promise.all(
- site.channels.map(async ({ slug }) => {
- const config = await readChannelConfig(paths, slug).catch(() => null);
- return { slug, name: config?.name ?? slug };
- }),
+ const settings = getSettings();
+ // The members this build publishes: an X channel is left out of a public
+ // site while X posts are private (lib/postsVisibility.ts, the rule the
+ // index build and compose narrow the site's data by).
+ const members = await Promise.all(
+ site.channels.map(async ({ slug }) => ({
+ slug,
+ config: await readChannelConfig(paths, slug).catch(() => null),
+ })),
);
+ const channels: OfflineChannel[] = members
+ .filter(({ config }) => postsVisibleTo(site, config, settings))
+ .map(({ slug, config }) => ({ slug, name: config?.name ?? slug }));
channels.sort((a, b) => a.name.localeCompare(b.name));
return (
diff --git a/export/e2e/x-posts-private.spec.ts b/export/e2e/x-posts-private.spec.ts
@@ -0,0 +1,104 @@
+import { expect, test, type Page } from "@playwright/test";
+import {
+ POST_CHANNEL,
+ POST_CHANNEL_SLUG,
+ POST_REPLY_ID,
+ POST_ROOT_ID,
+ postsPage,
+} from "./fixtures/data";
+import { installRoutes, openFilters } from "./helpers";
+
+// X posts are private (release 17 slice XP): with `social.x.visibility`
+// "private", a PUBLIC site's build carries no X channel and a PRIVATE site's
+// build carries all of it. The build itself — the index build's per-site posts
+// manifest and compose's posts tree — is pinned through the real code by
+// common/bin/compose-site.postsVisibility.test.ts: this suite's data is
+// route-mocked, never built. Here the two posts manifests that build writes
+// are served, and the visitor's side is checked: the Search in row's Posts box
+// and the post hits come and go with the manifest, with nothing special-cased.
+//
+// The fixture posts say "kappa" (two of them); no video does.
+
+const X_POST_SLUGS = [
+ `${POST_CHANNEL_SLUG}/${POST_ROOT_ID}`,
+ `${POST_CHANNEL_SLUG}/${POST_REPLY_ID}`,
+];
+
+const fulfillJson = (body: unknown) => ({
+ status: 200,
+ contentType: "application/json",
+ body: JSON.stringify(body),
+});
+
+// The site posts manifest compose writes: the X channel listed (a private
+// site's build), or no channel at all (a public site whose only posts were X
+// posts). Routed after installRoutes, so these answers win.
+async function servePostsManifest(page: Page, built: "public" | "private") {
+ await page.route("**/posts/manifest.json", (route) =>
+ route.fulfill(
+ fulfillJson({
+ version: 1,
+ channels:
+ built === "private"
+ ? [{ name: POST_CHANNEL, slug: POST_CHANNEL_SLUG, postCount: 4, platform: "twitter" }]
+ : [],
+ totalCount: built === "private" ? 4 : 0,
+ generatedAt: new Date().toISOString(),
+ }),
+ ),
+ );
+ await page.route(/\/posts\/[^/]+\/page-\d+\.json$/, (route) =>
+ route.fulfill(
+ fulfillJson(
+ postsPage().map((p) => ({
+ ...p,
+ platform: "twitter",
+ url: `https://x.com/tester/status/${p.id}`,
+ })),
+ ),
+ ),
+ );
+}
+
+const postsBox = (page: Page) =>
+ page.getByTestId("search-in-row").getByRole("checkbox", { name: "Posts", exact: true });
+
+async function search(page: Page, q: string) {
+ await page.locator('input[data-testid^="leaf-query-"]').first().fill(q);
+ await page.getByTestId("search-submit").click();
+}
+
+test.describe("X posts private", () => {
+ test.beforeEach(async ({ page }) => {
+ await installRoutes(page);
+ });
+
+ test("a public site built with X posts private has no Posts box and finds no X post", async ({
+ page,
+ }) => {
+ await servePostsManifest(page, "public");
+ await page.goto("/");
+ await openFilters(page);
+ await expect(page.getByRole("checkbox", { name: "Transcripts", exact: true })).toBeVisible();
+ await expect(postsBox(page)).toHaveCount(0);
+ await search(page, "kappa");
+ await expect(page.getByText(/^searched \d+\/\d+$/)).toBeVisible({ timeout: 15_000 });
+ await expect(page.getByTestId("results-summary")).toHaveText("Matching videos (0)");
+ await expect(page.locator(`[data-result-slug^="${POST_CHANNEL_SLUG}/"]`)).toHaveCount(0);
+ });
+
+ test("a private site's build shows the X posts", async ({ page }) => {
+ await servePostsManifest(page, "private");
+ await page.goto("/");
+ await openFilters(page);
+ await expect(postsBox(page)).toBeChecked();
+ await search(page, "kappa");
+ const cards = page.locator("[data-card-header]");
+ await expect(async () => {
+ const got = await cards.evaluateAll((els) =>
+ els.map((e) => e.getAttribute("data-result-slug") ?? ""),
+ );
+ expect(got.slice().sort()).toEqual(X_POST_SLUGS.slice().sort());
+ }).toPass({ timeout: 15_000 });
+ });
+});
diff --git a/plans/FACTS.md b/plans/FACTS.md
@@ -165,7 +165,7 @@ Never name the curated field `tags`. Never assume a `tags.json` is the keyword l
| --- | --- | --- |
| A video's visibility | `common/lib/availability.ts` (the `"unlisted"` state, `isUnlisted`), `common/lib/transcripts{,-server}.ts`, `common/components/shareUrl.ts`, `common/controller/buildIndex.ts` | The platform's own "unlisted" (reachable by link, not listed on the channel). |
| The hub's list has loaded | `export/app/components/hub/useHubSites.ts` — `listed` | `/hub-sites.json` has been answered and `/hub-summary.json` has settled. |
-| **A site the family lists** | `site.json` `listed` (`common/lib/siteSchema.ts` — `isListedSite`, `channelsOnlyOnUnlistedSites`) | Absent = listed; `false` keeps the site off the homepage, the hub and the other sites' footers, and out of the public totals. |
+| **A site the family lists** | `site.json` `listed` (`common/lib/siteSchema.ts` — `isListedSite`, `channelsOnlyOnUnlistedSites`) | Absent = listed; `false` keeps the site off the homepage, the hub and the other sites' footers, and out of the public totals. A PRIVATE site (`audience: "private"`, release 17 XP) is never listed, whatever `listed` says. |
A grep for either word finds all three; read the file before assuming which.
@@ -8251,6 +8251,26 @@ phase deletes from the destination.
**The source, not `cookieMode`, governs the X fetchers**: the browser source passes the spec
whatever the mode; the profile source keeps the old order (the jar, else the `"always"`-mode
spec, else a guest run).
+- **Where X posts may appear, `social.x.visibility`** (release 17 slice XP; `"public"` default |
+ `"private"`, kept by `sanitizeSocial` beside `cookieSource`). **saveSettings' merge is one level
+ deep, so a patch `{ social: { x } }` replaces the whole X block**: both X actions
+ (`xSessionActions.ts`) write over the block as it is (`socialXPatch`). The rule is
+ `common/lib/postsVisibility.ts` (`postsVisibleTo`, `publishedMemberSlugs`): while private, an X
+ channel (social, platform `twitter`) is built only into sites with `site.json` `audience:
+ "private"`; a public site leaves the channel out WHOLE (posts are all it holds) — its posts
+ manifest entry, posts tree, transcripts tree, channel list, `site.json` and `corpus.json` entry,
+ and `channel-sites.json`. Applied in `buildIndex`'s per-site loop (the site fingerprint names the
+ `withheld` members) and in `compose-site` (which prunes a previously shipped tree); the shared
+ posts tree and the LMDB posts sub-DB stay corpus-wide. A private site publishes no `hubUrl`
+ (`resolveHubUrl`), its `corpus.json` says `site.audience: "private"`, and every deploy path
+ refuses it or its bundle before any upload (`lib/builtExport.ts` `deployAudienceProblem`; the
+ bulk deploys skip it). Build & deploy all still BUILDS it.
+- **The hub carries no site's data** (release 17 XP review): `public/` is shared, so a hub built after
+ a site's compose used to ship that site's data trees. `compose-hub` now removes every per-site entry
+ first (`SITE_ONLY_PUBLIC_ENTRIES`) and writes the global `search-aliases.json`; `builtHubProblem`
+ refuses a hub bundle carrying a data tree. **compose-site's own stages (summaries, stats,
+ duplicates) are trusted from its cache only when `public/site.json` names the site**; the
+ per-channel trees keep their signatures across sites.
- **gallery-dl 1.32.9** takes `--cookies-from-browser BROWSER[/DOMAIN][+KEYRING][:PROFILE][::CONTAINER]`
(yt-dlp's syntax plus `/DOMAIN`) and reads every browser it supports, Chromium's encrypted store
included, on each run. The spec is passed verbatim (`galleryDlCookieChoice`).
diff --git a/plans/release-17.md b/plans/release-17.md
@@ -263,6 +263,7 @@ one short Transcribe (the hook on a relocated channel), `/storage`, `df`; then n
| **T3** migration + records | `r17/media-tier-migrate` | `common/bin/migrate-media-tier.ts`, `archilyzer.ts` wiring, fixture tests (tmp "platter"), FACTS "A channel's media is tiered", AGENTS.md's six things → seven, SETTINGS.md/CHANNEL.md regen, the release record, changelog | T1, T2 | dry run; resume from each phase; idempotent rerun; `--reclaim`; refusal on a marker; the free-space stop |
| **U1** umtool roots + `out/` | `r17/umtool-media-root` | `paths.mjs` (`MEDIA_ROOT`, `CACHE_DIR`), `lib/report/storage.mjs`, `driver.mjs`, `build-video.mjs:2615`, `export.mjs`, `kinds.mjs`, `umtool doctor`, `umtool storage move-out`, the e2e env | — (∥ T1) | `test:scripts` (+ mover tests), `next-build-trace.test.mjs`, the capped umtool build with the corpus linked, umtool e2e |
| **U2** deliverables switch | `r17/umtool-deliverables` | manifest `storage` field, `deliverableDir`, `cut.mjs:96`, `deliver.mjs:362`, `umtool storage deliverables`, bench "Move deliverables", `umtool check` | U1 | umtool unit + e2e: cut and share through a linked `clips/` |
+| **XP** X posts are private (operator-requested, beside the media tier) | `r17/x-posts-private` | `common/lib/postsVisibility.ts` (new) + test, `settingsSchema.ts` + `social/xCookieSource.ts` (`social.x.visibility`) + SETTINGS.md, `siteSchema.ts` + `site.ts` (`audience`, `isListedSite`, `resolveHubUrl`) + SITE.md, `buildIndex.ts` (the per-site loop only), `bin/compose-site.ts` + an integration test, `lib/corpus.ts`, `lib/builtExport.ts`, `publish/build.ts` + tests, `controller/poolSummary.ts` + test, `docker/publish-site.sh`, `export/app/offline/page.tsx`, editor `settings/{xSessionActions.ts,page.tsx,components/{XSessionSection,XPostsVisibilityControl}.tsx}`, `sites/{actions.ts,components/SiteForm.tsx,lib/{buildAction,deployAction}.ts}`, e2e `x-session`, `sites-crud`, export `x-posts-private` | — (∥ all) | the compose integration test (public vs private site, flip back, an X-only public site); deploy refusals before wrangler and before the upload |
Order: 0a → D0 ∥ T1 ∥ U1 → T2 ∥ U2 → T3 → parent: records, ONE editor rebuild + restart, umtool rebuild +
restart (the restart is the operator's: the permission layer refuses the `0.0.0.0` bind) → the migration
@@ -321,6 +322,30 @@ hand; a dirent `isFile()` filter over a video dir hides it."** The `.relocating.
- The `en` track → 0 cues bug (index prefers `en` over `en-orig`; some `en` VTTs parse to 0 cues).
- A channel export/import **bundle** built on `mediaTier.ts`'s classifier — the slice after this release.
+## Slice XP — the ruling (2026-10-01)
+
+- **Every X post is hidden from the public, for now; the data is kept, and stays readable by the MCP
+ and umtool for the operator's own questions and tasks.** Fetching is not changed by this slice.
+- **A setting, `social.x.visibility`: `"public"` (default) | `"private"`**, beside
+ `social.x.cookieSource`, chosen on `/settings` in the X account session section as "Where X posts
+ appear", with one sentence saying what private means and that sites already published change on
+ their next build and deploy. `"private"`: every X channel's posts (`sourceKind: "social"`,
+ `platform: "twitter"`) are left out of every PUBLIC site build and built only into PRIVATE sites.
+ Nothing on disk changes; flipping back is a rebuild.
+- **A site audience, `site.json` `audience`: `"public"` (default, absent) | `"private"`**, on the
+ site's form with a sentence. A private site is **never deployed** — every deploy path refuses it
+ with a sentence naming the audience, before any upload, where the release 13 W3 wrong-site guard
+ runs; a build-only still works — and **never listed**: no `hubUrl`, in no homepage or hub listing.
+ Its `corpus.json` says `"audience": "private"`.
+- **One predicate, `postsVisibleTo(site, channelConfig, settings)`**, pure and tested in
+ `common/lib`, called from the index build and from compose. The Search in row's Posts toggle keeps
+ working from what the build shipped (a public site whose only posts were X posts has no posts
+ corpus and no Posts toggle) — verified, not special-cased. The MCP needs no change; umtool's report
+ pipeline is checked for where it reads posts.
+- Not in scope: stopping fetches; a per-platform toggle for Bluesky; deleting anything; editing
+ `transcripts/**` (the rollout — a private site holding every channel, the setting flipped, the
+ public sites rebuilt — is the parent's, through the editor's own writers).
+
## Record
### Slice U1, as shipped — umtool's render scratch goes to a media root (2026-10-01)
@@ -538,4 +563,254 @@ media tier; a dangling link still reads as "no build" there (`umtool check` lear
mix render into it lands on the media root through the link (the write check is lexical; that matches
the semantics).
+### Slice XP, as shipped — X posts are private (2026-10-01)
+
+Branch `r17/x-posts-private` off `main` `90bd8384`, worktree `~/Projects/r13-lows-export` (editor 5501,
+test 5511, export 5510), one Opus implementer, beside the media-tier slices. Scratch files `XP-*` in the
+job's `tmp`. The ruling is above ("Slice XP — the ruling").
+
+**The listing side is release 14 slice HS's.** HS shipped `site.json` `listed` (absent = listed), the
+one predicate `isListedSite`, and every listing that reads it: the homepage summary,
+`channel-sites.json`, the pooled stats, the hub's `hub-sites.json` (and so its `corpus.json` and
+`llms.txt`), every footer, and the form's **List on the Archilyzer homepage and hub** checkbox. This
+slice adds no listing plumbing of its own: `isListedSite` gains one clause — a private site is never
+listed, whatever `listed` says — and the checkbox stays as it is. What is new is the deploy refusal,
+the private site's empty `hubUrl` and its `corpus.json` word, and "private content is built only into
+private sites".
+
+**What it does.**
+- **`social.x.visibility`: `"public"` (default, absent) | `"private"`**, beside `social.x.cookieSource`
+ (`XSocialSettings` and `sanitizeSocial` in `common/social/xCookieSource.ts`, the social block's home;
+ its doc in `settingsSchema.ts`; SETTINGS.md regenerated). On `/settings`, at the foot of the X account
+ session section, **Where X posts appear** (`XPostsVisibilityControl.tsx`): "Public — every site that
+ has the channel" | "Private — private sites only", with: "Private leaves every X channel and its posts
+ out of every public site and builds them only into sites whose audience is private, which are never
+ deployed; nothing is deleted and fetching goes on. Sites already published change on their next build
+ and deploy." Written by `setXPostsVisibilityAction` through `saveSettings`; "public" is written as no
+ key.
+- **`site.json` `audience`: `"public"` (default, absent) | `"private"`** (`siteSchema.ts`, only
+ `"private"` written, `isPrivateSite` the one predicate; SITE.md regenerated). The site form has an
+ **Audience** select with a sentence; `saveSiteAction` keeps only `"private"`.
+- **The rule, `common/lib/postsVisibility.ts`** (pure, tested): `postsVisibleTo(site, config, settings)`
+ — an X channel (`sourceKind: "social"`, `platform: "twitter"`) goes only to a private site while the
+ setting is private; every other channel is untouched — and `publishedMemberSlugs`, a site's members
+ narrowed by it. **A public site leaves the X channel out whole**: posts are all a social channel holds,
+ so without them it would be an empty checkbox and a name in `corpus.json`. Narrowed by the same call in
+ `buildIndex`'s **per-site loop** (the summaries, subs, posts and digests manifests; the site
+ fingerprint gains `withheld`, only when non-empty, so flipping the setting or the audience rebuilds
+ the site and nothing else moves) and in `compose-site` (every shared tree it copies, so a tree a
+ public site shipped before is pruned). The shared posts tree and the LMDB posts sub-DB stay
+ corpus-wide. `channel-sites.json` maps a channel only to the sites whose build carries it
+ (`channelSitesOf` takes the narrowing) and the export's `/offline` page lists the same members.
+- **A private build says so and belongs under no hub**: `corpus.json`'s `site.audience` is `"private"`;
+ `resolveHubUrl` gives a private site no `hubUrl` (absent from its `site.json` and `corpus.json`).
+- **Never deployed.** `lib/builtExport.ts`: `siteDeployProblem(site)` — `Site "x" is private (audience:
+ private): it is built for reading on this machine and is never deployed. Build it without deploying,
+ or set its audience to public on its Settings tab` — `builtAudienceProblem(outDir)` (a bundle whose
+ `corpus.json` says private, so a site switched back to public cannot ship its private build) and
+ `deployAudienceProblem`, both. Asked first, before any upload, where release 13 W3's
+ `builtBundleProblem` is asked: `runDeployIntoLog` (the last word before wrangler),
+ `runDockerDeployAllPhase` (skipped with the sentence, not failed, so Build & deploy all is not red
+ while a private site exists — the site is still built), `deploySite` (`archilyzer deploy site`, before
+ the R2 upload), the editor's `deployExportAction` and `buildAndDeployAction` (before a job exists;
+ Build & deploy asks again before its upload), the host Build & deploy all fallback
+ (`basicBuildAndDeployAll`, skipped before the upload), and `docker/publish-site.sh` (before building,
+ from `site.json`, and over the built `corpus.json` before publishing). The ops routes `build-deploy`
+ and `deploy-site` answer the actions' sentence (400). **Build** still builds a private site.
+- **The Search in row's Posts toggle**: no code changed. `hasPostsCorpus` is "the site posts manifest
+ lists a channel", so a public site whose only posts were X posts ships `channels: []` and no Posts
+ box — pinned by the integration test (no `postScheme` either) and the export spec.
+
+**Where the rule lives — one deviation.** The prompt named the social-channel branch of `scanSource`
+(`buildIndex.ts:333-341`). That branch is corpus-wide: it feeds the shared posts tree that every site,
+and the MCP over a private build, read, so it must keep building X posts. The rule sits in the per-site
+loop (`buildIndex.ts` ~1943 and the fingerprint), a different hunk from T1's `scanSource` guard.
+
+**Found on the way, fixed here.**
+- **`saveSettings` merges one level deep, so a patch `{ social: { x } }` replaced the whole X block**:
+ choosing a login source would have erased the visibility, and the reverse. Both X actions now write
+ over the block as it is (`socialXPatch`); the e2e case pins both directions.
+- **compose-site trusted its per-site cache after ANOTHER site's compose.** `public/` is one directory
+ every site composes into in turn (the basic build); the cache is per site, and a stage whose source
+ had not changed was skipped. So composing site B, then site A again with no new data, shipped B's
+ summaries — its whole channel list — as A's: a public site composed after a private one holding every
+ channel would have listed the private site's channels. The site's own stages (summaries, stats,
+ duplicates) are now trusted only when `public/site.json` names the site; `site.json` is cleared at
+ the start of a compose and written at its end, so a compose cut short leaves nothing to trust. The
+ per-channel trees keep their signatures (copies of the shared trees, the same bytes whichever site
+ copied them). Docker builds have a per-site `public/` and were not affected.
+- **compose never ships a posts tree the site's posts manifest does not list** (the index build's
+ word), so a channel config compose fails to read — read as visible — does not ship X posts.
+
+**The MCP and umtool.**
+- **The MCP needs no change**: it reads a composed export (`mcp/src/sources.ts`, `--local <dir>` |
+ `TRANSCRIPT_LOCAL_DIR`). A private site is composed into a directory of its own, without touching
+ `export/public`, from the checkout root:
+ ```sh
+ pnpm archilyzer index # or any editor build: the index build writes the per-site manifests
+ BUILD_ARCHIVES=0 EXPORT_PUBLIC_DIR="$HOME/archives/<private-id>" \
+ EXPORT_INDEX_DIR="$PWD/export/.export-index" pnpm archilyzer compose site <private-id>
+ claude mcp remove archilyzer -s local # the name must be free; use the scope it was added in
+ claude mcp add archilyzer \
+ --env ARCHILYZER_EDITOR_URL=http://localhost:3001 \
+ --env WORKER_TOKEN=… \
+ -- pnpm --silent -C "$PWD" archilyzer mcp --local "$HOME/archives/<private-id>"
+ ```
+ The two `--env` lines are what `fetch_clip` needs (the editor's own `WORKER_TOKEN`, from
+ `editor/.env`); the name stays `archilyzer` for `/ask` and `/sweep` (AGENTS.md).
+ (`EXPORT_PUBLIC_DIR` must be absolute — the command runs in `common/`; the compose cache lands beside
+ it, in `$HOME/archives/.compose-cache/`.) The site's editor **Build** works too: it composes into
+ `export/public` and builds into `export/out`. **The current registration, `--local
+ <checkout>/export/public`, reads whatever site was composed there last**: after a public site's build
+ it has no X posts, after the private site's it has them.
+- **umtool is unaffected**: the report pipeline's posts (the deck's posts room) are carried whole in the
+ report manifest — `posts[]` with `platform`, `date`, `text`, `url` (`validatePosts`,
+ `umtool/report-to-video/deck.mjs`), "added by editing the manifest" — and its cues come from the local
+ corpus or the archive `provenance.siteOrigin` names (videos only). It reads no public build's posts. A
+ sweep that looks posts up for a manifest goes through the MCP, pointed at the private build as above.
+
+**Commits**
+
+| Commit | What |
+|---|---|
+| `3ea76853` | `common:` `postsVisibility.ts` + test; `social.x.visibility`; `site.json` `audience` (`isListedSite`, `resolveHubUrl`); the per-site loop and compose narrowed; `corpus.json` `audience`; the deploy refusals (`builtExport`, `publish/build`) + tests; `channel-sites.json`; `/offline`; `publish-site.sh`; SETTINGS.md, SITE.md; the compose integration test |
+| `0195d52a` | `editor:` Where X posts appear; the X block written whole; the Audience select; the deploy actions refuse a private site |
+| `75ccbef3` | `editor(e2e), export(e2e):` `x-session` and `sites-crud` cases; `export/e2e/x-posts-private.spec.ts` |
+| `30c14aa0` | `common:` compose never ships a posts tree the index withheld; the cache trusted only over the site's own last compose |
+| `63002de3` | `common:` the per-channel trees keep their signatures across sites |
+| `4ed7d410` | `plans:` this section, the ruling, the slices row; FACTS; the editor changelog |
+| `6466a68e` | `common:` the hub carries no site's data; `builtHubProblem` refuses one that does (review HIGH 1) |
+| `385e4eb1` | `common:` a compose over a stale index lists no withheld channel; the comments (LOW 3, NIT 7) |
+| `8e448fde` | `editor:` the changelog (LOW 5) |
+| `da5c2912` | `plans:` the review, its record, the rollout steps and the gates after it |
+| `837d630a` | merge of `main` `bb877f93` (slice U1: umtool, `.gitignore`, and the two shared records — both sides kept, U1's section before this one). Re-gated on the merged tree: tsc clean; common **2,501/2,501**; export unit 98/98; homepage unit 23/23; editor unit 109/109; test:scripts 392 passed, 0 failed, 2 skipped (394). The merge touched no file the editor, export or hub e2e lists cover (umtool only), so they were not re-run |
+| this commit | `plans:` the merge in this table |
+
+#### Gates (logs `$T/XP-*.log`)
+
+- **tsc** (all workspaces) clean before every commit; last at `63002de3`'s tree (`XP-tsc5.log`).
+- **common:** **2,499/2,499** at `63002de3`, 161 s (`main`'s count + the new `postsVisibility.test.ts`
+ 7, `compose-site.postsVisibility.test.ts` 4, `build.test.ts` +3, `poolSummary.test.ts` +1); 2,498 at
+ `75ccbef3`. `archilyzer docs files --check` and `settings example --check` exit 0. **Editor unit:**
+ 109/109. **Export unit:** 98/98. **Homepage unit:** 23/23. **mcp:** 271/271 (no change there).
+- **test:scripts:** 367 passed, 1 failed, 2 skipped of 370 (`XP-scripts2.log`; the first run had 2
+ failed). The failures are `scripts/queue-lock.test.mjs`'s "prints a banner naming the holder while
+ waiting" (and once "serves waiters in arrival order"): timing cases (200 ms and 150 ms staggers) run
+ at a load average of 23–30 while other suites held the e2e queue; alone, the banner case still fails
+ under that load (10/11). This slice does not touch `scripts/` (`git diff 90bd8384 -- scripts/` is
+ empty).
+- **Builds** at `75ccbef3` (the later commits touch only `compose-site`, a bin no Next app bundles): the
+ capped editor build with the corpus linked (`ln -sT`, `systemd-run --scope -p MemoryMax=6G`, the link
+ removed after) exit 0, 172 s; `pnpm --filter export exec next build` exit 0, 83 s (the `export/public`
+ links made, 0 dangling); `pnpm --filter homepage run build:nodata` exit 0, 47 s.
+- **Numbers tool:** none. **Privacy gate:** `git diff main --name-only | xargs grep -lc …` names one
+ file, `plans/FACTS.md`, whose 3 matches are all on `main` already; 0 in this slice's added lines.
+
+ | Run | At | Specs | Result |
+ |---|---|---|---|
+ | 1 (editor) | `75ccbef3` | `sites-crud`, `settings`, `x-session`, `forms-keep-input`, `deploy-page`, `site-publish-preview`, `sites-homepage` | **62 passed**, 0 failed, 4.0 min (after 35 min in the queue) |
+ | 2 (export) | `63002de3` | `x-posts-private` (new, 2), `search-in`, `posts-search` | **20 passed**, 0 failed, 3.0 min (after 7 min in the queue) |
+
+ New cases: `x-session` "where X posts appear persists, beside the login source and without it" (the
+ whole-block write both ways); `sites-crud` "a private site saves its audience, and every deploy
+ refuses it before any job" (the form, the file, `build-deploy` and `deploy-site` answering the
+ sentence with 400, back to public with the key gone); export `x-posts-private` — the posts manifest a
+ public site built with X private ships (no channel: no Posts box, no X post for "kappa") and the one a
+ private site ships (the Posts box, both X posts). **The export suite's data is route-mocked, never
+ built**, so the build rule itself is proved by `common/bin/compose-site.postsVisibility.test.ts`
+ through the real `buildIndex` and compose: a public and a private site over one video, one X and one
+ Bluesky channel; the flip back; a public site whose only posts were X posts (empty manifest, no
+ `postScheme`); a config compose cannot read; a public site composed after the private one.
+
+#### Found and left
+
+- **An X channel's manifest-only transcripts tree** (pageCount 0) would still be copied into a public
+ site when compose fails to read the channel's config: a directory named for the channel, holding no
+ content. The posts tree, the posts manifest, the channel list and `corpus.json` follow the index build
+ and do not carry it.
+- The `/sites` list does not mark a private site; its form and every deploy refusal do.
+- **The posts reconcile ships only the channels the site's posts manifest lists, on every build**: a
+ social channel with 0 posts no longer gets a posts folder. Nothing advertised that folder (no posts
+ manifest entry, no `corpus.json` posts link), so nothing reads the difference.
+- **A private site changes the homepage's and the hub's totals.** A private site is unlisted, and
+ `channelsOnlyOnUnlistedSites` keeps a channel only unlisted sites expose out of every public total. A
+ private site holding every channel turns every pool-only channel (on no public site) into "only on
+ unlisted sites", so the homepage's and the hub's instance-wide totals drop by those channels at the
+ next homepage and hub build. Channels a public site also has are unaffected.
+- **Cloudflare Pages keeps old builds reachable.** A production redeploy replaces what the production
+ URL serves, nothing else: every earlier deployment stays live at its own
+ `<hash>.<project>.pages.dev`, and a preview alias (`<branch>.<project>.pages.dev`) keeps its last
+ build. The review found `tags-exclude.anilyzer.pages.dev/posts/manifest.json` still listing an X
+ channel. So after the rollout, X posts are gone from the production URLs only; whether to delete
+ the old deployments is the operator's call (rollout step 5).
+- `queue-lock.test.mjs`'s two timing cases failed under the machine's load (below); not this slice's file.
+
+#### Decisions the operator could overturn
+
+| What I did | The alternative |
+|---|---|
+| A public site leaves an X channel out WHOLE (posts, posts manifest, channel list, `site.json`, `corpus.json`, `channel-sites.json`) | Keep the channel listed with no posts: an empty checkbox and a name in `corpus.json` |
+| Build & deploy all builds a private site and SKIPS its deploy, with the sentence | Fail its deploy: the run goes red every time while a private site exists |
+| A bundle whose `corpus.json` says private is refused even when the site is public now | Trust the site's current audience only (a stale private build could ship) |
+| `docker/publish-site.sh` refuses a private site (the `site` service is the host's public face) | Let it publish locally behind Caddy |
+| `social.x.visibility` lives in `xCookieSource.ts` with the rest of the social block | A module of its own |
+| The hub's `search-aliases.json` is the global dictionary (it was whichever site composed last) | Ship none: hub-wide search in a reader (`reader-hub.ts`) would have no aliases |
+
+#### Rollout for this slice (the parent's, through the editor's own writers)
+
+1. Rebuild and restart the editor (the setting, the Audience field, the deploy refusals, the compose
+ and hub fixes).
+2. Create the private site (Sites → New, **Audience: Private**, every channel), then set **Where X posts
+ appear: Private** on `/settings`.
+3. Rebuild and deploy every public site that has an X channel (each by name; Build & deploy all skips the
+ private site's deploy and says why).
+4. **Rebuild and deploy the hub** — the live hub serves the last-built site's data, X posts included,
+ until it is rebuilt from this branch (the review's HIGH 1). Then the homepage, for the totals.
+5. **Old deployments** (the operator decides): every earlier production deployment and every preview
+ alias of a site that had X posts still serves them. To remove them: the Cloudflare dashboard →
+ Workers & Pages → the project → Deployments → a deployment's ⋯ menu → Delete deployment (a preview
+ alias's branch deployments are listed there too); or `pnpm dlx wrangler pages deployment list
+ --project-name <project>` then `pnpm dlx wrangler pages deployment delete <deployment-id>
+ --project-name <project>` (check `--help` for the force flag an aliased deployment needs). The
+ current production deployment cannot be deleted, and needs none.
+6. Build the private site (its **Build**, or the compose-to-a-directory commands above) and point the
+ MCP at it.
+
+#### Review
+
+**Verdict: SHIP AFTER FIXES** (`XP-review.md` in the job's scratch): two Highs, four Lows, three nits.
+
+| Finding | Where |
+|---|---|
+| HIGH 1: the hub bundle carried the last-composed site's data trees (the live hub serves jeralyzer's posts manifest, two X channels), through no gate | `6466a68e`: `compose-hub` removes every per-site entry from `public/` first (`SITE_ONLY_PUBLIC_ENTRIES`: summaries, transcripts, subs, posts, digests, stats, archives, `site.json`, `tags.json`, `duplicates.json`, `search-aliases.json`, `chart-templates.json`, `sitemap.xml`; a worktree link by the link only) and writes the global alias dictionary as the hub's; `builtHubProblem` refuses a hub bundle that still carries a data tree, so Deploy hub refuses one. Tests: `compose-hub.test.ts` +1, `builtExport.test.ts` extended. Rollout step 4 |
+| HIGH 2: Pages preview aliases and old deployments keep serving X posts | Record: "Found and left" and rollout step 5 (the operator decides; the dashboard and wrangler paths) |
+| LOW 3: a compose over an index built before the flip listed the X channel's name and count | `385e4eb1`: the served posts manifest and summaries manifest are narrowed to the published members (`site.json`, `corpus.json` follow); a narrowed summaries copy is not trusted by the next compose. Test: "a compose over an index built before the setting flipped lists no X channel anywhere" |
+| LOW 4: the MCP line lost `fetch_clip`'s env, and `add` fails over a registered name | This commit: `claude mcp remove archilyzer -s local` first, the two `--env` lines kept (`WORKER_TOKEN=…`) |
+| LOW 5: the changelog missed the compose-cache fix and the restart notes | `8e448fde`: a bullet for the compose-cache fix, one for the hub, and the restart/redeploy notes |
+| LOW 6: a private site holding every channel moves the homepage's and hub's totals | Record: "Found and left" |
+| NIT 7: the deploy-all comment said the bundle check runs first | `385e4eb1`: the comment says the audience check runs first and what that means for a private wrong-site bundle |
+| NIT 8: the posts reconcile drops a 0-post social channel's folder | Record: "Found and left" |
+| NIT 9: `/sites` does not mark a private site | Already listed as left |
+
+#### Gates after the review
+
+- **tsc** (all workspaces) clean at `385e4eb1`'s tree (`XP-tsc6.log`).
+- **common:** 2,500 passed, 1 failed of 2,501 (`XP-common4.log`; +2 since the first gates: the hub
+ clearing and the stale-index compose). The one failure is `relocateChannelMedia.test.ts`'s "reconcile:
+ an extra and a changed file on the destination are settled" (its diff listing counted `./` as
+ changed — a directory mtime); 3/3 alone, and the file is not this slice's. **Export unit:** 98/98.
+ **Homepage unit:** 23/23. **Editor unit:** 109/109.
+- **No rebuild:** the fixes touch two bins (`compose-hub`, `compose-site`), `builtExport.ts` and
+ comments in `publish/build.ts`; no Next app bundles a changed module beyond `builtExport` (the
+ editor's hub action reads `builtHubProblem`, a pure function, tsc-checked).
+
+ | Run | At | Specs | Result |
+ |---|---|---|---|
+ | 3 (editor) | `8e448fde` | the run-1 list | **62 passed**, 0 failed, 3.0 min (after 45 min in the queue) |
+ | 4 (export) | `8e448fde` | the run-2 list | **20 passed**, 0 failed, 1.1 min (after 50 min in the queue) |
+ | 5 (hub) | `8e448fde` | `e2e:hub`, the whole suite | **39 passed**, 0 failed, 1.6 min (after 12 min in the queue) |
+
+ The hub suite runs `next dev` in hub mode over `export/public` and never composes, so it shows the
+ hub app is unchanged; the clearing itself is pinned by `compose-hub.test.ts`.
+
## Rollout
