commit c93635a5aa0bad3820d626deb4d2d8222e9c3209
parent a210dace521bb77bc17e488119c7275a44fbdef1
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 1 Oct 2026 17:19:03 -0400
common: X posts are private — social.x.visibility, a site's audience, one predicate for both builds, and the deploy refusal
- lib/postsVisibility.ts (new, pure, tested): postsVisibleTo(site, config,
settings) and publishedMemberSlugs — an X channel (social, platform
twitter) is built only into private sites while social.x.visibility is
"private"; every other channel is untouched.
- settings: social.x.visibility ("public" default | "private"), kept by
sanitizeSocial beside cookieSource; SETTINGS.md regenerated.
- site.json: audience ("public" default | "private"; only private written);
isListedSite is false for a private site; resolveHubUrl gives a private
site no hubUrl; SITE.md regenerated.
- buildIndex's per-site loop and compose-site narrow the site's members by the
one predicate (the corpus-wide shared posts tree is unchanged); the site
fingerprint names withheld members; corpus.json says "audience": "private".
- builtExport: siteDeployProblem / builtAudienceProblem /
deployAudienceProblem; runDeployIntoLog, the container deploy phase and
deploySite refuse a private site or a private build before any upload.
- channel-sites.json maps a channel only to the sites whose build carries it;
the export's offline page lists the same members.
- docker/publish-site.sh refuses a private site before building and a private
build before publishing.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
19 files changed, 889 insertions(+), 24 deletions(-)
diff --git a/SETTINGS.md b/SETTINGS.md
@@ -160,7 +160,7 @@ Default: `"when-required"`
## `social`
-Per-platform settings of the social-post fetchers. Today one key: where the X fetchers' login comes from (`social.x.cookieSource`, chosen in the X account session section of /settings). See common/social/xCookieSource.ts.
+Per-platform settings of the social posts. Today two keys, both X's, both chosen in the X account session section of /settings: where the X fetchers' login comes from (`social.x.cookieSource`) and where X posts may appear (`social.x.visibility`). See common/social/xCookieSource.ts.
#### `social`
@@ -173,6 +173,7 @@ Per-platform settings of the social-post fetchers. Today one key: where the X fe
| Key | Default | Description |
|---|---|---|
| `cookieSource` | absent | Where the X fetchers' login comes from. `"browser"`: the operator's everyday browser, named by `cookiesFromBrowser` — gallery-dl is handed `--cookies-from-browser <spec>` and reads it on every run, and the Playwright fallback reads the same store (Firefox only; common/social/xBrowserLogin.ts), so the login lasts as long as the browser's. `"profile"`: the session broker's persistent profile ("Connect X account" on /settings) and the cookie jar it exports. ABSENT (the default) is resolved at read time, never stored: `"browser"` when `cookiesFromBrowser` is set and no profile is connected (no exported jar carrying an auth_token), else `"profile"`. The source, not `cookieMode`, governs the X fetchers. |
+| `visibility` | absent | Where X posts may appear. `"public"` (the default; absent): an X channel's posts are built into every site that has the channel. `"private"`: every X channel's posts (a channel with `sourceKind: "social"` and `platform: "twitter"`) are left out of every PUBLIC site build — the channel with them, since posts are all an X channel holds — and built only into PRIVATE sites (`site.json` `audience`). Nothing on disk changes and fetching does not; a site already published changes on its next build and deploy, and flipping back is a rebuild. Chosen in the X account session section of /settings; the rule is common/lib/postsVisibility.ts. |
Default:
diff --git a/SITE.md b/SITE.md
@@ -24,6 +24,7 @@ Regenerate this file with `pnpm --filter yt-dlp-transcript-common exec tsx bin/f
| [`accent`](#accent) | absent |
| [`siteUrl`](#siteurl) | absent |
| [`listed`](#listed) | `true` |
+| [`audience`](#audience) | absent |
| [`relatedSites`](#relatedsites) | `[]` |
| [`pwa`](#pwa) | `false` |
| [`archives`](#archives) | `true` |
@@ -164,6 +165,12 @@ Whether the family lists this site. Opt-OUT: absent/true = listed, only an expli
Default: `true`
+## `audience`
+
+Who this site is built for. `"public"` (the default; absent) or `"private"`: the operator's own reading copy, built on this machine and never deployed — every deploy path (Build & deploy, Deploy, `archilyzer deploy site`, Build & deploy all, docker/publish-site.sh) refuses it before any upload, while a build without a deploy still works. A private site is never listed (as `listed: false`, whatever `listed` says), publishes no `hubUrl`, and its `/corpus.json` says `"audience": "private"`. Content kept from the public — X posts while `social.x.visibility` is `"private"` — is built only into private sites. Only `"private"` is written.
+
+Default: absent
+
## `relatedSites`
Pulls specific siblings to the front of the footer's cross-site list, in named groups. Siblings not named here fall into a trailing "Other sites" group. Absent/empty = one flat list of every sibling.
diff --git a/common/bin/compose-site.postsVisibility.test.ts b/common/bin/compose-site.postsVisibility.test.ts
@@ -0,0 +1,287 @@
+// Integration: X posts are private (release 17 slice XP), through the REAL
+// index build and the REAL site compose, over a temp corpus.
+//
+// One video channel, one X channel and one Bluesky channel, on two sites that
+// both have all three: `pub` (public) and `priv` (`audience: "private"`). With
+// `social.x.visibility` "private", the public site's build carries no X channel
+// at all — no posts manifest entry, no posts tree, no channel in its channel
+// list, site.json or corpus.json — while the private one carries everything
+// and says `"audience": "private"` with no hubUrl. Flipping the setting back
+// is a rebuild. The rule itself is lib/postsVisibility.ts (its own tests).
+//
+// The export e2e cannot show this: its data is route-mocked, never built by
+// buildIndex and compose. export/e2e/x-posts-private.spec.ts serves the two
+// posts manifests this file pins and checks what a visitor sees.
+//
+// Run with: node_modules/.bin/tsx --test common/bin/compose-site.postsVisibility.test.ts
+
+import { after, test } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
+// Every path getPaths() can resolve to a place this file's code may write is
+// pinned under ROOT before anything calls it (buildIndex.test.ts's list).
+const ROOT = mkdtempSync(path.join(tmpdir(), "posts-visibility-"));
+const PINNED: Record<string, string> = {
+ TRANSCRIPTS_DIR: path.join(ROOT, "transcripts"),
+ SAVED_VIDEOS_DIR: path.join(ROOT, "saved-videos"),
+ SITES_DIR: path.join(ROOT, "transcripts", "sites"),
+ SETTINGS_FILE: path.join(ROOT, "settings.json"),
+ EXPORT_PUBLIC_DIR: path.join(ROOT, "public"),
+ EXPORT_INDEX_DIR: path.join(ROOT, ".export-index"),
+ EXPORT_BUILDS_DIR: path.join(ROOT, ".export-builds"),
+ EDITOR_CHANGELOG_FILE: path.join(ROOT, "editor-CHANGELOG.md"),
+ EXPORT_CHANGELOG_FILE: path.join(ROOT, "export-CHANGELOG.md"),
+ CHARTS_CONFIG_FILE: path.join(ROOT, "chart-templates.json"),
+ SEARCH_ALIASES_FILE: path.join(ROOT, "transcripts", "search-aliases.json"),
+ CURATED_TAGS_FILE: path.join(ROOT, "transcripts", "tags.json"),
+ ARCHILYZER_CONFIG_DIR: path.join(ROOT, "config"),
+ ARCHILYZER_SOURCE_SCRATCH: path.join(ROOT, "source-scratch"),
+};
+Object.assign(process.env, PINNED);
+after(() => rmSync(ROOT, { recursive: true, force: true }));
+
+const { getPaths } = await import("../lib/paths");
+const { buildIndex } = await import("../controller/buildIndex");
+const { writePosts } = await import("../lib/posts-server");
+const { main: composeSite } = await import("./compose-site");
+const { builtAudienceProblem, deployAudienceProblem } = await import("../lib/builtExport");
+
+const paths = getPaths();
+const VIDEOS = "vids";
+const X = "jer-x";
+const SKY = "jer-sky";
+const HUB = "https://hub.example.test";
+
+const writeJson = (file: string, value: unknown) => {
+ mkdirSync(path.dirname(file), { recursive: true });
+ writeFileSync(file, JSON.stringify(value, null, 2));
+};
+const readJson = <T>(file: string): T => JSON.parse(readFileSync(file, "utf8")) as T;
+
+// YouTube's rolling-caption shape (parseVtt keeps lines with inline timing).
+const VTT =
+ "WEBVTT\nKind: captions\nLanguage: en\n\n" +
+ "00:00:00.000 --> 00:00:05.000 align:start position:0%\n" +
+ "First<00:00:01.000><c> caption</c><00:00:02.000><c> line.</c>\n";
+
+function post(slug: string, id: string, platform: "twitter" | "bluesky", text: string) {
+ return {
+ id,
+ slug: `${slug}/${id}`,
+ channelSlug: slug,
+ author: slug,
+ createdAt: "2026-09-01T12:00:00.000Z",
+ uploadDate: "20260901",
+ text,
+ url: `https://example.test/${slug}/${id}`,
+ platform,
+ isReply: false,
+ isRepost: false,
+ links: [],
+ };
+}
+
+function writeSettings(visibility?: "public" | "private") {
+ writeJson(paths.settingsFile, {
+ homepageUrl: HUB,
+ ...(visibility ? { social: { x: { visibility } } } : {}),
+ });
+}
+
+async function seedCorpus() {
+ writeJson(path.join(paths.channelsDir, VIDEOS, "config.json"), {
+ handling: "youtube",
+ name: "Videos",
+ url: "https://www.youtube.com/@vids/videos",
+ });
+ const dir = path.join(paths.channelsDir, VIDEOS, "data", "v1");
+ writeJson(path.join(dir, "metadata.info.json"), {
+ id: "v1",
+ title: "Video one",
+ channel: VIDEOS,
+ upload_date: "20260601",
+ duration: 120,
+ webpage_url: "https://www.youtube.com/watch?v=v1",
+ extractor_key: "Youtube",
+ });
+ writeFileSync(path.join(dir, "transcript.en.vtt"), VTT);
+
+ writeJson(path.join(paths.channelsDir, X, "config.json"), {
+ handling: "youtube",
+ name: "Jer on X",
+ url: "https://x.com/jer",
+ sourceKind: "social",
+ platform: "twitter",
+ socialHandle: "jer",
+ });
+ await writePosts(path.join(paths.channelsDir, X), [
+ post(X, "1001", "twitter", "an x post"),
+ post(X, "1002", "twitter", "another x post"),
+ ]);
+ writeJson(path.join(paths.channelsDir, SKY, "config.json"), {
+ handling: "youtube",
+ name: "Jer on Bluesky",
+ url: "https://bsky.app/profile/jer.example",
+ sourceKind: "social",
+ platform: "bluesky",
+ socialHandle: "jer.example",
+ });
+ await writePosts(path.join(paths.channelsDir, SKY), [
+ post(SKY, "3kabc", "bluesky", "a bluesky post"),
+ ]);
+
+ const site = (siteId: string, extra: Record<string, unknown> = {}) =>
+ writeJson(path.join(paths.sitesDir, siteId, "site.json"), {
+ siteId,
+ siteTitle: siteId,
+ siteDescription: "fixture",
+ headerTitle: siteId,
+ homeTagline: "",
+ socialLinks: [],
+ groups: [{ id: "default", name: "All channels", selectedByDefault: true }],
+ defaultGroupId: "default",
+ channels: [VIDEOS, X, SKY].map((slug) => ({ slug, groupId: "default" })),
+ siteUrl: `https://${siteId}.example.test`,
+ archives: false,
+ ...extra,
+ });
+ site("pub");
+ site("priv", { audience: "private" });
+}
+
+const quiet = () => {};
+async function index() {
+ await buildIndex({ paths, onLog: quiet });
+}
+
+// What one site's compose put in public/.
+type Composed = {
+ postsManifest: { channels: { slug: string; platform: string; postCount: number }[]; totalCount: number };
+ postTrees: string[];
+ transcriptTrees: string[];
+ siteJson: { channels: { slug: string }[]; hubUrl?: string };
+ corpus: {
+ site: { id: string; hubUrl?: string; audience?: string };
+ channels: { slug: string; postCount?: number; manifests: { posts?: string } }[];
+ postScheme?: unknown;
+ totals: { channels: number };
+ };
+};
+async function compose(siteId: string): Promise<Composed> {
+ const log = console.log;
+ console.log = quiet;
+ try {
+ await composeSite({ siteId, paths });
+ } finally {
+ console.log = log;
+ }
+ const pub = paths.exportPublicDir;
+ const trees = (dir: string) =>
+ [VIDEOS, X, SKY].filter((slug) => existsSync(path.join(dir, slug)));
+ return {
+ postsManifest: readJson(path.join(pub, "posts", "manifest.json")),
+ postTrees: trees(path.join(pub, "posts")),
+ transcriptTrees: trees(path.join(pub, "transcripts")),
+ siteJson: readJson(path.join(pub, "site.json")),
+ corpus: readJson(path.join(pub, "corpus.json")),
+ };
+}
+
+const slugs = (xs: { slug: string }[]) => xs.map((c) => c.slug).sort();
+
+test("X private: a public site carries no X channel; a private site carries all of it and says so", async () => {
+ await seedCorpus();
+ writeSettings("private");
+ await index();
+
+ // The index build's per-site manifests, before any compose.
+ const sitePosts = (id: string) =>
+ readJson<Composed["postsManifest"]>(
+ path.join(paths.exportSitesIndexDir, id, "posts", "manifest.json"),
+ );
+ assert.deepEqual(slugs(sitePosts("pub").channels), [SKY]);
+ assert.deepEqual(slugs(sitePosts("priv").channels), [SKY, X]);
+ // The shared posts tree is corpus-wide and keeps X: the private site and
+ // the MCP over its build read it.
+ assert.ok(existsSync(path.join(paths.exportSharedPostsDir, X, "manifest.json")));
+
+ const pub = await compose("pub");
+ assert.deepEqual(slugs(pub.postsManifest.channels), [SKY]);
+ assert.equal(pub.postsManifest.totalCount, 1);
+ assert.deepEqual(pub.postTrees, [SKY]);
+ assert.deepEqual(pub.transcriptTrees, [VIDEOS, SKY]);
+ assert.deepEqual(slugs(pub.siteJson.channels), [SKY, VIDEOS]);
+ assert.deepEqual(slugs(pub.corpus.channels), [SKY, VIDEOS]);
+ assert.equal(pub.corpus.totals.channels, 2);
+ assert.equal(pub.corpus.site.audience, undefined);
+ assert.equal(pub.corpus.site.hubUrl, HUB);
+ assert.equal(pub.siteJson.hubUrl, HUB);
+ assert.equal(builtAudienceProblem(paths.exportPublicDir), null);
+ assert.equal(deployAudienceProblem({ siteId: "pub" }, paths.exportPublicDir), null);
+
+ const priv = await compose("priv");
+ assert.deepEqual(slugs(priv.postsManifest.channels), [SKY, X]);
+ assert.equal(priv.postsManifest.totalCount, 3);
+ assert.deepEqual(priv.postTrees, [X, SKY]);
+ assert.deepEqual(slugs(priv.corpus.channels), [SKY, X, VIDEOS]);
+ assert.equal(priv.corpus.channels.find((c) => c.slug === X)?.postCount, 2);
+ assert.ok(priv.corpus.channels.find((c) => c.slug === X)?.manifests.posts);
+ assert.equal(priv.corpus.site.audience, "private");
+ // A private site belongs under no hub.
+ assert.equal(priv.corpus.site.hubUrl, undefined);
+ assert.equal(priv.siteJson.hubUrl, undefined);
+ // And its bundle refuses to deploy — under its own id, and under another
+ // site's (a public site's identity on a private build is still refused).
+ assert.match(builtAudienceProblem(paths.exportPublicDir) ?? "", /private build of "priv"/);
+ assert.match(
+ deployAudienceProblem({ siteId: "priv", audience: "private" }, paths.exportPublicDir) ?? "",
+ /^Site "priv" is private \(audience: private\)/,
+ );
+
+ // Compose the public site again over the private one's public/: the X
+ // channel it carried is pruned from every tree.
+ const again = await compose("pub");
+ assert.deepEqual(again.postTrees, [SKY]);
+ assert.deepEqual(again.transcriptTrees, [VIDEOS, SKY]);
+ assert.equal(again.corpus.site.audience, undefined);
+});
+
+test("X public again: the next build puts X back on the public site", async () => {
+ writeSettings("public");
+ await index();
+ const pub = await compose("pub");
+ assert.deepEqual(slugs(pub.postsManifest.channels), [SKY, X]);
+ assert.deepEqual(pub.postTrees, [X, SKY]);
+ assert.deepEqual(slugs(pub.corpus.channels), [SKY, X, VIDEOS]);
+
+ // No setting at all is public too.
+ writeSettings();
+ await index();
+ assert.deepEqual(slugs((await compose("pub")).postsManifest.channels), [SKY, X]);
+});
+
+test("a public site whose only posts were X posts ships an empty posts manifest and no post scheme", async () => {
+ writeSettings("private");
+ writeJson(path.join(paths.sitesDir, "xonly", "site.json"), {
+ siteId: "xonly",
+ siteTitle: "xonly",
+ siteDescription: "fixture",
+ headerTitle: "xonly",
+ homeTagline: "",
+ groups: [{ id: "default", name: "All channels", selectedByDefault: true }],
+ defaultGroupId: "default",
+ channels: [VIDEOS, X].map((slug) => ({ slug, groupId: "default" })),
+ archives: false,
+ });
+ await index();
+ const xonly = await compose("xonly");
+ // SearchSessionContext's hasPostsCorpus is `channels.length > 0`: no Posts
+ // toggle on this site, with nothing special-cased.
+ assert.deepEqual(xonly.postsManifest.channels, []);
+ assert.deepEqual(xonly.postTrees, []);
+ assert.equal(xonly.corpus.postScheme, undefined);
+ assert.deepEqual(slugs(xonly.corpus.channels), [VIDEOS]);
+});
diff --git a/common/bin/compose-site.ts b/common/bin/compose-site.ts
@@ -60,6 +60,9 @@ import {
type ArchiveManifest,
type ArchiveManifestEntry,
} from "../lib/archiveOptions";
+import { readChannelConfig } from "../controller/channels";
+import { publishedMemberSlugs } from "../lib/postsVisibility";
+import { isPrivateSite } from "../lib/siteSchema";
import { runIfEntryPoint } from "./_cli";
import { copyPublicFile, ownDir, writePublicFile } from "./_publicFile";
@@ -98,7 +101,10 @@ async function emitFederationFiles(
// navigate the already-served paginated shards; they never enumerate per-video
// files, so the count is constant regardless of corpus size. Runs after
// site.json and the archives are composed (both feed into these files).
-async function emitAiFiles(paths: ReturnType<typeof getPaths>): Promise<void> {
+async function emitAiFiles(
+ site: Site,
+ paths: ReturnType<typeof getPaths>,
+): Promise<void> {
const sitePath = path.join(paths.exportPublicDir, "site.json");
if (!(await exists(sitePath))) return; // no composed data → nothing to describe
const descriptor = JSON.parse(
@@ -161,6 +167,9 @@ async function emitAiFiles(paths: ReturnType<typeof getPaths>): Promise<void> {
postCounts,
digestCounts,
hasTags,
+ // A private site says so in its own corpus.json — the bundle's word that
+ // the deploy guard (lib/builtExport.ts builtAudienceProblem) reads.
+ private: isPrivateSite(site),
});
await writePublicFile(
path.join(paths.exportPublicDir, "corpus.json"),
@@ -665,7 +674,28 @@ export async function main(
}
const paths = opts.paths ?? getPaths();
const site = getSite(siteId, paths);
- const memberSlugs = site.channels.map((c) => c.slug);
+ // The members this site may publish (lib/postsVisibility.ts): every member,
+ // less an X channel while `social.x.visibility` is "private" and the site is
+ // public. The index build's per-site manifests are narrowed by the same rule;
+ // narrowing the trees here prunes an X channel a public site shipped before.
+ const configs = new Map(
+ await Promise.all(
+ site.channels.map(
+ async (c) => [c.slug, await readChannelConfig(paths, c.slug).catch(() => null)] as const,
+ ),
+ ),
+ );
+ const memberSlugs = publishedMemberSlugs(
+ site,
+ (slug) => configs.get(slug),
+ getSettings(),
+ );
+ const withheld = site.channels.length - memberSlugs.length;
+ if (withheld > 0) {
+ console.log(
+ `[compose] ${withheld} X channel(s) left out: X posts are private (social.x.visibility) and this site is public.`,
+ );
+ }
// Incremental compose: skip stages whose source is unchanged since last build.
const cachePath = composeCachePath(paths, siteId);
@@ -934,7 +964,7 @@ export async function main(
// --- AI discovery: llms.txt / corpus.json / robots.txt / sitemap.xml ---
// (after site.json + archives — both feed into these fixed-count files)
- await emitAiFiles(paths);
+ await emitAiFiles(site, paths);
// Persist the incremental-compose signatures for the next build.
await writeComposeCache(cachePath, cache);
diff --git a/common/controller/buildIndex.ts b/common/controller/buildIndex.ts
@@ -141,6 +141,7 @@ import {
postsAvailabilityPath,
} from "../lib/posts-server";
import { isSocialChannel } from "../lib/channelConfig";
+import { publishedMemberSlugs } from "../lib/postsVisibility";
import {
DIGESTS_MANIFEST_VERSION,
SITE_DIGESTS_MANIFEST_VERSION,
@@ -1940,8 +1941,21 @@ export async function buildIndex({
let aggregateSummaries = 0;
let representativeChannelCount = 0;
+ // Which members a site may publish: an X channel is built only into private
+ // sites while `social.x.visibility` is "private" (lib/postsVisibility.ts —
+ // compose-site narrows its trees by the same rule).
+ const visibilitySettings = getSettings();
for (const site of sites) {
- const memberSlugs = site.channels.map((c) => c.slug);
+ const memberSlugs = publishedMemberSlugs(
+ site,
+ (slug) => channelConfigs.get(slug),
+ visibilitySettings,
+ );
+ // Members the rule leaves out of THIS build, named in the fingerprint below
+ // so flipping the setting (or the site's audience) rebuilds the site.
+ const withheld = site.channels
+ .map((c) => c.slug)
+ .filter((slug) => !memberSlugs.includes(slug));
const slugSet = new Set(memberSlugs);
const slugGroup = new Map<string, string>();
for (const m of site.channels) {
@@ -1972,6 +1986,9 @@ export async function buildIndex({
// yesterday's counts.
curatedRules: curated.rulesHash,
curatedAssign: curated.assignHash,
+ // Only when the visibility rule withholds a member, so a site it does
+ // not touch keeps the fingerprint it had.
+ ...(withheld.length > 0 ? { withheld } : {}),
});
const fpKey = `siteFp:${site.siteId}`;
if (
diff --git a/common/controller/poolSummary.test.ts b/common/controller/poolSummary.test.ts
@@ -26,3 +26,26 @@ test("channel-sites.json names listed sites only; a channel only an unlisted sit
});
assert.ok(!JSON.stringify(channelSitesOf(sites)).includes("fixture-unlisted"));
});
+
+// Release 17 slice XP: a PRIVATE site is never listed, so it is in neither; and
+// with X posts private an X channel a public site leaves out of its build is
+// not mapped to that site (buildPoolSummary passes the narrowing).
+test("channel-sites.json leaves out a private site, and an X channel its public site withholds", async () => {
+ const { publishedMemberSlugs } = await import("../lib/postsVisibility");
+ const sites = [
+ parseSite("fixture-a", { channels: [{ slug: "vids" }, { slug: "jer-x" }] }),
+ parseSite("fixture-private", {
+ audience: "private",
+ channels: [{ slug: "vids" }, { slug: "jer-x" }, { slug: "own" }],
+ }),
+ ];
+ const configs: Record<string, { sourceKind?: "social"; platform?: "twitter" }> = {
+ "jer-x": { sourceKind: "social", platform: "twitter" },
+ };
+ const privateX = { social: { x: { visibility: "private" } } };
+ assert.deepEqual(
+ channelSitesOf(sites, (site) => publishedMemberSlugs(site, (slug) => configs[slug], privateX)),
+ { vids: ["fixture-a"] },
+ );
+ assert.deepEqual(channelSitesOf(sites), { vids: ["fixture-a"], "jer-x": ["fixture-a"] });
+});
diff --git a/common/controller/poolSummary.ts b/common/controller/poolSummary.ts
@@ -12,6 +12,9 @@ import { mkdir, readFile } from "node:fs/promises";
import type { Paths } from "../lib/paths";
import { buildStats } from "./buildStats";
import { isListedSite, listSites, type Site } from "../lib/site";
+import { getSettings } from "../lib/settings";
+import { publishedMemberSlugs } from "../lib/postsVisibility";
+import { readChannelConfig } from "./channels";
import {
statsPageFileName,
type StatsManifest,
@@ -49,12 +52,21 @@ export async function readStatsPages(statsDir: string): Promise<VideoStat[]> {
// published `channel-sites.json`. A channel on multiple sites maps to all of
// them; a pool-only channel is simply absent, and so is an unlisted site
// (site.json `listed: false`) and a channel only unlisted sites expose.
-export function channelSitesOf(sites: readonly Site[]): ChannelSitesMap {
+//
+// `membersOf` answers the members a site's build publishes; absent, every
+// member. buildPoolSummary passes lib/postsVisibility.ts's narrowing, so an X
+// channel a public site leaves out while X posts are private is not mapped to
+// that site here either.
+export function channelSitesOf(
+ sites: readonly Site[],
+ membersOf: (site: Site) => readonly string[] = (site) =>
+ site.channels.map((c) => c.slug),
+): ChannelSitesMap {
const channelSites: ChannelSitesMap = {};
for (const site of sites) {
if (!isListedSite(site)) continue;
- for (const c of site.channels) {
- (channelSites[c.slug] ??= []).push(site.siteId);
+ for (const slug of membersOf(site)) {
+ (channelSites[slug] ??= []).push(site.siteId);
}
}
return channelSites;
@@ -79,7 +91,15 @@ export async function buildPoolSummary(opts: {
// side effect, which is harmless.
await buildStats({ paths, wholePoolStatsDir: statsDir });
const sites = listSites(paths);
- const channelSites = channelSitesOf(sites);
+ // The members each site's build publishes (lib/postsVisibility.ts).
+ const configs = new Map<string, Awaited<ReturnType<typeof readChannelConfig>>>();
+ for (const slug of new Set(sites.flatMap((s) => s.channels.map((c) => c.slug)))) {
+ configs.set(slug, await readChannelConfig(paths, slug).catch(() => null));
+ }
+ const settings = getSettings();
+ const channelSites = channelSitesOf(sites, (site) =>
+ publishedMemberSlugs(site, (slug) => configs.get(slug), settings),
+ );
const stats = await readStatsPages(statsDir);
const summary = buildHomepageSummary(
stats,
diff --git a/common/lib/builtExport.ts b/common/lib/builtExport.ts
@@ -98,6 +98,63 @@ export function builtBundleProblem(outDir: string, siteId: string): string | nul
return null;
}
+/**
+ * Why `site` may not be deployed because of who it is built for, as one
+ * sentence — or null when it may (release 17 slice XP).
+ *
+ * A PRIVATE site (`site.json` `audience: "private"`) is the operator's own
+ * reading copy: it may carry what the public may not (X posts while
+ * `social.x.visibility` is "private"), so no deploy path ships it. Every
+ * deploy path asks this BEFORE ANY UPLOAD — the R2 archive push included — in
+ * the place it asks builtBundleProblem: runDeployIntoLog, the container deploy
+ * phase, deploySite, and the editor's Build & deploy, Deploy and Build &
+ * deploy all. A build without a deploy is untouched.
+ */
+export function siteDeployProblem(site: {
+ siteId: string;
+ audience?: string;
+}): string | null {
+ if (site.audience !== "private") return null;
+ return (
+ `Site "${site.siteId}" is private (audience: private): it is built for reading ` +
+ `on this machine and is never deployed. Build it without deploying, or set its ` +
+ `audience to public on its Settings tab`
+ );
+}
+
+/**
+ * Why the bundle in `outDir` may not be deployed because it was built PRIVATE
+ * — its corpus.json says `"audience": "private"` (compose-site writes it for a
+ * private site) — as one sentence, or null. Asked beside siteDeployProblem, so
+ * a site switched to public after a private build still cannot ship that
+ * build: it is rebuilt first.
+ */
+export function builtAudienceProblem(outDir: string): string | null {
+ try {
+ const parsed: unknown = JSON.parse(readFileSync(path.join(outDir, "corpus.json"), "utf8"));
+ const site = (parsed as { site?: { audience?: unknown; id?: unknown } } | null)?.site;
+ if (site?.audience !== "private") return null;
+ const id = typeof site.id === "string" ? ` of "${site.id}"` : "";
+ return (
+ `${outDir} holds a private build${id} (its corpus.json says "audience": "private"), ` +
+ `which is never deployed`
+ );
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * Both audience refusals, the site's first: the one sentence a deploy path
+ * logs or throws, or null.
+ */
+export function deployAudienceProblem(
+ site: { siteId: string; audience?: string },
+ outDir: string,
+): string | null {
+ return siteDeployProblem(site) ?? builtAudienceProblem(outDir);
+}
+
// corpus.json's `site.id`, or null when there is no readable one.
function corpusSiteIdIn(outDir: string): string | null {
try {
diff --git a/common/lib/corpus.ts b/common/lib/corpus.ts
@@ -175,6 +175,9 @@ export type SiteCorpus = {
description: string;
url?: string;
hubUrl?: string;
+ // Present only on a PRIVATE site's build (site.json `audience`, release 17
+ // slice XP): the operator's own reading copy, which no deploy path ships.
+ audience?: "private";
};
totals: { channels: number; videos: number };
channels: CorpusChannel[];
@@ -229,6 +232,9 @@ export function buildSiteCorpus(
// visible tag has a non-zero count here). Absent/false leaves corpus.json
// shaped as before apart from the spec bump.
hasTags?: boolean;
+ // A private site's build (site.json `audience: "private"`): corpus.json's
+ // `site.audience` says so. Absent/false leaves corpus.json as before.
+ private?: boolean;
},
): SiteCorpus {
const base = descriptor.siteUrl;
@@ -268,6 +274,7 @@ export function buildSiteCorpus(
description: descriptor.siteDescription,
...(descriptor.siteUrl ? { url: descriptor.siteUrl } : {}),
...(descriptor.hubUrl ? { hubUrl: descriptor.hubUrl } : {}),
+ ...(opts.private ? { audience: "private" as const } : {}),
},
totals: { channels: channels.length, videos },
channels,
diff --git a/common/lib/postsVisibility.test.ts b/common/lib/postsVisibility.test.ts
@@ -0,0 +1,99 @@
+// The one rule for which sites an X channel's posts are built into (release 17
+// slice XP). The build that applies it is pinned by
+// bin/compose-site.postsVisibility.test.ts.
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ isXPostsChannel,
+ postsVisibleTo,
+ publishedMemberSlugs,
+ xPostsVisibility,
+} from "./postsVisibility";
+import { isListedSite, parseSite, siteToDisk } from "./siteSchema";
+import { sanitizeSocial } from "../social/xCookieSource";
+
+const X = { sourceKind: "social" as const, platform: "twitter" as const };
+const BLUESKY = { sourceKind: "social" as const, platform: "bluesky" as const };
+const VIDEOS = { platform: "youtube" as const };
+const PRIVATE_X = { social: { x: { visibility: "private" } } };
+const PUBLIC_X = { social: { x: { visibility: "public" } } };
+const publicSite = { audience: undefined };
+const privateSite = { audience: "private" as const };
+
+test("an X channel is a social channel on platform twitter, and nothing else is", () => {
+ assert.equal(isXPostsChannel(X), true);
+ assert.equal(isXPostsChannel(BLUESKY), false);
+ assert.equal(isXPostsChannel(VIDEOS), false);
+ // A video channel on X (were there one) is not a posts channel.
+ assert.equal(isXPostsChannel({ platform: "twitter" }), false);
+ assert.equal(isXPostsChannel(null), false);
+ assert.equal(isXPostsChannel(undefined), false);
+});
+
+test("social.x.visibility: absent and unknown read as public", () => {
+ assert.equal(xPostsVisibility({}), "public");
+ assert.equal(xPostsVisibility({ social: { x: {} } }), "public");
+ assert.equal(xPostsVisibility({ social: { x: { visibility: "hidden" } } }), "public");
+ assert.equal(xPostsVisibility(PUBLIC_X), "public");
+ assert.equal(xPostsVisibility(PRIVATE_X), "private");
+});
+
+test("public: X posts go wherever the channel is a member", () => {
+ for (const settings of [{}, PUBLIC_X]) {
+ assert.equal(postsVisibleTo(publicSite, X, settings), true);
+ assert.equal(postsVisibleTo(privateSite, X, settings), true);
+ }
+});
+
+test("private: X posts go to private sites only; every other channel is untouched", () => {
+ assert.equal(postsVisibleTo(publicSite, X, PRIVATE_X), false);
+ assert.equal(postsVisibleTo({}, X, PRIVATE_X), false);
+ assert.equal(postsVisibleTo(privateSite, X, PRIVATE_X), true);
+ for (const site of [publicSite, privateSite]) {
+ assert.equal(postsVisibleTo(site, BLUESKY, PRIVATE_X), true);
+ assert.equal(postsVisibleTo(site, VIDEOS, PRIVATE_X), true);
+ assert.equal(postsVisibleTo(site, null, PRIVATE_X), true);
+ }
+});
+
+test("publishedMemberSlugs narrows the membership, in its order", () => {
+ const configs: Record<string, object> = { vids: VIDEOS, x: X, sky: BLUESKY };
+ const channels = [{ slug: "x" }, { slug: "vids" }, { slug: "sky" }, { slug: "gone" }];
+ const configOf = (slug: string) => configs[slug] as typeof X | undefined;
+ assert.deepEqual(
+ publishedMemberSlugs({ channels }, configOf, PRIVATE_X),
+ ["vids", "sky", "gone"],
+ );
+ assert.deepEqual(
+ publishedMemberSlugs({ channels, audience: "private" }, configOf, PRIVATE_X),
+ ["x", "vids", "sky", "gone"],
+ );
+ assert.deepEqual(
+ publishedMemberSlugs({ channels }, configOf, {}),
+ ["x", "vids", "sky", "gone"],
+ );
+});
+
+test("site.json audience: only private is kept and written; a private site is never listed", () => {
+ assert.equal(parseSite("a", {}).audience, undefined);
+ assert.equal(parseSite("a", { audience: "public" }).audience, undefined);
+ assert.equal(parseSite("a", { audience: "secret" }).audience, undefined);
+ const priv = parseSite("a", { audience: "private" });
+ assert.equal(priv.audience, "private");
+ assert.equal(siteToDisk(priv).audience, "private");
+ assert.equal("audience" in siteToDisk(parseSite("a", {})), false);
+ assert.equal(isListedSite(priv), false);
+ assert.equal(isListedSite({ ...priv, listed: true }), false);
+ assert.equal(isListedSite(parseSite("a", {})), true);
+});
+
+test("sanitizeSocial keeps visibility beside cookieSource and drops an unknown one", () => {
+ assert.deepEqual(sanitizeSocial({ x: { visibility: "private" } }), {
+ x: { visibility: "private" },
+ });
+ assert.deepEqual(
+ sanitizeSocial({ x: { cookieSource: "browser", visibility: "public" } }),
+ { x: { cookieSource: "browser", visibility: "public" } },
+ );
+ assert.deepEqual(sanitizeSocial({ x: { visibility: "nobody" } }), { x: {} });
+});
diff --git a/common/lib/postsVisibility.ts b/common/lib/postsVisibility.ts
@@ -0,0 +1,75 @@
+// WHICH SITES A CHANNEL'S POSTS ARE BUILT INTO (release 17 slice XP).
+//
+// THE ONE RULE, asked by both places a site's posts are decided:
+// - the index build's per-site loop (controller/buildIndex.ts), which writes
+// each site's summaries, subs, posts and digests manifests from the site's
+// member channels;
+// - the site compose (bin/compose-site.ts), which copies the shared
+// per-channel trees (transcripts, subs, posts, digests) of those members
+// into the served public dir and writes /site.json and /corpus.json.
+// Both narrow the site's members through `publishedMemberSlugs`, so the
+// manifests and the trees can never disagree about a channel.
+//
+// The rule: with `social.x.visibility` "private", an X channel (a social
+// channel on platform "twitter") is built only into a PRIVATE site
+// (`site.json` `audience: "private"`). Posts are all a social channel holds — it
+// has no videos (buildIndex's scan skips it) — so a public site leaves the
+// channel out whole: no posts tree, no posts-manifest entry, no empty channel
+// in its channel list or its corpus.json. Every other channel, and every
+// channel on a private site, is unaffected. Nothing on disk changes and
+// fetching does not: the shared posts tree (exportSharedPostsDir) and the LMDB
+// posts sub-DB are corpus-wide and keep every channel, which is what a private
+// site and the MCP over a private build read.
+//
+// The Search in row's Posts toggle needs no rule of its own: it shows only
+// when the site's posts manifest lists a channel (SearchSessionContext
+// hasPostsCorpus), so a public site whose only posts were X posts ships an
+// empty posts manifest and no Posts toggle.
+//
+// Pure: no fs, no settings read. The callers pass the settings and the configs.
+
+import { isSocialChannel, type ChannelConfig } from "./channelConfig";
+import { isPrivateSite, type Site } from "./siteSchema";
+import type { XPostsVisibility } from "../social/xCookieSource";
+
+// An X channel: the posts of a social channel whose platform is X.
+export function isXPostsChannel(
+ config: Pick<ChannelConfig, "sourceKind" | "platform"> | null | undefined,
+): boolean {
+ return isSocialChannel(config) && config?.platform === "twitter";
+}
+
+// `social.x.visibility`, resolved: absent (or anything unknown) is "public".
+export function xPostsVisibility(settings: {
+ social?: { x?: { visibility?: unknown } };
+}): XPostsVisibility {
+ return settings.social?.x?.visibility === "private" ? "private" : "public";
+}
+
+// Whether `site` may carry the posts of the channel `config` describes. A
+// channel with no readable config is not an X channel as far as this rule
+// knows, and is left to whatever already decides its fate.
+export function postsVisibleTo(
+ site: Pick<Site, "audience">,
+ config: Pick<ChannelConfig, "sourceKind" | "platform"> | null | undefined,
+ settings: { social?: { x?: { visibility?: unknown } } },
+): boolean {
+ if (!isXPostsChannel(config)) return true;
+ if (xPostsVisibility(settings) === "public") return true;
+ return isPrivateSite(site);
+}
+
+// The site's member slugs, in membership order, less the channels whose posts
+// it may not carry (an X channel holds nothing else). `configOf` answers a
+// slug's channel config, or null/undefined when it has none.
+export function publishedMemberSlugs(
+ site: Pick<Site, "audience" | "channels">,
+ configOf: (
+ slug: string,
+ ) => Pick<ChannelConfig, "sourceKind" | "platform"> | null | undefined,
+ settings: { social?: { x?: { visibility?: unknown } } },
+): string[] {
+ return site.channels
+ .map((c) => c.slug)
+ .filter((slug) => postsVisibleTo(site, configOf(slug), settings));
+}
diff --git a/common/lib/settingsSchema.ts b/common/lib/settingsSchema.ts
@@ -116,6 +116,15 @@ export const X_SOCIAL_SETTINGS_FIELD_DOCS: FieldDocs<XSocialSettings> = {
"stored: `\"browser\"` when `cookiesFromBrowser` is set and no profile is connected (no " +
"exported jar carrying an auth_token), else `\"profile\"`. The source, not `cookieMode`, " +
"governs the X fetchers.",
+ visibility:
+ "Where X posts may appear. `\"public\"` (the default; absent): an X channel's posts are " +
+ "built into every site that has the channel. `\"private\"`: every X channel's posts (a " +
+ "channel with `sourceKind: \"social\"` and `platform: \"twitter\"`) are left out of every " +
+ "PUBLIC site build — the channel with them, since posts are all an X channel holds — and " +
+ "built only into PRIVATE sites (`site.json` `audience`). Nothing on disk changes and " +
+ "fetching does not; a site already published changes on its next build and deploy, and " +
+ "flipping back is a rebuild. Chosen in the X account session section of /settings; the " +
+ "rule is common/lib/postsVisibility.ts.",
};
export type { AutoQueueSettings } from "./autoQueueTypes";
export type { ChannelPriority } from "./channelPriority";
@@ -1477,7 +1486,7 @@ export const siteSettingsSchema = z.object({
"How yt-dlp invocations use the configured cookies (see common/lib/cookiePolicy.ts): \"always\" passes them on every invocation, \"when-required\" (default; the historical behavior) only to retry an auth/age failure, \"defer\" never in normal runs — auth-gated videos are excluded from batches and collected into the per-channel \"Needs cookies\" bucket for a manual cookie run. Per-channel override available (ChannelConfig.cookieMode).",
),
social: settingsField((v): SocialSettings => sanitizeSocial(v)).describe(
- "Per-platform settings of the social-post fetchers. Today one key: where the X fetchers' login comes from (`social.x.cookieSource`, chosen in the X account session section of /settings). See common/social/xCookieSource.ts.",
+ "Per-platform settings of the social posts. Today two keys, both X's, both chosen in the X account session section of /settings: where the X fetchers' login comes from (`social.x.cookieSource`) and where X posts may appear (`social.x.visibility`). See common/social/xCookieSource.ts.",
),
sleepBetweenDownloadsSeconds: settingsField((v): number => clampSleepBetweenDownloadsSeconds(v)).describe(
"Pause (seconds) inserted between per-video yt-dlp invocations in managed batch downloads. yt-dlp's own `-t sleep` only paces requests within one invocation, so without this the managed loop hammers the source IP back-to-back. 0 disables. Per-channel override available.",
diff --git a/common/lib/site.ts b/common/lib/site.ts
@@ -14,6 +14,7 @@ import { socialLinksForSave } from "./socialLinks";
import { readJsonFileSync, writeJsonAtomic } from "./jsonFile-server";
import {
isListedSite,
+ isPrivateSite,
isValidSiteId,
parseSite,
parseSiteUrl,
@@ -87,11 +88,14 @@ export function siteStatsDir(paths: Paths, siteId: string): string {
}
// The hub URL this site points visitors toward: its own override, else the
-// family default (SiteSettings.homepageUrl). Undefined when neither is set.
+// family default (SiteSettings.homepageUrl). Undefined when neither is set —
+// and always for a PRIVATE site (`audience: "private"`), which belongs under no
+// hub: a hub tells its members by the hubUrl they publish.
export function resolveHubUrl(
site: Site,
settings: SiteSettings = getSettings(),
): string | undefined {
+ if (isPrivateSite(site)) return undefined;
return site.hubUrl ?? parseSiteUrl(settings.homepageUrl);
}
diff --git a/common/lib/siteSchema.ts b/common/lib/siteSchema.ts
@@ -68,6 +68,27 @@ export const RELATED_SITE_GROUP_FIELD_DOCS: FieldDocs<RelatedSiteGroup> = {
"Sibling site ids, in display order. Invalid and repeated ids are dropped, and a group left with none is dropped. Ids are resolved against the live pool at render time, so an id for a site that does not exist (yet) is harmless — it is skipped.",
};
+// Who a site is built for (release 17 slice XP). "public" (the default, never
+// written) is every site there has ever been. "private" is the operator's own
+// reading copy: never deployed (publish/build.ts asks siteDeployProblem in
+// lib/builtExport.ts before any upload), never listed (isListedSite below),
+// publishing no hubUrl (lib/site.ts resolveHubUrl), and the only kind of site
+// that content kept from the public (X posts while `social.x.visibility` is
+// "private", lib/postsVisibility.ts) is built into.
+export type SiteAudience = "public" | "private";
+
+export const SITE_AUDIENCES: readonly SiteAudience[] = ["public", "private"];
+
+export function isSiteAudience(v: unknown): v is SiteAudience {
+ return v === "public" || v === "private";
+}
+
+// THE ONE PREDICATE for a private site. Absent or anything but "private" reads
+// as public.
+export function isPrivateSite(site: Pick<Site, "audience">): boolean {
+ return site.audience === "private";
+}
+
// A Site is a selection + presentation layer over the single global channel
// pool. Each field is documented in SITE_FIELD_DOCS below.
export type Site = {
@@ -85,6 +106,7 @@ export type Site = {
accent?: string;
siteUrl?: string;
listed?: boolean;
+ audience?: SiteAudience;
relatedSites?: RelatedSiteGroup[];
pwa?: boolean;
archives?: boolean;
@@ -119,6 +141,8 @@ export const SITE_FIELD_DOCS: FieldDocs<Site> = {
"Absolute public URL of this site's deployment, e.g. `https://jeralyzer.pages.dev` (trimmed, trailing slashes removed; anything not absolute http(s) is dropped). Drives the cross-site footer: a site with no siteUrl is omitted from every other site's list.",
listed:
"Whether the family lists this site. Opt-OUT: absent/true = listed, only an explicit `false` is written. An unlisted site still builds and deploys as before, and its own pages are unchanged; it is left out of the homepage (cards, chart, `/stats`), the hub (members, federated search, `/corpus.json`, `/llms.txt`), every other site's footer, and the published `channel-sites.json` and pooled `stats/`. A channel only unlisted sites expose is in none of the family's public totals; a channel a listed site also exposes is credited to the listed one.",
+ audience:
+ 'Who this site is built for. `"public"` (the default; absent) or `"private"`: the operator\'s own reading copy, built on this machine and never deployed — every deploy path (Build & deploy, Deploy, `archilyzer deploy site`, Build & deploy all, docker/publish-site.sh) refuses it before any upload, while a build without a deploy still works. A private site is never listed (as `listed: false`, whatever `listed` says), publishes no `hubUrl`, and its `/corpus.json` says `"audience": "private"`. Content kept from the public — X posts while `social.x.visibility` is `"private"` — is built only into private sites. Only `"private"` is written.',
relatedSites:
"Pulls specific siblings to the front of the footer's cross-site list, in named groups. Siblings not named here fall into a trailing \"Other sites\" group. Absent/empty = one flat list of every sibling.",
pwa:
@@ -150,8 +174,11 @@ export function isValidSiteId(id: unknown): id is string {
// (lib/site.ts resolveRelatedSites). The editor's own pages list every site.
// Here, beside the key, and exported from lib/site like isValidSiteId, so the
// pure summary builder can use it without importing file I/O.
-export function isListedSite(site: Pick<Site, "listed">): boolean {
- return site.listed !== false;
+//
+// A PRIVATE site (`audience: "private"`) is never listed, whatever `listed`
+// says: it is never deployed, so there is nothing at its URL to list.
+export function isListedSite(site: Pick<Site, "listed" | "audience">): boolean {
+ return site.listed !== false && !isPrivateSite(site);
}
// The channels whose content belongs to unlisted sites alone: exposed by at
@@ -160,7 +187,7 @@ export function isListedSite(site: Pick<Site, "listed">): boolean {
// and a channel no site exposes (pool-only) is not here either — the family's
// instance-wide totals have always counted it.
export function channelsOnlyOnUnlistedSites(
- sites: readonly Pick<Site, "listed" | "channels">[],
+ sites: readonly Pick<Site, "listed" | "audience" | "channels">[],
): Set<string> {
const onListed = new Set<string>();
const onUnlisted = new Set<string>();
@@ -281,6 +308,10 @@ export const siteFieldsSchema = z.object({
siteUrl: settingsField(parseSiteUrl).describe(d.siteUrl),
// Opt-out: only an explicit false unlists. Absent/true stays listed.
listed: settingsField((v): boolean => v !== false).describe(d.listed),
+ // Only "private" is kept; absent (and anything else) is the public default.
+ audience: settingsField((v): SiteAudience | undefined =>
+ v === "private" ? "private" : undefined,
+ ).describe(d.audience),
relatedSites: settingsField(parseRelatedSites).describe(d.relatedSites),
pwa: settingsField((v): boolean => v === true).describe(d.pwa),
// Opt-out: only an explicit false disables. Absent/true stays on.
@@ -372,6 +403,8 @@ export function siteToDisk(site: Site): Site {
...(siteUrl ? { siteUrl } : {}),
// Listed is the default: only the opt-out is persisted.
...(site.listed === false ? { listed: false } : {}),
+ // Public is the default: only the private audience is persisted.
+ ...(isPrivateSite(site) ? { audience: "private" as const } : {}),
...(relatedSites.length > 0 ? { relatedSites } : {}),
...(site.pwa ? { pwa: true } : {}),
// Persist only the non-default: archives is on unless explicitly disabled.
diff --git a/common/publish/build.test.ts b/common/publish/build.test.ts
@@ -23,6 +23,7 @@ import {
homepageDeployArgs,
homepageOutDir,
deployHomepage,
+ deploySite,
dockerSiteOutDir,
dockerSiteStagingDir,
resolveOutDir,
@@ -393,3 +394,120 @@ test("runDeployIntoLog refuses a bundle that is not the site's own before wrangl
rmSync(root, { recursive: true, force: true });
}
});
+
+// A PRIVATE site (site.json `audience: "private"`, release 17 slice XP) is never
+// deployed, and neither is a bundle built private (its corpus.json says so):
+// refused at the same door as the wrong-site bundle, before wrangler, in words
+// naming the audience. The fake `pnpm` is the test above's.
+function writePrivateBundle(dir: string, siteId: string): void {
+ writeBundle(dir, siteId, siteId);
+ writeFileSync(
+ path.join(dir, "corpus.json"),
+ JSON.stringify({ site: { id: siteId, audience: "private" } }),
+ );
+}
+
+test("runDeployIntoLog refuses a private site and a private build before wrangler", async () => {
+ const root = mkdtempSync(path.join(os.tmpdir(), "deploy-private-"));
+ const bin = path.join(root, "bin");
+ const argvFile = path.join(root, "pnpm-argv");
+ mkdirSync(bin);
+ writeFileSync(path.join(bin, "pnpm"), `#!/bin/sh\nprintf '%s\\n' "$@" >> '${argvFile}'\n`);
+ chmodSync(path.join(bin, "pnpm"), 0o755);
+ const savedPath = process.env.PATH;
+ process.env.PATH = bin;
+ const signal = new AbortController().signal;
+ const testPaths = { ...paths, exportDir: root } as Paths;
+ try {
+ await runChildIntoLog(() => {}, signal, { command: "pnpm", args: ["--fake?"], cwd: root, env: { ...process.env } });
+ assert.equal(readFileSync(argvFile, "utf8"), "--fake?\n");
+ rmSync(argvFile);
+
+ // The site is private: its own, well-formed bundle is still refused.
+ const own = path.join(root, "own", "out");
+ writeBundle(own, "mine", "mine");
+ const priv = { siteId: "mine", cloudflareProject: "w3c-never-real", audience: "private" } as Site;
+ let log: string[] = [];
+ assert.equal(await runDeployIntoLog((l) => log.push(l), signal, priv, own, testPaths), 1);
+ assert.equal(log.length, 1, log.join(""));
+ assert.match(
+ log[0],
+ /^\[deploy\] REFUSED — Site "mine" is private \(audience: private\): it is built for reading on this machine and is never deployed\./,
+ );
+ assert.match(log[0], /Nothing was sent to Cloudflare Pages\.\n$/);
+
+ // The site is public now, but the bundle was built private.
+ const built = path.join(root, "built", "out");
+ writePrivateBundle(built, "mine");
+ log = [];
+ const pub = { siteId: "mine", cloudflareProject: "w3c-never-real" } as Site;
+ assert.equal(await runDeployIntoLog((l) => log.push(l), signal, pub, built, testPaths), 1);
+ assert.match(log[0], /holds a private build of "mine" \(its corpus\.json says "audience": "private"\)/);
+ assert.equal(existsSync(argvFile), false, "pnpm was spawned");
+ } finally {
+ process.env.PATH = savedPath;
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("runDockerDeployAllPhase skips a private site before the upload, in the audience's words", async () => {
+ const root = mkdtempSync(path.join(os.tmpdir(), "deploy-all-private-"));
+ try {
+ const outFor = (id: string) => path.join(root, id, "out");
+ writeBundle(outFor("mine"), "mine", "mine");
+ writePrivateBundle(outFor("built"), "built");
+ const log: string[] = [];
+ // A Cloudflare project on each: without the audience check the run would
+ // reach the upload, which the log would show.
+ const sites = [
+ { siteId: "mine", audience: "private", cloudflareProject: "w3c-never-real" },
+ { siteId: "built", cloudflareProject: "w3c-never-real" },
+ ] as Site[];
+ const outcomes = await runDockerDeployAllPhase(
+ (l) => log.push(l),
+ new AbortController().signal,
+ sites,
+ new Set(["mine", "built"]),
+ { ...paths, exportBuildsDir: root } as Paths,
+ outFor,
+ );
+ assert.deepEqual(outcomes.map((o) => [o.siteId, o.status]), [["mine", "skipped"], ["built", "skipped"]]);
+ assert.match(outcomes[0].reason!, /^Site "mine" is private \(audience: private\)/);
+ assert.match(outcomes[1].reason!, /private build of "built"/);
+ assert.ok(log.some((l) => l.startsWith("[mine] deploy skipped — Site \"mine\" is private")), log.join("\n"));
+ assert.ok(!log.some((l) => l.startsWith("=== Deploy")), "nothing reached the deploy");
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("deploySite (archilyzer deploy site) refuses a private site before anything, and a private build before the upload", async () => {
+ const root = mkdtempSync(path.join(os.tmpdir(), "deploy-site-private-"));
+ try {
+ const sitesDir = path.join(root, "sites");
+ const site = (id: string, extra: Record<string, unknown> = {}) => {
+ mkdirSync(path.join(sitesDir, id), { recursive: true });
+ writeFileSync(
+ path.join(sitesDir, id, "site.json"),
+ JSON.stringify({ siteId: id, cloudflareProject: "w3c-never-real", ...extra }),
+ );
+ };
+ site("mine", { audience: "private" });
+ site("other");
+ const testPaths = { ...paths, exportDir: root, sitesDir } as Paths;
+ const log: string[] = [];
+ await assert.rejects(
+ deploySite("mine", { paths: testPaths, onLog: (l) => log.push(l) }),
+ /^Error: Site "mine" is private \(audience: private\): it is built for reading on this machine and is never deployed\. Build it without deploying, or set its audience to public on its Settings tab\.$/,
+ );
+ // Public, but export/out holds a private build of it.
+ writePrivateBundle(path.join(root, "out"), "other");
+ await assert.rejects(
+ deploySite("other", { paths: testPaths, onLog: (l) => log.push(l) }),
+ /private build of "other".*Build other again, then deploy\.$/,
+ );
+ assert.deepEqual(log, [], "nothing was logged: no upload, no deploy");
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
diff --git a/common/publish/build.ts b/common/publish/build.ts
@@ -14,7 +14,14 @@ import { createReadStream, existsSync } from "node:fs";
import { S3Client, HeadObjectCommand } from "@aws-sdk/client-s3";
import { Upload } from "@aws-sdk/lib-storage";
import { runChildIntoLog } from "../jobs/runChild";
-import { builtBundleProblem, builtHubProblem, builtSiteProblem } from "../lib/builtExport";
+import {
+ builtAudienceProblem,
+ builtBundleProblem,
+ builtHubProblem,
+ builtSiteProblem,
+ deployAudienceProblem,
+ siteDeployProblem,
+} from "../lib/builtExport";
import { getHomepageConfig } from "../lib/homepage";
import {
deploymentUrlIn,
@@ -340,6 +347,16 @@ export async function runDeployIntoLog(
// job (a build of another site, the hub) can rewrite export/out. Nothing runs
// between this check and the spawn. The hub and the homepage deploy through
// runPagesDeployIntoLog and never come here.
+ //
+ // A PRIVATE site, or a bundle built private, is refused first: it is never
+ // deployed, whatever the bundle's identity (deployAudienceProblem).
+ const audienceProblem = deployAudienceProblem(site, outDir);
+ if (audienceProblem) {
+ onLog(
+ `[deploy] REFUSED — ${audienceProblem}. Nothing was sent to Cloudflare Pages.\n`,
+ );
+ return 1;
+ }
const bundleProblem = builtBundleProblem(outDir, site.siteId);
if (bundleProblem) {
onLog(
@@ -608,6 +625,14 @@ export async function runDockerDeployAllPhase(
// the check build-site.sh makes before it hands the bundle back, made again
// over whatever the per-site dir holds now. First, so that nothing past it
// (the R2 upload, the Pages deploy) is ever reached with another site's data.
+ // A private site (or a private build) is not deployed at all: skipped, in
+ // its own words, so a family with one private site does not fail every run.
+ const audienceProblem = deployAudienceProblem(site, outDirFor(site.siteId));
+ if (audienceProblem) {
+ onLog(`[${site.siteId}] deploy skipped — ${audienceProblem}`);
+ outcomes.push({ siteId: site.siteId, status: "skipped", reason: audienceProblem });
+ continue;
+ }
const bundleProblem = builtBundleProblem(outDirFor(site.siteId), site.siteId);
if (bundleProblem) {
onLog(`[${site.siteId}] deploy REFUSED — ${bundleProblem}`);
@@ -753,6 +778,9 @@ export async function deploySite(
}
const branch = opts.previewBranch?.trim() || undefined;
const site = getSite(siteId.trim(), paths);
+ // Before anything else is asked of a private site: it is never deployed.
+ const privateProblem = siteDeployProblem(site);
+ if (privateProblem) throw new Error(`${privateProblem}.`);
if (!site.cloudflareProject) {
throw new Error(
`Site "${site.siteId}" has no Cloudflare Pages project configured.`,
@@ -761,6 +789,9 @@ export async function deploySite(
const outDir = resolveOutDir(site.siteId, paths);
const builtProblem = builtSiteProblem(outDir, site.siteId);
if (builtProblem) throw new Error(builtProblem);
+ // Before the R2 upload below: a bundle built private is never deployed.
+ const builtPrivate = builtAudienceProblem(outDir);
+ if (builtPrivate) throw new Error(`${builtPrivate}. Build ${site.siteId} again, then deploy.`);
// The production path logs no banner and gains none here: its log has
// always opened on wrangler's own first line.
if (branch) {
diff --git a/common/social/xCookieSource.ts b/common/social/xCookieSource.ts
@@ -27,11 +27,28 @@ export function isXCookieSource(v: unknown): v is XCookieSource {
return v === "browser" || v === "profile";
}
-// The `social` block of settings.json. Only X has a login to choose today; the
-// block is per platform so a second one does not need a second top-level key.
+// WHERE X POSTS MAY APPEAR — `social.x.visibility` (release 17 slice XP).
+// "public" — the default (absent): an X channel's posts are built into every
+// site that has the channel, as every other channel's are.
+// "private" — an X channel's posts are left out of every PUBLIC site build and
+// built only into PRIVATE sites (`site.json` `audience`). Nothing
+// on disk changes, and fetching does not; flipping back is a
+// rebuild. The rule itself is lib/postsVisibility.ts.
+export type XPostsVisibility = "public" | "private";
+
+export const X_POSTS_VISIBILITIES: readonly XPostsVisibility[] = ["public", "private"];
+
+export function isXPostsVisibility(v: unknown): v is XPostsVisibility {
+ return v === "public" || v === "private";
+}
+
+// The `social` block of settings.json. Only X has settings today; the block is
+// per platform so a second one does not need a second top-level key.
export type XSocialSettings = {
// Absent = the read-time default above.
cookieSource?: XCookieSource;
+ // Absent = "public".
+ visibility?: XPostsVisibility;
};
export type SocialSettings = {
@@ -39,7 +56,8 @@ export type SocialSettings = {
};
// Total over `unknown`, as every settings coercion is: anything that is not a
-// known source reads as absent (the default), and unknown keys are dropped.
+// known source or visibility reads as absent (the default), and unknown keys
+// are dropped.
export function sanitizeSocial(value: unknown): SocialSettings {
const r = (value && typeof value === "object" && !Array.isArray(value)
? value
@@ -48,7 +66,10 @@ export function sanitizeSocial(value: unknown): SocialSettings {
? r.x
: {}) as Record<string, unknown>;
return {
- x: isXCookieSource(x.cookieSource) ? { cookieSource: x.cookieSource } : {},
+ x: {
+ ...(isXCookieSource(x.cookieSource) ? { cookieSource: x.cookieSource } : {}),
+ ...(isXPostsVisibility(x.visibility) ? { visibility: x.visibility } : {}),
+ },
};
}
diff --git a/docker/publish-site.sh b/docker/publish-site.sh
@@ -29,6 +29,20 @@ if [ -z "${SITE_ID}" ]; then
fi
export SITE_ID
+
+# A PRIVATE site (site.json `audience: "private"`) is never deployed, and the
+# volume this script fills is what the `site` service serves: refused before
+# the build, and again over the built corpus.json below (lib/builtExport.ts).
+# Building it without publishing is `archilyzer build site <id>`.
+SITE_JSON="${SITES_DIR:-${TRANSCRIPTS_DIR:-/data/transcripts}/sites}/${SITE_ID}/site.json"
+refuse_private() {
+ echo "[publish-site] REFUSED — site '${SITE_ID}' is private (audience: private): it is built for reading on this machine and is never deployed. Nothing was published to ${SITE_OUT}. Build it without publishing: pnpm --filter yt-dlp-transcript-common exec tsx bin/archilyzer.ts build site ${SITE_ID}" >&2
+ exit 1
+}
+if grep -Eq '"audience"[[:space:]]*:[[:space:]]*"private"' "${SITE_JSON}" 2>/dev/null; then
+ refuse_private
+fi
+
cd /repo
echo "[publish-site] building '${SITE_ID}'"
@@ -37,6 +51,9 @@ echo "[publish-site] building '${SITE_ID}'"
pnpm --filter yt-dlp-transcript-common exec tsx bin/archilyzer.ts build site "${SITE_ID}"
[ -d /repo/export/out ] || { echo "[publish-site] no export/out after build" >&2; exit 1; }
+if grep -Eq '"audience"[[:space:]]*:[[:space:]]*"private"' /repo/export/out/corpus.json 2>/dev/null; then
+ refuse_private
+fi
echo "[publish-site] publishing -> ${SITE_OUT}"
mkdir -p "${SITE_OUT}"
diff --git a/export/app/offline/page.tsx b/export/app/offline/page.tsx
@@ -1,6 +1,8 @@
import type { Metadata } from "next";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels";
+import { getSettings } from "yt-dlp-transcript-common/lib/settings";
+import { postsVisibleTo } from "yt-dlp-transcript-common/lib/postsVisibility";
import { currentSite } from "../lib/site";
import { OfflineManager, type OfflineChannel } from "../components/OfflineManager";
@@ -15,12 +17,19 @@ export const metadata: Metadata = {
export default async function OfflinePage() {
const site = currentSite();
const paths = getPaths();
- const channels: OfflineChannel[] = await Promise.all(
- site.channels.map(async ({ slug }) => {
- const config = await readChannelConfig(paths, slug).catch(() => null);
- return { slug, name: config?.name ?? slug };
- }),
+ const settings = getSettings();
+ // The members this build publishes: an X channel is left out of a public
+ // site while X posts are private (lib/postsVisibility.ts, the rule the
+ // index build and compose narrow the site's data by).
+ const members = await Promise.all(
+ site.channels.map(async ({ slug }) => ({
+ slug,
+ config: await readChannelConfig(paths, slug).catch(() => null),
+ })),
);
+ const channels: OfflineChannel[] = members
+ .filter(({ config }) => postsVisibleTo(site, config, settings))
+ .map(({ slug, config }) => ({ slug, name: config?.name ?? slug }));
channels.sort((a, b) => a.name.localeCompare(b.name));
return (
