#!/usr/bin/env tsx // Composes the served public dir for the Archilyzer HUB build (the export app // built with INSTANCE_MODE=hub) ahead of `next build`. Unlike compose-site.ts // there is no SITE_ID and no per-site data: the hub is a federating shell that // reads every archive cross-origin at runtime. It emits: // // public/hub-sites.json <- the built-in trusted pool (listSites with a siteUrl, // listed — site.json `listed`, isListedSite; never // a private or a cited report site) // public/hub-summary.json <- the official instances' numbers, the homepage's // own (lib/hubSummary.ts) — OPTIONAL: skipped when // there is no index to walk // public/_headers <- CORS for the hub's own served JSON // public/sw.js <- the hub service worker (the hub always ships a PWA) // public/search-aliases.json <- the global alias dictionary // public/posts/ <- TOMBSTONES only, while X posts are private: // an empty posts/manifest.json and, per X // channel with a shared posts tree that a // non-private site carries, its manifest at // pageCount 0 and `[]` pages // (publish/tombstones.ts), served no-store // // and REMOVES every per-site entry a site's compose left in public/ // (SITE_ONLY_PUBLIC_ENTRIES below): the hub holds no site's data. // // The hub's branding ("Archilyzer") is resolved at build/render time from the // HomepageConfig (see export/app/lib/site.ts hubSite()), not composed here. import path from "node:path"; import { existsSync } from "node:fs"; import { cp, rm, access } from "node:fs/promises"; import { getPaths, type Paths } from "../lib/paths"; import { accentHex } from "../lib/accent"; import { isListedSite, listSites, resolveHubUrl } from "../lib/site"; import { getHomepageConfig } from "../lib/homepage"; import { SITE_DESCRIPTOR_VERSION } from "../lib/siteDescriptor"; import { buildHubCorpus, renderHubLlmsTxt, renderRobotsTxt, type HubMemberInput, } from "../lib/corpus"; import { HUB_CORS_PATHS, renderHeadersFile } from "../lib/archive/headers"; import { buildPoolSummary } from "../controller/poolSummary"; import { HUB_SUMMARY_FILE, toHubSummary } from "../lib/hubSummary"; import { readGlobalAliases } from "../lib/aliasesStore"; import { getSettings } from "../lib/settings"; import { emptyPostsManifest, tombstoneNoStoreForHub, withdrawnXChannels, writePostsTombstones, type PostsTombstone, } from "../publish/tombstones"; import { runIfEntryPoint } from "./_cli"; import { writePublicFile } from "./_publicFile"; async function exists(p: string): Promise { try { await access(p); return true; } catch { return false; } } // The official instances' figures, from the SAME buildHomepageSummary call the // homepage's compose makes (controller/poolSummary.ts), projected to what the // hub's cards read. Whole-pool stats pages land in the export staging dir (not // served), never in public/. // // Optional by contract: with no index to walk (a checkout with no corpus, or a // test) the file is not written and any stale one is removed, so the hub shows // its cards without figures rather than last build's numbers. A failure here // never fails the hub build — the shelf degrades, it does not break. async function composeHubSummary( paths: Paths, publicDir: string, ): Promise { const dest = path.join(publicDir, HUB_SUMMARY_FILE); if (!existsSync(paths.lmdbPath)) { await rm(dest, { force: true }); return "no index to summarise — hub-summary.json skipped"; } try { const { summary } = await buildPoolSummary({ paths, statsDir: path.join(paths.exportIndexDir, "hub-stats"), }); const hubSummary = toHubSummary(summary); await writePublicFile(dest, JSON.stringify(hubSummary)); return `hub-summary.json covers ${hubSummary.sites.length} official instance(s)`; } catch (err) { await rm(dest, { force: true }); return `hub-summary.json skipped: ${err instanceof Error ? err.message : String(err)}`; } } // THE HUB CARRIES NO SITE'S DATA (release 17 slice XP, the review's HIGH 1). // public/ is the one directory every site composes into in turn, and the hub // builds from it next: whatever the last site's compose left there — its data // trees and its per-site files — `next build` copied into the hub's out/, and // the hub deploy shipped it. The live hub served jeralyzer's posts manifest; // after a PRIVATE site's build it would have served every X post. So the hub's // compose removes every per-site entry first. A worktree's public/ entries are // links into the primary checkout: rm removes the link, never its target. export const SITE_ONLY_PUBLIC_ENTRIES: readonly string[] = [ "summaries", "transcripts", "subs", "posts", "digests", "stats", "archives", // A report site's reports, moments and cited media (publish/composeReports.ts). "reports", "m", "media", "site.json", "tags.json", "duplicates.json", "search-aliases.json", "chart-templates.json", "sitemap.xml", ]; // THE HUB'S TOMBSTONES (release 18). A hub built over a site's compose once // shipped that site's posts (the review's HIGH 1 above), and Cloudflare's edge // kept serving them after the deploy that removed them. Removing is not enough // at the edge, so while X posts are private the hub REPLACES every path an X // channel's posts could have been served from: an empty posts manifest, and per // X channel with a shared posts tree that a non-private site carries, its // manifest at pageCount 0 and an empty page for each page the shared tree holds // now — all served no-store. A channel only private sites (or no site) carry is // never named here: private data is never named on the hub. With X // posts public, or no X channel, the hub ships no posts/ at all, as before. async function composeHubTombstones( paths: Paths, publicDir: string, settings: { social?: { x?: { visibility?: unknown } } }, ): Promise { const slugs = await withdrawnXChannels(paths, settings, listSites(paths)); if (slugs.length === 0) return []; const postsDir = path.join(publicDir, "posts"); const tombstones = await writePostsTombstones({ postsDir, sharedPostsDir: paths.exportSharedPostsDir, slugs, }); await writePublicFile( path.join(postsDir, "manifest.json"), JSON.stringify(emptyPostsManifest(new Date().toISOString())), ); return tombstones; } export async function main( opts: { paths?: Paths; settings?: { social?: { x?: { visibility?: unknown } } } } = {}, ): Promise { const paths = opts.paths ?? getPaths(); const publicDir = paths.exportPublicDir; for (const entry of SITE_ONLY_PUBLIC_ENTRIES) { await rm(path.join(publicDir, entry), { recursive: true, force: true }); } const tombstones = await composeHubTombstones(paths, publicDir, opts.settings ?? getSettings()); // The hub's own alias dictionary is the global one (no site's overrides): // what a hub reader loads for hub-wide search (lib/archive/reader-hub.ts), // where it used to get whichever site had composed last. await writePublicFile( path.join(publicDir, "search-aliases.json"), JSON.stringify({ aliases: readGlobalAliases(paths).aliases }), ); // Built-in pool: every configured site that publishes a public URL and is // listed. An unlisted site (`listed: false`) still builds and deploys, but the // hub does not list it: not a member, not in federated search, not in the // hub's corpus.json or llms.txt (both are built from this list). The entry // the hub registry loads at boot to seed its trusted built-in pool, and the // input buildHubCorpus maps — ONE type for both (lib/archive/contract.ts // HubMemberInput), where this file used to restate it as a local // `HubSiteEntry` with the same fields under a different name. The origin is // derived from siteUrl on the client (coerceBuiltin), so a pool site without // a siteUrl can't be federated and is dropped — by both consumers. const builtins: HubMemberInput[] = []; for (const site of listSites(paths)) { if (!site.siteUrl || !isListedSite(site)) continue; builtins.push({ siteId: site.siteId, siteTitle: site.siteTitle, siteUrl: site.siteUrl, // The hub draws it as a colour: an accent id is published as its hex. ...(accentHex(site.accent) ? { accent: accentHex(site.accent) } : {}), ...(resolveHubUrl(site) ? { hubUrl: resolveHubUrl(site) } : {}), pwa: site.pwa === true, contract: SITE_DESCRIPTOR_VERSION, }); } // Every file below is written with writePublicFile, never through a link: // in a worktree these paths are links into the primary checkout, whose live // hub files a worktree compose (e2e:2origin's build:hub) used to overwrite. await writePublicFile( path.join(publicDir, "hub-sites.json"), JSON.stringify(builtins), ); // Aggregate AI-discovery surface for the federation (fixed file count): a hub // corpus.json directory of member origins + llms.txt + robots.txt. The hub // holds no shard data — these point at each member's own corpus.json. const hub = getHomepageConfig(paths); const hubCorpus = buildHubCorpus(builtins, { hubTitle: hub.siteTitle, hubUrl: hub.siteUrl, generatedAt: new Date().toISOString(), }); await writePublicFile( path.join(publicDir, "corpus.json"), JSON.stringify(hubCorpus), ); await writePublicFile( path.join(publicDir, "llms.txt"), renderHubLlmsTxt(hubCorpus), ); await writePublicFile( path.join(publicDir, "robots.txt"), renderRobotsTxt({ siteUrl: hub.siteUrl }), ); await writePublicFile( path.join(publicDir, "_headers"), renderHeadersFile("compose-hub.ts", HUB_CORS_PATHS, { noStore: tombstoneNoStoreForHub(tombstones), }), ); // The hub always ships a PWA. Copy the hub service worker into place. Until // the dedicated hub SW lands (Phase 7), fall back to the site SW so the PWA // still registers. const swDest = path.join(publicDir, "sw.js"); await rm(swDest, { force: true }); const hubSw = path.join(paths.monorepoRoot, "export", "service-worker", "sw-hub.js"); const siteSw = path.join(paths.monorepoRoot, "export", "service-worker", "site-sw.js"); const swSrc = (await exists(hubSw)) ? hubSw : siteSw; if (await exists(swSrc)) await cp(swSrc, swDest); const summaryNote = await composeHubSummary(paths, publicDir); const tombstoneNote = tombstones.length > 0 ? `; ${tombstones.length} withdrawn X channel(s) shipped as tombstones, served no-store` : ""; console.log( `compose-hub: ${builtins.length} built-in pool site(s) into ${publicDir}; ${summaryNote}${tombstoneNote}.`, ); } runIfEntryPoint(import.meta.url, () => main());