// What a site's (and the hub's) build reads, as one sha1 — computed by the // update-index stage and stored in the IndexStamp (release 18). // // THE RULE: a site is fresh exactly when compose AND the export build would // produce the same bundle. So the signature is made of what they read, signed // with compose's own `dirSignature` (lib/dirSignature.ts) and compose's own // manifest-only rule: // // - the site's whole `.export-index/sites//` tree (summaries, stats, // chart templates, tag counts, the subs/posts/digests manifests, the // report media and exports); // - each PUBLISHED member channel's shared transcripts / subs / posts / // digests tree, `manifest.json` ignored (its `generatedAt` churns); // - the bytes of `site.json`, and the signature of the site's config dir // (`sites//`: tags, aliases, reports); // - the corpus-wide files compose reads at compose time (search aliases, // curated tags, the duplicates report and its overrides), by size + mtime; // - the `archiveStorage`, `social.x.visibility` and `buildArchives` settings; // - what `next build` renders beyond compose (release 18 S1 review): the // resolved social links, the hub url, and the footer's sibling sites. // // A superset of those inputs is CONSERVATIVE: a needless rebuild, never a // wrong skip. `hubSig` = sha1(stampId, homepage.json, each listed // site's id + siteUrl + title). import { createHash } from "node:crypto"; import { existsSync } from "node:fs"; import { readFile, stat } from "node:fs/promises"; import path from "node:path"; import { open } from "lmdb"; import { dirSignature } from "../lib/dirSignature"; import { DUPLICATES_FILENAME, DUPLICATE_OVERRIDES_FILENAME } from "../lib/duplicates"; import type { Paths } from "../lib/paths"; import { publishedMemberSlugs } from "../lib/postsVisibility"; import type { SiteSettings } from "../lib/settings"; import { resolveHubUrl, resolveRelatedSites, resolveSocialLinks, siteConfigFile, siteDir, siteIndexDir, type Site, } from "../lib/site"; import { isListedSite } from "../lib/siteSchema"; import { INDEX_SCANNED_AT_KEY } from "../lib/stats"; import { readChannelConfig } from "../controller/channels"; import type { ChannelConfig } from "../lib/channelConfig"; // compose-site.ts MANIFEST_ONLY_SIGNATURE — a tree holding only its manifest is // a channel with nothing in it, signed by a constant (compose's rule). const MANIFEST_ONLY = "manifest-only"; // lib/chartsStore.ts siteTemplatesStagingPath's basename. const CHART_TEMPLATES = "chart-templates.json"; const sha1 = (s: string | Buffer) => createHash("sha1").update(s).digest("hex"); async function fileBytesSig(file: string): Promise { try { return sha1(await readFile(file)); } catch { return ""; } } async function fileStatSig(file: string): Promise { try { const s = await stat(file); return `${s.size}\t${s.mtimeMs}`; } catch { return ""; } } /** A memo over the shared per-channel trees: each is walked once per stamp. */ export type TreeSigCache = Map; async function channelTreeSig(root: string, slug: string, cache: TreeSigCache): Promise { const dir = path.join(root, slug); const hit = cache.get(dir); if (hit !== undefined) return hit; let sig = await dirSignature(dir, "manifest.json"); if (sig === "" && existsSync(path.join(dir, "manifest.json"))) sig = MANIFEST_ONLY; cache.set(dir, sig); return sig; } export type SiteSigInputs = { paths: Paths; site: Site; settings: Pick; // Every configured site: the footer's sibling list is part of the bundle. sites: Site[]; // The site's members' configs (the visibility rule reads their platform). configOf: (slug: string) => ChannelConfig | null | undefined; cache?: TreeSigCache; }; /** The site's inputSig (see the header). */ export async function siteInputSig(i: SiteSigInputs): Promise { const { paths, site, settings } = i; const cache = i.cache ?? new Map(); const members = [...publishedMemberSlugs(site, i.configOf, settings)].sort(); const lines: string[] = ["inputSig v1"]; lines.push(`members\t${members.join(",")}`); // `build templates` rewrites chart-templates.json on every run (and compose // copies it on every run): signed by its bytes, not its mtime. const indexDir = siteIndexDir(paths, site.siteId); lines.push(`site-index\t${await dirSignature(indexDir, CHART_TEMPLATES)}`); lines.push(`${CHART_TEMPLATES}\t${await fileBytesSig(path.join(indexDir, CHART_TEMPLATES))}`); for (const slug of members) { for (const [tree, root] of [ ["transcripts", paths.exportSharedTranscriptsDir], ["subs", paths.exportSharedSubsDir], ["posts", paths.exportSharedPostsDir], ["digests", paths.exportSharedDigestsDir], ] as const) { lines.push(`${tree}/${slug}\t${await channelTreeSig(root, slug, cache)}`); } } lines.push(`site.json\t${await fileBytesSig(siteConfigFile(paths, site.siteId))}`); lines.push(`site-dir\t${await dirSignature(siteDir(paths, site.siteId))}`); for (const file of [ paths.globalAliasesFile, paths.globalTagsFile, path.join(paths.transcriptsDir, DUPLICATES_FILENAME), path.join(paths.transcriptsDir, DUPLICATE_OVERRIDES_FILENAME), ]) { lines.push(`${path.basename(file)}\t${await fileStatSig(file)}`); } lines.push( `settings\t${JSON.stringify({ archiveStorage: settings.archiveStorage ?? null, xVisibility: settings.social?.x?.visibility ?? null, buildArchives: settings.buildArchives ?? null, })}`, ); // What the export BUILD renders from beyond compose's inputs, resolved as it // resolves them: the header/footer social links, the hub link the descriptor // carries, and the footer's sibling sites (their urls, titles, listing). const full = settings as SiteSettings; lines.push(`social-links\t${JSON.stringify(resolveSocialLinks(site, full) ?? null)}`); lines.push(`hub-url\t${resolveHubUrl(site, full) ?? ""}`); lines.push(`related-sites\t${JSON.stringify(resolveRelatedSites(site, i.sites))}`); return sha1(lines.join("\n")); } /** The settings the index stage reads (see indexSettingsSig). */ export type IndexSettingsInputs = Pick< SiteSettings, | "socialLinks" | "homepageUrl" | "buildArchives" | "archiveStorage" | "social" | "maxTranscriptPageBytes" | "storage" >; /** * sha1 over the settings keys the index stage and the builds it signs read — * NOT the settings file's mtime, which every pause click, priority change and * drive auto-pause moves (release 18 S4 step 1). The keys: the social links and * the homepage url (the builds render them), `buildArchives`, `archiveStorage`, * `social.x.visibility` (which members a site publishes), the transcript page * size (the index shards by it) and the storage locations' roots (which * channels the index can reach). A superset is conservative: a short-circuited * index update, never a missed one. The charts config is its own file, judged * by its mtime. */ export function indexSettingsSig(settings: Partial): string { return sha1( JSON.stringify({ v: 1, socialLinks: settings.socialLinks ?? null, homepageUrl: settings.homepageUrl ?? null, buildArchives: settings.buildArchives ?? null, archiveStorage: settings.archiveStorage ?? null, xVisibility: settings.social?.x?.visibility ?? null, maxTranscriptPageBytes: settings.maxTranscriptPageBytes ?? null, locations: (settings.storage?.locations ?? []).map((l) => [l.id, l.root]), }), ); } /** Every member's channel config, read once (unreadable = null). */ export async function readMemberConfigs( paths: Paths, sites: Site[], ): Promise> { const slugs = new Set(sites.flatMap((s) => s.channels.map((c) => c.slug))); const out = new Map(); await Promise.all( [...slugs].map(async (slug) => { out.set(slug, await readChannelConfig(paths, slug).catch(() => null)); }), ); return out; } /** The hub's signature: the index it was built from, its config, the pool it lists. */ export async function hubInputSig( paths: Paths, stampId: string, sites: Site[], ): Promise { const lines = [`hubSig v1`, `stamp\t${stampId}`]; lines.push(`homepage.json\t${await fileBytesSig(paths.homepageConfigFile)}`); for (const s of [...sites].sort((a, b) => a.siteId.localeCompare(b.siteId))) { if (!isListedSite(s) || !s.siteUrl) continue; lines.push(`site\t${s.siteId}\t${s.siteUrl}\t${s.siteTitle}`); } return sha1(lines.join("\n")); } export type IndexMeta = { generation: number; scannedAt: number | null; // sha1 of each site's stored fingerprints, null when absent. siteFp: Record; statsFp: Record; }; /** * The LMDB index's own bookkeeping, read-only, after the index and stats * builds have closed it: `generation`, INDEX_SCANNED_AT_KEY and each site's * `siteFp:` / `statsFp:`. An absent index reads as zeros. */ export function readIndexMeta(paths: Paths, siteIds: string[]): IndexMeta { const out: IndexMeta = { generation: 0, scannedAt: null, siteFp: {}, statsFp: {} }; for (const id of siteIds) { out.siteFp[id] = null; out.statsFp[id] = null; } if (!existsSync(paths.lmdbPath)) return out; const root = open({ path: paths.lmdbPath, readOnly: true, maxDbs: 18, compression: true }); try { const meta = root.openDB({ name: "meta", encoding: "msgpack" }); // An index that stats never ran over has no statsMeta sub-DB. let statsMeta: typeof meta | null = null; try { statsMeta = root.openDB({ name: "statsMeta", encoding: "msgpack" }); } catch { statsMeta = null; } const gen = meta.get("generation"); out.generation = typeof gen === "number" ? gen : 0; const scanned = meta.get(INDEX_SCANNED_AT_KEY); out.scannedAt = typeof scanned === "number" ? scanned : null; for (const id of siteIds) { const fp = meta.get(`siteFp:${id}`); const sfp = statsMeta?.get(`statsFp:${id}`); out.siteFp[id] = typeof fp === "string" ? sha1(fp) : null; out.statsFp[id] = typeof sfp === "string" ? sha1(sfp) : null; } } finally { root.close(); } return out; }