Archilyzer · Source

archilyzer

Archilyzer
git clone https://archilyzer.pages.dev/source/archilyzer.git
Log | Files | Refs | README | LICENSE

commit 695cec67147f4d3b004dd0dcfc794dd0843cc770
parent 40a70b8e8dcc5e760ec69fcbfb437e1e7c6efe4b
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date:   Fri, 25 Sep 2026 18:44:31 -0400

common: compose-hub writes hub-summary.json — the homepage's official totals and per-site card figures, from the same buildHomepageSummary call (controller/poolSummary.ts); optional, skipped with no index

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Diffstat:
Mcommon/bin/compose-homepage.ts | 62+++++++++++---------------------------------------------------
Acommon/bin/compose-hub.test.ts | 55+++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mcommon/bin/compose-hub.ts | 42+++++++++++++++++++++++++++++++++++++++++-
Mcommon/bin/compose-site.ts | 2++
Acommon/controller/poolSummary.ts | 87+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Acommon/lib/hubSummary.test.ts | 141+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Acommon/lib/hubSummary.ts | 164+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mcommon/publish/build.ts | 3++-
8 files changed, 503 insertions(+), 53 deletions(-)

diff --git a/common/bin/compose-homepage.ts b/common/bin/compose-homepage.ts @@ -12,20 +12,9 @@ // prebuild chains it), same as the export pipeline. import path from "node:path"; -import { mkdir, readFile } from "node:fs/promises"; import { getPaths, type Paths } from "../lib/paths"; import { writeJsonAtomic as writeJsonAtomicShared } from "../lib/jsonFile-server"; -import { buildStats } from "../controller/buildStats"; -import { listSites } from "../lib/site"; -import { - statsPageFileName, - type StatsManifest, - type VideoStat, -} from "../lib/stats"; -import { - buildHomepageSummary, - type ChannelSitesMap, -} from "../lib/homepageSummary"; +import { buildPoolSummary } from "../controller/poolSummary"; import { runIfEntryPoint } from "./_cli"; // Where the homepage Next.js app serves static assets from. Overridable for e2e @@ -42,59 +31,30 @@ function writeJsonAtomic(filePath: string, value: unknown): Promise<void> { return writeJsonAtomicShared(filePath, value, { indent: 0, newline: false }); } -// Read the whole-pool stats dataset back from the pages buildStats just wrote, so -// the lightweight homepage summary can be derived from the same records without -// re-extracting. Pages are local files (fast) capped well under memory limits. -async function readStatsPages(statsDir: string): Promise<VideoStat[]> { - let manifest: StatsManifest; - try { - manifest = JSON.parse( - await readFile(path.join(statsDir, "manifest.json"), "utf8"), - ) as StatsManifest; - } catch { - return []; - } - const out: VideoStat[] = []; - for (let i = 0; i < manifest.pageCount; i++) { - const page = JSON.parse( - await readFile(path.join(statsDir, statsPageFileName(i)), "utf8"), - ) as VideoStat[]; - out.push(...page); - } - return out; -} - export async function main(opts: { paths?: Paths } = {}): Promise<void> { const paths = opts.paths ?? getPaths(); const publicDir = homepagePublicDir(paths.monorepoRoot); const statsDir = path.join(publicDir, "stats"); - await mkdir(statsDir, { recursive: true }); - // Whole-pool stats dataset (every non-excluded channel) for the cross-site - // charts. buildStats also refreshes the per-site bundles as a side effect, - // which is harmless. - await buildStats({ paths, wholePoolStatsDir: statsDir }); + // Whole-pool stats (every non-excluded channel) for the cross-site charts, + // then the summary over them. The hub's cards read a projection of the SAME + // call (compose-hub.ts, lib/hubSummary.ts) — the input gathering lives in + // controller/poolSummary.ts so the two cannot drift. + const { summary, channelSites, sites } = await buildPoolSummary({ + paths, + statsDir, + }); // channel slug -> the ids of the content sites that expose it. Drives - // `groupBy: "site"` in the hub's dashboard (see channelSites.tsx). A channel - // on multiple sites maps to all of them; a pool-only channel is simply absent. - const sites = listSites(paths); - const channelSites: ChannelSitesMap = {}; - for (const site of sites) { - for (const c of site.channels) { - (channelSites[c.slug] ??= []).push(site.siteId); - } - } + // `groupBy: "site"` in the hub's dashboard (see channelSites.tsx). await writeJsonAtomic( path.join(publicDir, "channel-sites.json"), channelSites, ); - // Lightweight cross-site summary for the hub landing page (pre-binned monthly + // Lightweight cross-site summary for the landing page (pre-binned monthly // counts, per-site totals/sparklines, recent additions). Embedded into the // SSG HTML so the front page never fetches the full stats dataset. - const stats = await readStatsPages(statsDir); - const summary = buildHomepageSummary(stats, channelSites, sites, new Date()); await writeJsonAtomic( path.join(publicDir, "homepage-summary.json"), summary, diff --git a/common/bin/compose-hub.test.ts b/common/bin/compose-hub.test.ts @@ -0,0 +1,55 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { getPaths, type Paths } from "../lib/paths"; +import { main } from "./compose-hub"; + +// Run with: +// pnpm --filter yt-dlp-transcript-common test +// +// hub-summary.json is OPTIONAL: a hub composed with no index to walk ships no +// numbers — and must not ship the LAST build's numbers either. + +function fixturePaths(root: string): Paths { + const sitesDir = path.join(root, "sites"); + const homepageDir = path.join(sitesDir, "_homepage"); + mkdirSync(homepageDir, { recursive: true }); + const exportPublicDir = path.join(root, "public"); + mkdirSync(exportPublicDir, { recursive: true }); + return { + ...getPaths(), + transcriptsDir: root, + channelsDir: path.join(root, "channels"), + sitesDir, + homepageDir, + homepageConfigFile: path.join(homepageDir, "homepage.json"), + homepageChartTemplatesFile: path.join(homepageDir, "chart-templates.json"), + lmdbPath: path.join(root, "index.mdb"), // never created: no index + exportPublicDir, + exportIndexDir: path.join(root, ".export-index"), + }; +} + +test("with no index, compose-hub writes the pool and no hub-summary.json, removing a stale one", async () => { + const root = mkdtempSync(path.join(tmpdir(), "compose-hub-")); + const log = console.log; + try { + const paths = fixturePaths(root); + const stale = path.join(paths.exportPublicDir, "hub-summary.json"); + writeFileSync(stale, '{"version":1,"sites":[]}'); + console.log = () => {}; + await main({ paths }); + console.log = log; + assert.ok(!existsSync(stale), "a stale summary must not survive"); + assert.deepEqual( + JSON.parse(readFileSync(path.join(paths.exportPublicDir, "hub-sites.json"), "utf8")), + [], + ); + assert.ok(!existsSync(paths.lmdbPath), "composing must not create an index"); + } finally { + console.log = log; + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/common/bin/compose-hub.ts b/common/bin/compose-hub.ts @@ -5,6 +5,9 @@ // reads every archive cross-origin at runtime. It emits: // // public/hub-sites.json <- the built-in trusted pool (listSites with a siteUrl) +// public/hub-summary.json <- the official instances' numbers, the homepage's +// own (lib/hubSummary.ts) — OPTIONAL: skipped when +// there is no index to walk // public/_headers <- CORS for the hub's own served JSON // public/sw.js <- the hub service worker (the hub always ships a PWA) // @@ -12,6 +15,7 @@ // HomepageConfig (see export/app/lib/site.ts hubSite()), not composed here. import path from "node:path"; +import { existsSync } from "node:fs"; import { cp, rm, writeFile, access } from "node:fs/promises"; import { getPaths, type Paths } from "../lib/paths"; import { listSites, resolveHubUrl } from "../lib/site"; @@ -24,6 +28,8 @@ import { type HubMemberInput, } from "../lib/corpus"; import { HUB_CORS_PATHS, renderHeadersFile } from "../lib/archive/headers"; +import { buildPoolSummary } from "../controller/poolSummary"; +import { HUB_SUMMARY_FILE, toHubSummary } from "../lib/hubSummary"; import { runIfEntryPoint } from "./_cli"; async function exists(p: string): Promise<boolean> { @@ -35,6 +41,38 @@ async function exists(p: string): Promise<boolean> { } } +// The official instances' figures, from the SAME buildHomepageSummary call the +// homepage's compose makes (controller/poolSummary.ts), projected to what the +// hub's cards read. Whole-pool stats pages land in the export staging dir (not +// served), never in public/. +// +// Optional by contract: with no index to walk (a checkout with no corpus, or a +// test) the file is not written and any stale one is removed, so the hub shows +// its cards without figures rather than last build's numbers. A failure here +// never fails the hub build — the shelf degrades, it does not break. +async function composeHubSummary( + paths: Paths, + publicDir: string, +): Promise<string> { + const dest = path.join(publicDir, HUB_SUMMARY_FILE); + if (!existsSync(paths.lmdbPath)) { + await rm(dest, { force: true }); + return "no index to summarise — hub-summary.json skipped"; + } + try { + const { summary } = await buildPoolSummary({ + paths, + statsDir: path.join(paths.exportIndexDir, "hub-stats"), + }); + const hubSummary = toHubSummary(summary); + await writeFile(dest, JSON.stringify(hubSummary)); + return `hub-summary.json covers ${hubSummary.sites.length} official instance(s)`; + } catch (err) { + await rm(dest, { force: true }); + return `hub-summary.json skipped: ${err instanceof Error ? err.message : String(err)}`; + } +} + export async function main(opts: { paths?: Paths } = {}): Promise<void> { const paths = opts.paths ?? getPaths(); const publicDir = paths.exportPublicDir; @@ -101,8 +139,10 @@ export async function main(opts: { paths?: Paths } = {}): Promise<void> { const swSrc = (await exists(hubSw)) ? hubSw : siteSw; if (await exists(swSrc)) await cp(swSrc, swDest); + const summaryNote = await composeHubSummary(paths, publicDir); + console.log( - `compose-hub: ${builtins.length} built-in pool site(s) into ${publicDir}.`, + `compose-hub: ${builtins.length} built-in pool site(s) into ${publicDir}; ${summaryNote}.`, ); } diff --git a/common/bin/compose-site.ts b/common/bin/compose-site.ts @@ -914,6 +914,8 @@ export async function main( // out/ and makes the bundle ambiguous to builtHubProblem (and to a site's // own registry, which treats the file as a hub's trusted pool). await rm(path.join(paths.exportPublicDir, "hub-sites.json"), { force: true }); + // …and neither is the hub's copy of the official instances' numbers. + await rm(path.join(paths.exportPublicDir, "hub-summary.json"), { force: true }); // --- service worker (only when this instance ships a PWA) --- await composeServiceWorker(site, paths); diff --git a/common/controller/poolSummary.ts b/common/controller/poolSummary.ts @@ -0,0 +1,87 @@ +// The homepage summary's inputs, gathered ONCE for both consumers: the homepage +// (compose-homepage writes the whole summary) and the hub (compose-hub writes +// the `official` + per-site projection, lib/hubSummary.ts). Both call +// buildPoolSummary, so the two sites cannot disagree about a number. +// +// NOT read-only: buildStats keeps its incremental per-video state in the index +// LMDB (`statsByPath`), and writes the whole-pool stats pages into `statsDir`. +// Requires build:index to have populated the cues sub-DB first. + +import path from "node:path"; +import { mkdir, readFile } from "node:fs/promises"; +import type { Paths } from "../lib/paths"; +import { buildStats } from "./buildStats"; +import { listSites, type Site } from "../lib/site"; +import { + statsPageFileName, + type StatsManifest, + type VideoStat, +} from "../lib/stats"; +import { + buildHomepageSummary, + type ChannelSitesMap, + type HomepageSummary, +} from "../lib/homepageSummary"; + +// Read the whole-pool stats dataset back from the pages buildStats just wrote, so +// the lightweight summary can be derived from the same records without +// re-extracting. Pages are local files (fast) capped well under memory limits. +export async function readStatsPages(statsDir: string): Promise<VideoStat[]> { + let manifest: StatsManifest; + try { + manifest = JSON.parse( + await readFile(path.join(statsDir, "manifest.json"), "utf8"), + ) as StatsManifest; + } catch { + return []; + } + const out: VideoStat[] = []; + for (let i = 0; i < manifest.pageCount; i++) { + const page = JSON.parse( + await readFile(path.join(statsDir, statsPageFileName(i)), "utf8"), + ) as VideoStat[]; + out.push(...page); + } + return out; +} + +// channel slug -> the ids of the content sites that expose it. A channel on +// multiple sites maps to all of them; a pool-only channel is simply absent. +export function channelSitesOf(sites: Site[]): ChannelSitesMap { + const channelSites: ChannelSitesMap = {}; + for (const site of sites) { + for (const c of site.channels) { + (channelSites[c.slug] ??= []).push(site.siteId); + } + } + return channelSites; +} + +export type PoolSummary = { + summary: HomepageSummary; + channelSites: ChannelSitesMap; + sites: Site[]; +}; + +// Refresh the whole-pool stats into `statsDir`, then summarise them. +export async function buildPoolSummary(opts: { + paths: Paths; + statsDir: string; + now?: Date; +}): Promise<PoolSummary> { + const { paths, statsDir } = opts; + await mkdir(statsDir, { recursive: true }); + // Whole-pool stats dataset (every non-excluded channel). buildStats also + // refreshes the per-site bundles as a side effect, which is harmless. + await buildStats({ paths, wholePoolStatsDir: statsDir }); + const sites = listSites(paths); + const channelSites = channelSitesOf(sites); + const stats = await readStatsPages(statsDir); + const summary = buildHomepageSummary( + stats, + channelSites, + sites, + opts.now ?? new Date(), + ); + return { summary, channelSites, sites }; +} diff --git a/common/lib/hubSummary.test.ts b/common/lib/hubSummary.test.ts @@ -0,0 +1,141 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + HUB_SUMMARY_VERSION, + hubSummarySiteFor, + parseHubSummary, + toHubSummary, +} from "./hubSummary"; +import type { HomepageSummary } from "./homepageSummary"; + +// Run with: +// pnpm --filter yt-dlp-transcript-common test + +const stat = (total: number) => ({ total, thisMonth: 0, last12: [] }); + +function homepageSummary(): HomepageSummary { + return { + version: 5, + generatedAt: "2026-09-25T00:00:00.000Z", + totals: { + transcripts: 99, + downloads: 99, + sites: 2, + channels: 9, + hoursArchived: 999, + transcribedThisMonth: 0, + downloadedThisMonth: 0, + }, + channels: [], + series: {} as HomepageSummary["series"], + sites: [ + { + siteId: "alpha", + siteTitle: "Alpha", + siteDescription: "The first archive.", + siteUrl: "https://alpha.example", + transcribed: stat(30), + downloaded: stat(40), + channels: 3, + recordings: 40, + hoursArchived: 12, + gone: 1, + accent: "#123456", + }, + { + // A pre-v5 site: no per-site figures beyond the transcript count. + siteId: "beta", + siteTitle: "Beta", + siteDescription: "", + siteUrl: "https://beta.example", + transcribed: stat(5), + downloaded: stat(6), + }, + ], + recent: [], + official: { + sites: 2, + channels: 3, + recordings: 40, + transcripts: 35, + hoursArchived: 12, + gone: 1, + }, + }; +} + +test("toHubSummary projects the homepage's official totals and card figures", () => { + const hub = toHubSummary(homepageSummary()); + assert.equal(hub.version, HUB_SUMMARY_VERSION); + assert.equal(hub.generatedAt, "2026-09-25T00:00:00.000Z"); + assert.deepEqual(hub.official, homepageSummary().official); + assert.deepEqual(hub.sites, [ + { + siteId: "alpha", + siteTitle: "Alpha", + siteDescription: "The first archive.", + siteUrl: "https://alpha.example", + channels: 3, + recordings: 40, + transcripts: 30, + hoursArchived: 12, + gone: 1, + accent: "#123456", + }, + { + siteId: "beta", + siteTitle: "Beta", + siteUrl: "https://beta.example", + transcripts: 5, + }, + ]); +}); + +test("a summary without `official` projects to null, not zeros", () => { + const s = homepageSummary(); + delete s.official; + assert.equal(toHubSummary(s).official, null); +}); + +test("parseHubSummary round-trips what toHubSummary writes", () => { + const hub = toHubSummary(homepageSummary()); + assert.deepEqual(parseHubSummary(JSON.parse(JSON.stringify(hub))), hub); +}); + +test("parseHubSummary degrades instead of throwing", () => { + assert.equal(parseHubSummary(null), null); + assert.equal(parseHubSummary("nope"), null); + assert.equal(parseHubSummary([]), null); + // A version this reader does not know may mean something else. + assert.equal(parseHubSummary({ version: 99, sites: [] }), null); + + const parsed = parseHubSummary({ + version: HUB_SUMMARY_VERSION, + generatedAt: 7, + official: { sites: 1 }, // incomplete -> null, never half a total + sites: [ + { siteId: "ok", siteTitle: "OK", siteUrl: "https://ok.example", channels: -1, recordings: "3", transcripts: 4 }, + { siteId: "no-url", siteTitle: "No URL" }, + "junk", + ], + }); + assert.deepEqual(parsed, { + version: HUB_SUMMARY_VERSION, + generatedAt: "", + official: null, + sites: [ + { siteId: "ok", siteTitle: "OK", siteUrl: "https://ok.example", transcripts: 4 }, + ], + }); +}); + +test("hubSummarySiteFor matches by siteId, then by origin", () => { + const hub = toHubSummary(homepageSummary()); + assert.equal(hubSummarySiteFor(hub, { siteId: "alpha" })?.siteTitle, "Alpha"); + assert.equal( + hubSummarySiteFor(hub, { siteId: "renamed", origin: "https://beta.example" })?.siteId, + "beta", + ); + assert.equal(hubSummarySiteFor(hub, { siteId: "gamma", origin: "https://gamma.example" }), null); + assert.equal(hubSummarySiteFor(null, { siteId: "alpha" }), null); +}); diff --git a/common/lib/hubSummary.ts b/common/lib/hubSummary.ts @@ -0,0 +1,164 @@ +// The hub's build-time numbers: the homepage's "Official instances" figures, +// shipped to the hub as `public/hub-summary.json` so its cards say exactly what +// the homepage's cards say. +// +// Why build time and not a live sum: a member's public `corpus.json` carries +// channel and video counts only — no recordings-with-a-download-date, no hours, +// no "gone" — so a live sum could never match the homepage. Both files are +// projections of ONE `buildHomepageSummary` call over the same stats walk +// (controller/poolSummary.ts), and the hub is rebuilt alongside the sites. +// +// The file is OPTIONAL. An older hub build, or a hub composed where there is no +// index to walk, has none; the hub then renders its cards without figures — +// never an error. It is read same-origin, so it is not in HUB_CORS_PATHS, and +// the deploy guard (builtHubProblem) does not require it. +// +// Pure and client-safe: the hub page parses it in the browser. + +import type { + HomepageOfficialTotals, + HomepageSummary, +} from "./homepageSummary"; + +export const HUB_SUMMARY_FILE = "hub-summary.json"; +export const HUB_SUMMARY_VERSION = 1; + +// One official instance's card figures. Every figure is optional: the homepage +// summary's v5 per-site fields are, and a card omits what it does not have. +export type HubSummarySite = { + siteId: string; + siteTitle: string; + siteDescription?: string; + siteUrl: string; + channels?: number; + recordings?: number; + transcripts?: number; // the homepage summary's `transcribed.total` + hoursArchived?: number; + gone?: number; + accent?: string; +}; + +export type HubSummary = { + version: number; + generatedAt: string; + official: HomepageOfficialTotals | null; + sites: HubSummarySite[]; +}; + +// The projection: only what the hub's cards and H1 read, from the SAME summary +// object compose-homepage writes. +export function toHubSummary(summary: HomepageSummary): HubSummary { + return { + version: HUB_SUMMARY_VERSION, + generatedAt: summary.generatedAt, + official: summary.official ?? null, + sites: summary.sites.map((s) => ({ + siteId: s.siteId, + siteTitle: s.siteTitle, + ...(s.siteDescription ? { siteDescription: s.siteDescription } : {}), + siteUrl: s.siteUrl, + ...(s.channels !== undefined ? { channels: s.channels } : {}), + ...(s.recordings !== undefined ? { recordings: s.recordings } : {}), + transcripts: s.transcribed.total, + ...(s.hoursArchived !== undefined + ? { hoursArchived: s.hoursArchived } + : {}), + ...(s.gone !== undefined ? { gone: s.gone } : {}), + ...(s.accent ? { accent: s.accent } : {}), + })), + }; +} + +function isObject(v: unknown): v is Record<string, unknown> { + return typeof v === "object" && v !== null && !Array.isArray(v); +} + +function count(v: unknown): number | undefined { + return typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : undefined; +} + +function text(v: unknown): string | undefined { + return typeof v === "string" && v.trim() ? v : undefined; +} + +function parseOfficial(v: unknown): HomepageOfficialTotals | null { + if (!isObject(v)) return null; + const keys = [ + "sites", + "channels", + "recordings", + "transcripts", + "hoursArchived", + "gone", + ] as const; + const out = {} as HomepageOfficialTotals; + for (const k of keys) { + const n = count(v[k]); + if (n === undefined) return null; + out[k] = n; + } + return out; +} + +function parseSite(v: unknown): HubSummarySite | null { + if (!isObject(v)) return null; + const siteId = text(v.siteId); + const siteTitle = text(v.siteTitle); + const siteUrl = text(v.siteUrl); + if (!siteId || !siteTitle || !siteUrl) return null; + const out: HubSummarySite = { siteId, siteTitle, siteUrl }; + const desc = text(v.siteDescription); + if (desc) out.siteDescription = desc; + for (const k of [ + "channels", + "recordings", + "transcripts", + "hoursArchived", + "gone", + ] as const) { + const n = count(v[k]); + if (n !== undefined) out[k] = n; + } + const accent = text(v.accent); + if (accent) out.accent = accent; + return out; +} + +// Lenient reader for the fetched file: anything unreadable is null (the hub +// then shows its cards without figures), a malformed site entry is dropped, a +// malformed figure is omitted. A version this reader does not know is null — +// it may mean something else. +export function parseHubSummary(json: unknown): HubSummary | null { + if (!isObject(json)) return null; + if (json.version !== HUB_SUMMARY_VERSION) return null; + const sites = Array.isArray(json.sites) + ? json.sites.map(parseSite).filter((s): s is HubSummarySite => s !== null) + : []; + return { + version: HUB_SUMMARY_VERSION, + generatedAt: typeof json.generatedAt === "string" ? json.generatedAt : "", + official: parseOfficial(json.official), + sites, + }; +} + +// The card figures for one built-in member, matched by siteId first and by +// origin as a fallback (a site renamed between builds keeps its URL). +export function hubSummarySiteFor( + summary: HubSummary | null, + member: { siteId: string; origin?: string }, +): HubSummarySite | null { + if (!summary) return null; + const byId = summary.sites.find((s) => s.siteId === member.siteId); + if (byId) return byId; + if (!member.origin) return null; + return ( + summary.sites.find((s) => { + try { + return new URL(s.siteUrl).origin === member.origin; + } catch { + return false; + } + }) ?? null + ); +} diff --git a/common/publish/build.ts b/common/publish/build.ts @@ -813,7 +813,8 @@ export function hubProjectProblem(project: string | undefined): string | null { /** * The hub build, as the children it runs: compose:hub (hub-sites.json, - * corpus.json, llms.txt, robots.txt, _headers, sw.js into export/public), then + * hub-summary.json when there is an index, corpus.json, llms.txt, robots.txt, + * _headers, sw.js into export/public), then * `next build` with INSTANCE_MODE=hub — both in export/. */ export function buildHubSteps(opts: {