Archilyzer · Source

archilyzer

Archilyzer
git clone https://archilyzer.pages.dev/source/archilyzer.git
Log | Files | Refs | README | LICENSE

commit 5dbead9a3e3836d7f1432eb2a78cf2fd774f6d3c
parent 4386f369e7ee7b974fbbecfa16c966af52444fb8
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date:   Fri, 25 Sep 2026 19:39:43 -0400

Merge one-core/c1-hub-look — release 9 slice C1: the hub in the family's register (archilyzer theme, Official instances with the homepage's figures via hub-summary.json, --chart-3 its own colour)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Diffstat:
M.gitignore | 1+
Mcommon/bin/compose-homepage.ts | 62+++++++++++---------------------------------------------------
Acommon/bin/compose-hub.test.ts | 55+++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mcommon/bin/compose-hub.ts | 42+++++++++++++++++++++++++++++++++++++++++-
Mcommon/bin/compose-site.ts | 2++
Mcommon/components/siteRegistry.ts | 2+-
Acommon/controller/poolSummary.ts | 87+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Acommon/lib/hubSummary.test.ts | 141+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Acommon/lib/hubSummary.ts | 164+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mcommon/publish/build.ts | 3++-
Mcommon/styles/tokens.css | 11+++++++++--
Meditor/CHANGELOG.md | 2++
Mexport/app/(workspace)/ask/page.tsx | 2+-
Mexport/app/components/hub/AddArchive.tsx | 13+++++++------
Mexport/app/components/hub/ArchiveShelf.tsx | 247+++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------------------
Mexport/app/components/hub/HubHome.tsx | 5+++--
Mexport/app/components/hub/HubStats.tsx | 38+++++++++++++++++++-------------------
Aexport/app/components/hub/useHubSummary.ts | 34++++++++++++++++++++++++++++++++++
Mexport/app/layout.tsx | 31+++++++++++++++++++++++++++----
Mexport/e2e-2origin/federation.spec.ts | 2+-
Mexport/e2e-hub/ask-grounding.spec.ts | 2+-
Mexport/e2e-hub/ask.spec.ts | 2+-
Mexport/e2e-hub/federation.spec.ts | 11+++++++++--
Aexport/e2e-hub/official-instances.spec.ts | 132+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mplans/release-9.md | 153++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
25 files changed, 1082 insertions(+), 162 deletions(-)

diff --git a/.gitignore b/.gitignore @@ -68,6 +68,7 @@ yarn-error.log* /export/public/_headers /export/public/sw.js /export/public/hub-sites.json +/export/public/hub-summary.json /export/public/corpus.json /export/public/llms.txt /export/public/robots.txt diff --git a/common/bin/compose-homepage.ts b/common/bin/compose-homepage.ts @@ -12,20 +12,9 @@ // prebuild chains it), same as the export pipeline. import path from "node:path"; -import { mkdir, readFile } from "node:fs/promises"; import { getPaths, type Paths } from "../lib/paths"; import { writeJsonAtomic as writeJsonAtomicShared } from "../lib/jsonFile-server"; -import { buildStats } from "../controller/buildStats"; -import { listSites } from "../lib/site"; -import { - statsPageFileName, - type StatsManifest, - type VideoStat, -} from "../lib/stats"; -import { - buildHomepageSummary, - type ChannelSitesMap, -} from "../lib/homepageSummary"; +import { buildPoolSummary } from "../controller/poolSummary"; import { runIfEntryPoint } from "./_cli"; // Where the homepage Next.js app serves static assets from. Overridable for e2e @@ -42,59 +31,30 @@ function writeJsonAtomic(filePath: string, value: unknown): Promise<void> { return writeJsonAtomicShared(filePath, value, { indent: 0, newline: false }); } -// Read the whole-pool stats dataset back from the pages buildStats just wrote, so -// the lightweight homepage summary can be derived from the same records without -// re-extracting. Pages are local files (fast) capped well under memory limits. -async function readStatsPages(statsDir: string): Promise<VideoStat[]> { - let manifest: StatsManifest; - try { - manifest = JSON.parse( - await readFile(path.join(statsDir, "manifest.json"), "utf8"), - ) as StatsManifest; - } catch { - return []; - } - const out: VideoStat[] = []; - for (let i = 0; i < manifest.pageCount; i++) { - const page = JSON.parse( - await readFile(path.join(statsDir, statsPageFileName(i)), "utf8"), - ) as VideoStat[]; - out.push(...page); - } - return out; -} - export async function main(opts: { paths?: Paths } = {}): Promise<void> { const paths = opts.paths ?? getPaths(); const publicDir = homepagePublicDir(paths.monorepoRoot); const statsDir = path.join(publicDir, "stats"); - await mkdir(statsDir, { recursive: true }); - // Whole-pool stats dataset (every non-excluded channel) for the cross-site - // charts. buildStats also refreshes the per-site bundles as a side effect, - // which is harmless. - await buildStats({ paths, wholePoolStatsDir: statsDir }); + // Whole-pool stats (every non-excluded channel) for the cross-site charts, + // then the summary over them. The hub's cards read a projection of the SAME + // call (compose-hub.ts, lib/hubSummary.ts) — the input gathering lives in + // controller/poolSummary.ts so the two cannot drift. + const { summary, channelSites, sites } = await buildPoolSummary({ + paths, + statsDir, + }); // channel slug -> the ids of the content sites that expose it. Drives - // `groupBy: "site"` in the hub's dashboard (see channelSites.tsx). A channel - // on multiple sites maps to all of them; a pool-only channel is simply absent. - const sites = listSites(paths); - const channelSites: ChannelSitesMap = {}; - for (const site of sites) { - for (const c of site.channels) { - (channelSites[c.slug] ??= []).push(site.siteId); - } - } + // `groupBy: "site"` in the hub's dashboard (see channelSites.tsx). await writeJsonAtomic( path.join(publicDir, "channel-sites.json"), channelSites, ); - // Lightweight cross-site summary for the hub landing page (pre-binned monthly + // Lightweight cross-site summary for the landing page (pre-binned monthly // counts, per-site totals/sparklines, recent additions). Embedded into the // SSG HTML so the front page never fetches the full stats dataset. - const stats = await readStatsPages(statsDir); - const summary = buildHomepageSummary(stats, channelSites, sites, new Date()); await writeJsonAtomic( path.join(publicDir, "homepage-summary.json"), summary, diff --git a/common/bin/compose-hub.test.ts b/common/bin/compose-hub.test.ts @@ -0,0 +1,55 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { getPaths, type Paths } from "../lib/paths"; +import { main } from "./compose-hub"; + +// Run with: +// pnpm --filter yt-dlp-transcript-common test +// +// hub-summary.json is OPTIONAL: a hub composed with no index to walk ships no +// numbers — and must not ship the LAST build's numbers either. + +function fixturePaths(root: string): Paths { + const sitesDir = path.join(root, "sites"); + const homepageDir = path.join(sitesDir, "_homepage"); + mkdirSync(homepageDir, { recursive: true }); + const exportPublicDir = path.join(root, "public"); + mkdirSync(exportPublicDir, { recursive: true }); + return { + ...getPaths(), + transcriptsDir: root, + channelsDir: path.join(root, "channels"), + sitesDir, + homepageDir, + homepageConfigFile: path.join(homepageDir, "homepage.json"), + homepageChartTemplatesFile: path.join(homepageDir, "chart-templates.json"), + lmdbPath: path.join(root, "index.mdb"), // never created: no index + exportPublicDir, + exportIndexDir: path.join(root, ".export-index"), + }; +} + +test("with no index, compose-hub writes the pool and no hub-summary.json, removing a stale one", async () => { + const root = mkdtempSync(path.join(tmpdir(), "compose-hub-")); + const log = console.log; + try { + const paths = fixturePaths(root); + const stale = path.join(paths.exportPublicDir, "hub-summary.json"); + writeFileSync(stale, '{"version":1,"sites":[]}'); + console.log = () => {}; + await main({ paths }); + console.log = log; + assert.ok(!existsSync(stale), "a stale summary must not survive"); + assert.deepEqual( + JSON.parse(readFileSync(path.join(paths.exportPublicDir, "hub-sites.json"), "utf8")), + [], + ); + assert.ok(!existsSync(paths.lmdbPath), "composing must not create an index"); + } finally { + console.log = log; + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/common/bin/compose-hub.ts b/common/bin/compose-hub.ts @@ -5,6 +5,9 @@ // reads every archive cross-origin at runtime. It emits: // // public/hub-sites.json <- the built-in trusted pool (listSites with a siteUrl) +// public/hub-summary.json <- the official instances' numbers, the homepage's +// own (lib/hubSummary.ts) — OPTIONAL: skipped when +// there is no index to walk // public/_headers <- CORS for the hub's own served JSON // public/sw.js <- the hub service worker (the hub always ships a PWA) // @@ -12,6 +15,7 @@ // HomepageConfig (see export/app/lib/site.ts hubSite()), not composed here. import path from "node:path"; +import { existsSync } from "node:fs"; import { cp, rm, writeFile, access } from "node:fs/promises"; import { getPaths, type Paths } from "../lib/paths"; import { listSites, resolveHubUrl } from "../lib/site"; @@ -24,6 +28,8 @@ import { type HubMemberInput, } from "../lib/corpus"; import { HUB_CORS_PATHS, renderHeadersFile } from "../lib/archive/headers"; +import { buildPoolSummary } from "../controller/poolSummary"; +import { HUB_SUMMARY_FILE, toHubSummary } from "../lib/hubSummary"; import { runIfEntryPoint } from "./_cli"; async function exists(p: string): Promise<boolean> { @@ -35,6 +41,38 @@ async function exists(p: string): Promise<boolean> { } } +// The official instances' figures, from the SAME buildHomepageSummary call the +// homepage's compose makes (controller/poolSummary.ts), projected to what the +// hub's cards read. Whole-pool stats pages land in the export staging dir (not +// served), never in public/. +// +// Optional by contract: with no index to walk (a checkout with no corpus, or a +// test) the file is not written and any stale one is removed, so the hub shows +// its cards without figures rather than last build's numbers. A failure here +// never fails the hub build — the shelf degrades, it does not break. +async function composeHubSummary( + paths: Paths, + publicDir: string, +): Promise<string> { + const dest = path.join(publicDir, HUB_SUMMARY_FILE); + if (!existsSync(paths.lmdbPath)) { + await rm(dest, { force: true }); + return "no index to summarise — hub-summary.json skipped"; + } + try { + const { summary } = await buildPoolSummary({ + paths, + statsDir: path.join(paths.exportIndexDir, "hub-stats"), + }); + const hubSummary = toHubSummary(summary); + await writeFile(dest, JSON.stringify(hubSummary)); + return `hub-summary.json covers ${hubSummary.sites.length} official instance(s)`; + } catch (err) { + await rm(dest, { force: true }); + return `hub-summary.json skipped: ${err instanceof Error ? err.message : String(err)}`; + } +} + export async function main(opts: { paths?: Paths } = {}): Promise<void> { const paths = opts.paths ?? getPaths(); const publicDir = paths.exportPublicDir; @@ -101,8 +139,10 @@ export async function main(opts: { paths?: Paths } = {}): Promise<void> { const swSrc = (await exists(hubSw)) ? hubSw : siteSw; if (await exists(swSrc)) await cp(swSrc, swDest); + const summaryNote = await composeHubSummary(paths, publicDir); + console.log( - `compose-hub: ${builtins.length} built-in pool site(s) into ${publicDir}.`, + `compose-hub: ${builtins.length} built-in pool site(s) into ${publicDir}; ${summaryNote}.`, ); } diff --git a/common/bin/compose-site.ts b/common/bin/compose-site.ts @@ -914,6 +914,8 @@ export async function main( // out/ and makes the bundle ambiguous to builtHubProblem (and to a site's // own registry, which treats the file as a hub's trusted pool). await rm(path.join(paths.exportPublicDir, "hub-sites.json"), { force: true }); + // …and neither is the hub's copy of the official instances' numbers. + await rm(path.join(paths.exportPublicDir, "hub-summary.json"), { force: true }); // --- service worker (only when this instance ships a PWA) --- await composeServiceWorker(site, paths); diff --git a/common/components/siteRegistry.ts b/common/components/siteRegistry.ts @@ -224,7 +224,7 @@ export type ValidateResult = // add-flow states. const BAD_URL = "Enter a full site URL, like https://archive.example.com."; const MIXED = "That site is served over http, so a secure hub can't read it."; -const ALREADY = "That archive is already on your shelf."; +const ALREADY = "That archive is already on this hub."; const UNREADABLE = "Couldn't read that site — it may not be an Archilyzer archive, or it isn't sharing its data."; const WRONG_CONTRACT = diff --git a/common/controller/poolSummary.ts b/common/controller/poolSummary.ts @@ -0,0 +1,87 @@ +// The homepage summary's inputs, gathered ONCE for both consumers: the homepage +// (compose-homepage writes the whole summary) and the hub (compose-hub writes +// the `official` + per-site projection, lib/hubSummary.ts). Both call +// buildPoolSummary, so the two sites cannot disagree about a number. +// +// NOT read-only: buildStats keeps its incremental per-video state in the index +// LMDB (`statsByPath`), and writes the whole-pool stats pages into `statsDir`. +// Requires build:index to have populated the cues sub-DB first. + +import path from "node:path"; +import { mkdir, readFile } from "node:fs/promises"; +import type { Paths } from "../lib/paths"; +import { buildStats } from "./buildStats"; +import { listSites, type Site } from "../lib/site"; +import { + statsPageFileName, + type StatsManifest, + type VideoStat, +} from "../lib/stats"; +import { + buildHomepageSummary, + type ChannelSitesMap, + type HomepageSummary, +} from "../lib/homepageSummary"; + +// Read the whole-pool stats dataset back from the pages buildStats just wrote, so +// the lightweight summary can be derived from the same records without +// re-extracting. Pages are local files (fast) capped well under memory limits. +export async function readStatsPages(statsDir: string): Promise<VideoStat[]> { + let manifest: StatsManifest; + try { + manifest = JSON.parse( + await readFile(path.join(statsDir, "manifest.json"), "utf8"), + ) as StatsManifest; + } catch { + return []; + } + const out: VideoStat[] = []; + for (let i = 0; i < manifest.pageCount; i++) { + const page = JSON.parse( + await readFile(path.join(statsDir, statsPageFileName(i)), "utf8"), + ) as VideoStat[]; + out.push(...page); + } + return out; +} + +// channel slug -> the ids of the content sites that expose it. A channel on +// multiple sites maps to all of them; a pool-only channel is simply absent. +export function channelSitesOf(sites: Site[]): ChannelSitesMap { + const channelSites: ChannelSitesMap = {}; + for (const site of sites) { + for (const c of site.channels) { + (channelSites[c.slug] ??= []).push(site.siteId); + } + } + return channelSites; +} + +export type PoolSummary = { + summary: HomepageSummary; + channelSites: ChannelSitesMap; + sites: Site[]; +}; + +// Refresh the whole-pool stats into `statsDir`, then summarise them. +export async function buildPoolSummary(opts: { + paths: Paths; + statsDir: string; + now?: Date; +}): Promise<PoolSummary> { + const { paths, statsDir } = opts; + await mkdir(statsDir, { recursive: true }); + // Whole-pool stats dataset (every non-excluded channel). buildStats also + // refreshes the per-site bundles as a side effect, which is harmless. + await buildStats({ paths, wholePoolStatsDir: statsDir }); + const sites = listSites(paths); + const channelSites = channelSitesOf(sites); + const stats = await readStatsPages(statsDir); + const summary = buildHomepageSummary( + stats, + channelSites, + sites, + opts.now ?? new Date(), + ); + return { summary, channelSites, sites }; +} diff --git a/common/lib/hubSummary.test.ts b/common/lib/hubSummary.test.ts @@ -0,0 +1,141 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + HUB_SUMMARY_VERSION, + hubSummarySiteFor, + parseHubSummary, + toHubSummary, +} from "./hubSummary"; +import type { HomepageSummary } from "./homepageSummary"; + +// Run with: +// pnpm --filter yt-dlp-transcript-common test + +const stat = (total: number) => ({ total, thisMonth: 0, last12: [] }); + +function homepageSummary(): HomepageSummary { + return { + version: 5, + generatedAt: "2026-09-25T00:00:00.000Z", + totals: { + transcripts: 99, + downloads: 99, + sites: 2, + channels: 9, + hoursArchived: 999, + transcribedThisMonth: 0, + downloadedThisMonth: 0, + }, + channels: [], + series: {} as HomepageSummary["series"], + sites: [ + { + siteId: "alpha", + siteTitle: "Alpha", + siteDescription: "The first archive.", + siteUrl: "https://alpha.example", + transcribed: stat(30), + downloaded: stat(40), + channels: 3, + recordings: 40, + hoursArchived: 12, + gone: 1, + accent: "#123456", + }, + { + // A pre-v5 site: no per-site figures beyond the transcript count. + siteId: "beta", + siteTitle: "Beta", + siteDescription: "", + siteUrl: "https://beta.example", + transcribed: stat(5), + downloaded: stat(6), + }, + ], + recent: [], + official: { + sites: 2, + channels: 3, + recordings: 40, + transcripts: 35, + hoursArchived: 12, + gone: 1, + }, + }; +} + +test("toHubSummary projects the homepage's official totals and card figures", () => { + const hub = toHubSummary(homepageSummary()); + assert.equal(hub.version, HUB_SUMMARY_VERSION); + assert.equal(hub.generatedAt, "2026-09-25T00:00:00.000Z"); + assert.deepEqual(hub.official, homepageSummary().official); + assert.deepEqual(hub.sites, [ + { + siteId: "alpha", + siteTitle: "Alpha", + siteDescription: "The first archive.", + siteUrl: "https://alpha.example", + channels: 3, + recordings: 40, + transcripts: 30, + hoursArchived: 12, + gone: 1, + accent: "#123456", + }, + { + siteId: "beta", + siteTitle: "Beta", + siteUrl: "https://beta.example", + transcripts: 5, + }, + ]); +}); + +test("a summary without `official` projects to null, not zeros", () => { + const s = homepageSummary(); + delete s.official; + assert.equal(toHubSummary(s).official, null); +}); + +test("parseHubSummary round-trips what toHubSummary writes", () => { + const hub = toHubSummary(homepageSummary()); + assert.deepEqual(parseHubSummary(JSON.parse(JSON.stringify(hub))), hub); +}); + +test("parseHubSummary degrades instead of throwing", () => { + assert.equal(parseHubSummary(null), null); + assert.equal(parseHubSummary("nope"), null); + assert.equal(parseHubSummary([]), null); + // A version this reader does not know may mean something else. + assert.equal(parseHubSummary({ version: 99, sites: [] }), null); + + const parsed = parseHubSummary({ + version: HUB_SUMMARY_VERSION, + generatedAt: 7, + official: { sites: 1 }, // incomplete -> null, never half a total + sites: [ + { siteId: "ok", siteTitle: "OK", siteUrl: "https://ok.example", channels: -1, recordings: "3", transcripts: 4 }, + { siteId: "no-url", siteTitle: "No URL" }, + "junk", + ], + }); + assert.deepEqual(parsed, { + version: HUB_SUMMARY_VERSION, + generatedAt: "", + official: null, + sites: [ + { siteId: "ok", siteTitle: "OK", siteUrl: "https://ok.example", transcripts: 4 }, + ], + }); +}); + +test("hubSummarySiteFor matches by siteId, then by origin", () => { + const hub = toHubSummary(homepageSummary()); + assert.equal(hubSummarySiteFor(hub, { siteId: "alpha" })?.siteTitle, "Alpha"); + assert.equal( + hubSummarySiteFor(hub, { siteId: "renamed", origin: "https://beta.example" })?.siteId, + "beta", + ); + assert.equal(hubSummarySiteFor(hub, { siteId: "gamma", origin: "https://gamma.example" }), null); + assert.equal(hubSummarySiteFor(null, { siteId: "alpha" }), null); +}); diff --git a/common/lib/hubSummary.ts b/common/lib/hubSummary.ts @@ -0,0 +1,164 @@ +// The hub's build-time numbers: the homepage's "Official instances" figures, +// shipped to the hub as `public/hub-summary.json` so its cards say exactly what +// the homepage's cards say. +// +// Why build time and not a live sum: a member's public `corpus.json` carries +// channel and video counts only — no recordings-with-a-download-date, no hours, +// no "gone" — so a live sum could never match the homepage. Both files are +// projections of ONE `buildHomepageSummary` call over the same stats walk +// (controller/poolSummary.ts), and the hub is rebuilt alongside the sites. +// +// The file is OPTIONAL. An older hub build, or a hub composed where there is no +// index to walk, has none; the hub then renders its cards without figures — +// never an error. It is read same-origin, so it is not in HUB_CORS_PATHS, and +// the deploy guard (builtHubProblem) does not require it. +// +// Pure and client-safe: the hub page parses it in the browser. + +import type { + HomepageOfficialTotals, + HomepageSummary, +} from "./homepageSummary"; + +export const HUB_SUMMARY_FILE = "hub-summary.json"; +export const HUB_SUMMARY_VERSION = 1; + +// One official instance's card figures. Every figure is optional: the homepage +// summary's v5 per-site fields are, and a card omits what it does not have. +export type HubSummarySite = { + siteId: string; + siteTitle: string; + siteDescription?: string; + siteUrl: string; + channels?: number; + recordings?: number; + transcripts?: number; // the homepage summary's `transcribed.total` + hoursArchived?: number; + gone?: number; + accent?: string; +}; + +export type HubSummary = { + version: number; + generatedAt: string; + official: HomepageOfficialTotals | null; + sites: HubSummarySite[]; +}; + +// The projection: only what the hub's cards and H1 read, from the SAME summary +// object compose-homepage writes. +export function toHubSummary(summary: HomepageSummary): HubSummary { + return { + version: HUB_SUMMARY_VERSION, + generatedAt: summary.generatedAt, + official: summary.official ?? null, + sites: summary.sites.map((s) => ({ + siteId: s.siteId, + siteTitle: s.siteTitle, + ...(s.siteDescription ? { siteDescription: s.siteDescription } : {}), + siteUrl: s.siteUrl, + ...(s.channels !== undefined ? { channels: s.channels } : {}), + ...(s.recordings !== undefined ? { recordings: s.recordings } : {}), + transcripts: s.transcribed.total, + ...(s.hoursArchived !== undefined + ? { hoursArchived: s.hoursArchived } + : {}), + ...(s.gone !== undefined ? { gone: s.gone } : {}), + ...(s.accent ? { accent: s.accent } : {}), + })), + }; +} + +function isObject(v: unknown): v is Record<string, unknown> { + return typeof v === "object" && v !== null && !Array.isArray(v); +} + +function count(v: unknown): number | undefined { + return typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : undefined; +} + +function text(v: unknown): string | undefined { + return typeof v === "string" && v.trim() ? v : undefined; +} + +function parseOfficial(v: unknown): HomepageOfficialTotals | null { + if (!isObject(v)) return null; + const keys = [ + "sites", + "channels", + "recordings", + "transcripts", + "hoursArchived", + "gone", + ] as const; + const out = {} as HomepageOfficialTotals; + for (const k of keys) { + const n = count(v[k]); + if (n === undefined) return null; + out[k] = n; + } + return out; +} + +function parseSite(v: unknown): HubSummarySite | null { + if (!isObject(v)) return null; + const siteId = text(v.siteId); + const siteTitle = text(v.siteTitle); + const siteUrl = text(v.siteUrl); + if (!siteId || !siteTitle || !siteUrl) return null; + const out: HubSummarySite = { siteId, siteTitle, siteUrl }; + const desc = text(v.siteDescription); + if (desc) out.siteDescription = desc; + for (const k of [ + "channels", + "recordings", + "transcripts", + "hoursArchived", + "gone", + ] as const) { + const n = count(v[k]); + if (n !== undefined) out[k] = n; + } + const accent = text(v.accent); + if (accent) out.accent = accent; + return out; +} + +// Lenient reader for the fetched file: anything unreadable is null (the hub +// then shows its cards without figures), a malformed site entry is dropped, a +// malformed figure is omitted. A version this reader does not know is null — +// it may mean something else. +export function parseHubSummary(json: unknown): HubSummary | null { + if (!isObject(json)) return null; + if (json.version !== HUB_SUMMARY_VERSION) return null; + const sites = Array.isArray(json.sites) + ? json.sites.map(parseSite).filter((s): s is HubSummarySite => s !== null) + : []; + return { + version: HUB_SUMMARY_VERSION, + generatedAt: typeof json.generatedAt === "string" ? json.generatedAt : "", + official: parseOfficial(json.official), + sites, + }; +} + +// The card figures for one built-in member, matched by siteId first and by +// origin as a fallback (a site renamed between builds keeps its URL). +export function hubSummarySiteFor( + summary: HubSummary | null, + member: { siteId: string; origin?: string }, +): HubSummarySite | null { + if (!summary) return null; + const byId = summary.sites.find((s) => s.siteId === member.siteId); + if (byId) return byId; + if (!member.origin) return null; + return ( + summary.sites.find((s) => { + try { + return new URL(s.siteUrl).origin === member.origin; + } catch { + return false; + } + }) ?? null + ); +} diff --git a/common/publish/build.ts b/common/publish/build.ts @@ -813,7 +813,8 @@ export function hubProjectProblem(project: string | undefined): string | null { /** * The hub build, as the children it runs: compose:hub (hub-sites.json, - * corpus.json, llms.txt, robots.txt, _headers, sw.js into export/public), then + * hub-summary.json when there is an index, corpus.json, llms.txt, robots.txt, + * _headers, sw.js into export/public), then * `next build` with INSTANCE_MODE=hub — both in export/. */ export function buildHubSteps(opts: { diff --git a/common/styles/tokens.css b/common/styles/tokens.css @@ -674,7 +674,12 @@ html[data-theme="archilyzer"] .font-display { --chart-1: #2f7f77; --chart-2: #1c5f96; - --chart-3: #a8412d; + /* --chart-3 is a series hue, not --state-gone: it used to be the same hex, + so the third archive in a chart read as "gone". A green, lightness-stepped + away from the gone red so it stays apart under deutan/protan simulation + too (OKLab dE vs gone 25.9 normal / 11.2 deutan; vs chart-2 26.9, chart-4 + 30.3; 3.29:1 on --chart-surface). */ + --chart-3: #5a9e3a; --chart-4: #6a4f9c; --chart-5: #8a6a1f; --chart-surface: #ffffff; @@ -731,7 +736,9 @@ html[data-theme="archilyzer"] .font-display { --chart-1: #5fa8a0; --chart-2: #6aa5d8; - --chart-3: #c4553f; + /* Same green, dark step (dE vs gone 29.1 normal / 17.1 deutan; vs chart-2 + 20.8, chart-4 26.6; 8.03:1 on --chart-surface). */ + --chart-3: #86c86a; --chart-4: #a98ede; --chart-5: #d3a03f; --chart-surface: #1a2229; diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md @@ -7,6 +7,8 @@ - **`/jobs` no longer calls a slow but moving job stuck.** "STUCK · POSSIBLY-STALLED" now needs the job's progress to have stood still for 10 minutes, not just the job to be 10 minutes old with nothing in flight. A metadata scan at ~10 videos a minute read stuck on 2026-09-25. - **Jobs left `queued` by a restart are settled at boot.** A job still waiting when the server stopped used to sit on `/jobs` as queued forever. On boot each one ends `cancelled` with its reason, and only a few are queued again, through the same path as Retry. A sync is never re-queued, because the scheduler re-derives syncs at its own pace. Nothing queued more than a day before the restart is re-queued. Of several identical jobs, only the newest is re-queued. With `ARCHILYZER_IDLE_BOOT` set, none are. The server log gets one summary line, plus one line per job queued again. - Smaller fixes from the release 8 reviews: a manual 429 cooldown merged with the runner's keeps the higher failure count as well as the later end; a channel's video-title memo keeps its titles when a new video directory appears, instead of re-reading every one; and the homepage counts "Transcripts" from the same pass that places them on the chart, counts "gone at the source, still here" only for recordings it holds, and drops the two empty columns its stats strip had when nothing is gone. No number on today's homepage changes. +- **The hub looks and reads like the homepage, and its list of archives is "Official instances" with the homepage's numbers.** The hub now opens in the project's own dark look (the *Archilyzer* theme), as the homepage does; published sites keep theirs. Its front page opens on one sentence, "<hours> hours of speech from every official archive, searchable at once." (the official instances' hours, as on the homepage), and one paragraph. **Official instances** has a card for each built-in archive: its name linking out, its description, and its channels, recordings, transcripts and hours, the same figures the homepage shows. A card whose site sets no accent colour takes the homepage card's colour for that site. Archives a visitor adds are listed under **Archives you added**, with their channel count and a Remove button, above the add form; they are kept only in that visitor's browser. The line under the cards reads "Searching 5 archives · … channels · … transcripts right now." and counts what the browser actually loaded, added archives included. The figures come from a new file the hub build writes, `hub-summary.json`, made by the same code that writes the homepage's summary. A hub built without it (an older build, or a machine with no index) shows the cards without figures. "The shelf", "Your archives" and the Member/Added badges are gone. Needs a rebuild and deploy of the hub. +- **The Archilyzer theme's third chart colour is no longer the "gone" red.** It is now a green (`#5a9e3a` light, `#86c86a` dark). The homepage chart and cards draw the third official instance in it, and a red layer read as "recordings gone". Needs a rebuild and deploy of the homepage. - **The yt-dlp clip command is back on sites with transcript downloads turned off.** Turning off `transcriptDownloads` (site.json, or the hub's homepage.json) hid three buttons in the transcript viewer. One of them, the yt-dlp button, only copies a `yt-dlp --download-sections` command for a marked clip to the clipboard and serves no file, so it is not a download. It now shows on every site. The switch still hides the Download menu (txt / srt / json) and Copy MD. The site and hub form labels in the editor say so. No setting changed; a site picks this up at its next build and deploy. - **Export sites: a search restored from the last visit waits for you.** Opening a site (or `/ask`, or the hub) still loads the last query, the filters and the profile from the browser, but no longer runs the search on the first page of a visit; moving between pages after a search keeps it running, so the hub's chat still grounds in the search just done on its front page. The results show the video listing under those filters, and the bar says "Press Enter or click Search to apply", as for any unapplied edit. Search or Enter runs it; so does loading a profile. A link with a query in it (`?qt=`) still runs on arrival. Going straight to the hub's `/ask` in a new visit leaves the restored search held, and that page has no search bar: search on the hub's front page first. The restored search used to re-fetch transcript shards (up to 8 MB each) on every visit to a device that had not cached them. Needs a rebuild and deploy of every export site. - **The project homepage is rewritten, and it can be deployed as a preview.** The front page of `archilyzer.pages.dev` now opens on one line and a chart. The chart shows the official instances' transcripts by the month each video was published, from 2009 to now, stacked by instance. It is drawn when the site is built, so no chart script loads. Under it is one strip of numbers for the official instances: hours of speech, transcripts, recordings, channels, instances, and recordings gone at the source but still here. The **Official instances** section has a card for each public site, with its channels, recordings, transcripts and hours. **What it does** is now three short paragraphs. The recent-acquisitions list, "How it works" and "What this isn't" are gone from the page. The homepage summary (`homepage/public/homepage-summary.json`) is version 5. It adds `monthly`, `official` and per-site numbers and removes nothing, so `/stats` is unchanged. `archilyzer deploy homepage --preview <branch>` deploys a preview the way `deploy hub --preview` does. Without the flag it still deploys to production (`main`). diff --git a/export/app/(workspace)/ask/page.tsx b/export/app/(workspace)/ask/page.tsx @@ -37,7 +37,7 @@ export default function AskPage() { <Link href="/use-with-ai" className="text-brand hover:underline"> Use with AI </Link>{" "} - for other ways to use this archive. + for other ways to use {isHub ? "every archive on this hub" : "this archive"}. </p> </header> diff --git a/export/app/components/hub/AddArchive.tsx b/export/app/components/hub/AddArchive.tsx @@ -1,10 +1,11 @@ "use client"; -// The "add a volume to the shelf" control: paste an archive URL, it validates -// against the federation contract, and on success the archive slides onto the -// shelf. Written as direction — validating → success, or a plain reason it -// couldn't be read. Self-contained: owns its input + status and commits to the -// shared registry. +// The add-an-archive control: paste an archive URL, it validates against the +// federation contract, and on success the archive joins "Archives you added". +// Written as direction — validating → success, or a plain reason it couldn't +// be read. Self-contained: owns its input + status and commits to the shared +// registry. The button is the family's CTA — ink on ground, not a coloured +// fill (tokens.css, the archilyzer block). import { useState } from "react"; import { @@ -65,7 +66,7 @@ export default function AddArchive() { <button type="submit" disabled={busy || !url.trim()} - className="rounded-md bg-brand px-4 py-2 text-sm font-medium text-brand-ink transition-colors hover:bg-brand-strong disabled:opacity-50" + className="rounded-md bg-foreground px-4 py-2 text-sm font-medium text-background transition-colors hover:bg-brand hover:text-brand-ink disabled:opacity-50 disabled:hover:bg-foreground disabled:hover:text-background" > {busy ? "Validating…" : "Add"} </button> diff --git a/export/app/components/hub/ArchiveShelf.tsx b/export/app/components/hub/ArchiveShelf.tsx @@ -1,11 +1,25 @@ "use client"; -// The hub's hero: a shelf of archives, not a KPI wall. Each federated archive is -// a first-class, identity-bearing spine in its OWN accent — the registry itself -// is the thing you curate, so it leads. Adding a site slides a new volume onto -// the shelf. Chrome stays neutral/brass so the per-site accents are the only -// color; the load stagger (globals.css .shelf-spine) is the only motion and -// respects reduced-motion. +// The hub's front: what it is, the official instances it searches, the +// archives a visitor added, and the form to add another — in the homepage's +// register (homepage/app/page.tsx): short declarative sentences, no eyebrow +// over a band, no subtitle under a heading, and each heading after the H1 +// names what is under it. +// +// Two kinds of number, deliberately kept apart. An official card shows the +// homepage's own figures for that archive, composed at hub build time +// (/hub-summary.json, useHubSummary) — the same numbers the homepage prints, +// omitted one by one when the file lacks them. An added archive has no such +// file behind it, so its card shows only the live channel count its +// descriptor gave. The live federation total is HubStats, under the shelf. +// +// Each card wears its archive's OWN accent when the site sets one (tokens.css, +// the archilyzer family: the tool has no colour, the archives do). An official +// instance that sets none falls back to seriesColor() at its index in the +// summary's `sites` — the order the homepage's ArchiveCards and growth chart +// use — so a hub card and its homepage card wear the same colour. With no +// summary it is the card's own index. An added archive with no accent wears +// the family's signal colour. import { X } from "lucide-react"; import { Badge } from "yt-dlp-transcript-common/components/ui/badge"; @@ -13,107 +27,204 @@ import { useRegistry, type RegisteredSite, } from "yt-dlp-transcript-common/components/siteRegistry"; +import { + hubSummarySiteFor, + type HubSummarySite, +} from "yt-dlp-transcript-common/lib/hubSummary"; +import { seriesColor } from "yt-dlp-transcript-common/lib/homepageChart"; import AddArchive from "./AddArchive"; +import { useHubSummary } from "./useHubSummary"; -// Fall back to the family brass when a site declares no accent. +// Fall back to the family's signal colour when a site declares no accent. const FALLBACK_ACCENT = "var(--brand)"; -function Spine({ +function Figure({ value, unit }: { value: number | undefined; unit: string }) { + if (value === undefined) return null; + return ( + <div className="flex min-w-0 flex-col-reverse justify-end gap-1"> + <dt className="text-xs text-faint">{unit}</dt> + <dd className="text-[0.9375rem] tabular-nums text-foreground"> + {value.toLocaleString()} + </dd> + </div> + ); +} + +function ArchiveCard({ site, + figures, index, + fallbackAccent = FALLBACK_ACCENT, onRemove, }: { site: RegisteredSite; + fallbackAccent?: string; + // Official: the build-time figures (null until/unless the summary loads). + // Added: undefined — the card shows the descriptor's channel count only. + figures?: HubSummarySite | null; index: number; - onRemove: (origin: string) => void; + onRemove?: (origin: string) => void; }) { - const accent = site.accent || FALLBACK_ACCENT; - const member = site.kind === "builtin"; + const accent = site.accent || figures?.accent || fallbackAccent; + const href = site.siteUrl || site.origin; + const description = figures?.siteDescription; + const channels = + figures === undefined ? site.channelCount : figures?.channels; + const anyFigure = + channels !== undefined || + figures?.recordings !== undefined || + figures?.transcripts !== undefined || + figures?.hoursArchived !== undefined; return ( <li data-testid="shelf-spine" - className="shelf-spine group relative flex w-full min-w-0 max-w-full flex-col gap-2 rounded-md border border-border bg-card/60 py-3 pl-4 pr-3 sm:w-auto sm:min-w-52 sm:max-w-64" + className="shelf-spine group relative flex min-w-0 flex-col gap-4 border-r border-b border-border bg-surface p-5 pl-6" style={{ animationDelay: `${Math.min(index, 8) * 45}ms` }} > - {/* The spine: a thick accent edge in the site's own color. */} <span aria-hidden="true" - className="absolute inset-y-1.5 left-0 w-1 rounded-full" + className="absolute inset-y-0 left-0 w-[3px]" style={{ background: accent }} /> - <div className="flex items-start justify-between gap-2"> - <span className="font-display text-base font-semibold leading-tight tracking-tight text-foreground"> - {site.siteTitle} - </span> - {site.kind === "external" && ( - <button - type="button" - onClick={() => onRemove(site.origin)} - aria-label={`Remove ${site.siteTitle}`} - // Reveal-on-hover is not an affordance on a touch screen: a device - // with no hover shows it always. `hover:` never gates a function. - className="-mr-1 -mt-0.5 rounded p-0.5 text-muted-foreground/60 opacity-0 transition-opacity pointer-coarse:opacity-100 hover:text-destructive focus-visible:opacity-100 group-hover:opacity-100" - > - <X className="size-3.5" aria-hidden="true" /> - </button> + <div className="flex flex-col gap-1.5"> + <div className="flex items-start justify-between gap-2"> + <h3 className="font-display text-lg font-semibold text-foreground"> + <a + href={href} + target="_blank" + rel="noopener noreferrer" + className="underline decoration-transparent underline-offset-4 transition-colors hover:decoration-border-strong" + > + {site.siteTitle} + <span aria-hidden="true" className="ml-1.5 text-sm text-faint"> + ↗ + </span> + </a> + </h3> + {onRemove && ( + <button + type="button" + onClick={() => onRemove(site.origin)} + aria-label={`Remove ${site.siteTitle}`} + // Reveal-on-hover is not an affordance on a touch screen: a device + // with no hover shows it always. `hover:` never gates a function. + className="-mr-1 mt-0.5 rounded p-0.5 text-muted-foreground/60 opacity-0 transition-opacity pointer-coarse:opacity-100 hover:text-destructive focus-visible:opacity-100 group-hover:opacity-100" + > + <X className="size-3.5" aria-hidden="true" /> + </button> + )} + </div> + {description && ( + <p className="text-sm leading-relaxed text-muted-foreground"> + {description} + </p> )} </div> - {typeof site.channelCount === "number" && ( - <span className="font-mono text-[0.7rem] uppercase tracking-[0.12em] text-muted-foreground"> - {site.channelCount} channel{site.channelCount === 1 ? "" : "s"} - </span> - )} - <div className="mt-auto flex flex-wrap items-center gap-1.5 pt-1"> - <Badge variant={member ? "secondary" : "outline"}> - {member ? "Member" : "Added"} - </Badge> - <Badge variant={site.pwa ? "info" : "ghost"}> - {site.pwa ? "Installable" : "Data-only"} - </Badge> + <div className="mt-auto flex flex-col gap-3"> + {anyFigure && ( + <dl className="grid grid-cols-4 gap-3"> + <Figure value={channels} unit="channels" /> + <Figure value={figures?.recordings} unit="recordings" /> + <Figure value={figures?.transcripts} unit="transcripts" /> + <Figure value={figures?.hoursArchived} unit="hours" /> + </dl> + )} + <div> + <Badge variant={site.pwa ? "info" : "ghost"}> + {site.pwa ? "Installable" : "Data-only"} + </Badge> + </div> </div> </li> ); } +const GRID = + "grid list-none grid-cols-1 border-t border-l border-border sm:grid-cols-2 lg:grid-cols-3"; + export default function ArchiveShelf() { const { sites, removeSite } = useRegistry(); + const summary = useHubSummary(); + const official = sites.filter((s) => s.kind === "builtin"); + const added = sites.filter((s) => s.kind === "external"); + const hours = summary?.official?.hoursArchived; return ( - <section className="flex flex-col gap-4"> - <div className="flex flex-col gap-1"> - <span className="font-mono text-xs uppercase tracking-[0.16em] text-muted-foreground"> - The shelf - </span> - <h1 className="font-display text-3xl font-semibold tracking-tight text-foreground"> - Your archives + <section className="flex flex-col gap-10 pb-4"> + <div className="flex flex-col gap-4"> + <h1 className="font-display text-[1.75rem] font-semibold leading-[1.12] tracking-[-0.01em] text-foreground sm:text-[2.25rem]"> + {hours ? ( + <> + <span className="font-medium tabular-nums"> + {hours.toLocaleString()} + </span>{" "} + hours of speech from every official archive, searchable at once. + </> + ) : ( + <> + {official.length > 0 + ? "Every official archive, searchable at once." + : "Every archive you add, searchable at once."} + </> + )} </h1> - <p className="max-w-prose text-sm text-muted-foreground"> - Every archive you add is searchable together below. Paste a site&apos;s - URL to shelve it. + <p className="max-w-2xl text-[1.0625rem] leading-relaxed text-muted-foreground"> + One search runs across all of them, in your browser. You can add + any other Archilyzer archive to your own search, below. It stays in + this browser until you remove it. </p> </div> - <AddArchive /> - - {sites.length === 0 ? ( - <div className="rounded-lg border border-dashed border-border/70 px-5 py-8 text-center"> - <p className="text-sm text-muted-foreground"> - The shelf is empty. Add an archive above to start searching across - it. + {official.length > 0 ? ( + <div> + <h2 className="font-display text-2xl font-semibold text-foreground"> + Official instances + </h2> + <p className="mt-2 mb-6 max-w-xl leading-relaxed text-muted-foreground"> + The archives I run. Anyone can run their own. </p> + <ul className={GRID}> + {official.map((site, i) => { + const figures = hubSummarySiteFor(summary, site); + const at = figures ? (summary?.sites.indexOf(figures) ?? -1) : -1; + return ( + <ArchiveCard + key={site.origin || site.siteId} + site={site} + figures={figures} + index={i} + fallbackAccent={seriesColor(at >= 0 ? at : i)} + /> + ); + })} + </ul> </div> ) : ( - <ul className="flex list-none flex-wrap gap-3 border-b-2 border-border/70 pb-4"> - {sites.map((site, i) => ( - <Spine - key={site.origin || site.siteId} - site={site} - index={i} - onRemove={removeSite} - /> - ))} - </ul> + <p className="max-w-xl leading-relaxed text-muted-foreground"> + This hub has no official instances. Add an archive to search it. + </p> )} + + <div> + {added.length > 0 && ( + <> + <h2 className="font-display text-2xl font-semibold text-foreground"> + Archives you added + </h2> + <ul className={`${GRID} mt-6 mb-6`}> + {added.map((site, i) => ( + <ArchiveCard + key={site.origin || site.siteId} + site={site} + index={i} + onRemove={removeSite} + /> + ))} + </ul> + </> + )} + <AddArchive /> + </div> </section> ); } diff --git a/export/app/components/hub/HubHome.tsx b/export/app/components/hub/HubHome.tsx @@ -1,7 +1,8 @@ "use client"; -// The hub home: the archive shelf (hero + registry) above one unified search -// across every shelved archive. The shelf/add-flow live in ArchiveShelf; this +// The hub home: the archives (official instances, the visitor's added ones, +// the add form) above one unified search across all of them. The cards and +// add-flow live in ArchiveShelf; this // component wires the registry into the multi-origin data source so the shared // TranscriptSearch renders one merged, origin-qualified view. diff --git a/export/app/components/hub/HubStats.tsx b/export/app/components/hub/HubStats.tsx @@ -1,15 +1,22 @@ "use client"; -// A demoted stats line, not a KPI wall: the shelf is the hero, so the totals sit -// quietly beneath it as a single mono strip. Numbers are computed from the LIVE -// merged data across the shelved archives (not a build-time summary), so they -// always reflect exactly what's federated right now. +// One line under the shelf: what this browser is searching right now. These +// numbers are the LIVE federation — computed from the merged data of every +// archive on the hub, the visitor's added ones included — so they are not the +// cards' numbers and are not meant to match them. The official cards show the +// build-time figures the homepage prints (hub-summary.json); this line shows +// what actually loaded. import { useSearchData } from "yt-dlp-transcript-common/components/SearchDataContext"; import { useRegistry } from "yt-dlp-transcript-common/components/siteRegistry"; -function fmt(n: number): string { - return n.toLocaleString(); +function part(n: number, one: string, many: string) { + return ( + <> + <span className="tabular-nums text-foreground">{n.toLocaleString()}</span>{" "} + {n === 1 ? one : many} + </> + ); } export default function HubStats() { @@ -19,20 +26,13 @@ export default function HubStats() { if (sites.length === 0) return null; - const parts: [number, string][] = [ - [sites.length, sites.length === 1 ? "archive" : "archives"], - [channels.length, channels.length === 1 ? "channel" : "channels"], - [transcripts, transcripts === 1 ? "transcript" : "transcripts"], - ]; - return ( - <p className="font-mono text-xs uppercase tracking-[0.12em] text-muted-foreground"> - {parts.map(([n, label], i) => ( - <span key={label}> - {i > 0 && <span className="mx-2 text-muted-foreground/40">·</span>} - <span className="text-foreground tabular-nums">{fmt(n)}</span> {label} - </span> - ))} + <p className="text-sm text-muted-foreground"> + Searching {part(sites.length, "archive", "archives")} + <span className="mx-2 text-muted-foreground/40">·</span> + {part(channels.length, "channel", "channels")} + <span className="mx-2 text-muted-foreground/40">·</span> + {part(transcripts, "transcript", "transcripts")} right now. </p> ); } diff --git a/export/app/components/hub/useHubSummary.ts b/export/app/components/hub/useHubSummary.ts @@ -0,0 +1,34 @@ +"use client"; + +// The official instances' numbers, composed at hub build time +// (common/bin/compose-hub.ts -> /hub-summary.json) from the same summary the +// homepage renders. Same-origin, fetched once. OPTIONAL: an older hub build or +// a hub composed with no index has no file, and every failure — 404, bad JSON, +// an unknown version — reads as "no numbers", never as an error on the page. + +import { useQuery } from "@tanstack/react-query"; +import { + HUB_SUMMARY_FILE, + parseHubSummary, + type HubSummary, +} from "yt-dlp-transcript-common/lib/hubSummary"; + +async function fetchHubSummary(): Promise<HubSummary | null> { + try { + const res = await fetch(`/${HUB_SUMMARY_FILE}`); + if (!res.ok) return null; + return parseHubSummary(await res.json()); + } catch { + return null; + } +} + +export function useHubSummary(): HubSummary | null { + const { data } = useQuery({ + queryKey: ["hub-summary"], + queryFn: fetchHubSummary, + staleTime: Infinity, + retry: false, + }); + return data ?? null; +} diff --git a/export/app/layout.tsx b/export/app/layout.tsx @@ -5,7 +5,7 @@ import { ThemeProvider } from "yt-dlp-transcript-common/components/ThemeProvider import { QueryProvider } from "yt-dlp-transcript-common/components/QueryProvider"; import { parseAccent, siteAccentVars } from "yt-dlp-transcript-common/lib/accent"; import { currentSite } from "./lib/site"; -import { shipsPwa } from "./lib/mode"; +import { instanceMode, shipsPwa } from "./lib/mode"; import Header from "./components/Header"; import Footer from "./components/Footer"; import { ServiceWorkerRegister } from "./components/ServiceWorkerRegister"; @@ -16,6 +16,20 @@ import "./globals.css"; // defaults to base. const FALLBACK_THEME_COLOR = "#2563eb"; +// The hub is the TOOL, not an archive: it wears the project's own instrument +// face — the "archilyzer" family, dark by default — exactly as the homepage +// does (homepage/app/layout.tsx), so the two read as one product. Every site +// build keeps the base family and the visitor's system mode. The browser chrome +// colour is the homepage's too (the family's dark ground, tokens.css). +// instanceMode() is server-only; this whole file is a server component. +const HUB_THEME = { defaultTheme: "archilyzer", defaultMode: "dark" } as const; +const SITE_THEME = { defaultTheme: "base", defaultMode: "system" } as const; +const HUB_THEME_COLOR = "#151b20"; + +function themeDefaults() { + return instanceMode() === "hub" ? HUB_THEME : SITE_THEME; +} + export function generateMetadata(): Metadata { const s = currentSite(); return { @@ -46,7 +60,9 @@ export function generateMetadata(): Metadata { export function generateViewport(): Viewport { return { // Browser chrome color tracks the site's brand accent (falls back to base brand). - themeColor: parseAccent(currentSite().accent) ?? FALLBACK_THEME_COLOR, + themeColor: + parseAccent(currentSite().accent) ?? + (instanceMode() === "hub" ? HUB_THEME_COLOR : FALLBACK_THEME_COLOR), width: "device-width", initialScale: 1, // Let the layout run under the notch/home indicator; the safe-area insets @@ -68,6 +84,7 @@ export default async function RootLayout({ // Per-site accent: bake the brand-color override onto <html> at prerender so // it's present on first paint (before any CSS/JS). Unset → family brass. const accentVars = siteAccentVars(currentSite().accent); + const theme = themeDefaults(); return ( <html lang="en" @@ -76,9 +93,15 @@ export default async function RootLayout({ style={accentVars ?? undefined} > <body className="min-h-full flex flex-col bg-background text-foreground font-sans"> - <ThemeScript defaultTheme="base" defaultMode="system" /> + <ThemeScript + defaultTheme={theme.defaultTheme} + defaultMode={theme.defaultMode} + /> {shipsPwa() && <ServiceWorkerRegister />} - <ThemeProvider defaultTheme="base" defaultMode="system"> + <ThemeProvider + defaultTheme={theme.defaultTheme} + defaultMode={theme.defaultMode} + > <QueryProvider> <Header /> <main className="flex-1 w-full max-w-6xl mx-auto px-4 sm:px-6 py-6"> diff --git a/export/e2e-2origin/federation.spec.ts b/export/e2e-2origin/federation.spec.ts @@ -19,7 +19,7 @@ test.describe("real two-origin federation (CORS + service worker)", () => { }) => { await page.goto("/"); await expect( - page.getByRole("heading", { name: "Your archives" }), + page.getByRole("heading", { level: 1, name: /searchable at once\./ }), ).toBeVisible(); await addArchive(page, ORIGIN_B); // The spine's title came from Origin B's /site.json, fetched cross-origin — diff --git a/export/e2e-hub/ask-grounding.spec.ts b/export/e2e-hub/ask-grounding.spec.ts @@ -124,7 +124,7 @@ test("hub: a search run on / still grounds the chat after the per-card Ask", asy await stubBuiltins(page); await mockOriginB(page); await page.goto("/"); - await expect(page.getByRole("heading", { name: "Your archives" })).toBeVisible(); + await expect(page.getByRole("heading", { level: 1, name: /searchable at once\./ })).toBeVisible(); await addArchive(page, ORIGIN_B); await expect( page.getByTestId("shelf-spine").filter({ hasText: "Origin B" }), diff --git a/export/e2e-hub/ask.spec.ts b/export/e2e-hub/ask.spec.ts @@ -10,7 +10,7 @@ import { expect, test, type Page } from "@playwright/test"; // This spec is the cheap guard on the rendered page; the build itself is the // other half, covered by the 2-origin suite which builds the hub for real. -// No built-in pool: this route has to stand up on a hub with an empty shelf, +// No built-in pool: this route has to stand up on a hub with no archives, // which is what a fresh hub is. async function stubBuiltins(page: Page) { await page.route("**/hub-sites.json", (r) => diff --git a/export/e2e-hub/federation.spec.ts b/export/e2e-hub/federation.spec.ts @@ -122,7 +122,7 @@ test.describe("hub federation — cross-origin browse + search", () => { await stubBuiltins(page); await page.goto("/"); await expect( - page.getByRole("heading", { name: "Your archives" }), + page.getByRole("heading", { level: 1, name: /searchable at once\./ }), ).toBeVisible(); }); @@ -139,13 +139,20 @@ test.describe("hub federation — cross-origin browse + search", () => { await expect(builtWith).toHaveAttribute("href", PROJECT_URL); }); - test("adds an archive by URL and shows it on the shelf", async ({ page }) => { + test("adds an archive by URL and shows it under Archives you added", async ({ page }) => { await mockOriginB(page); await addArchive(page, ORIGIN_B); // Spine carries the descriptor's siteTitle → the site was read cross-origin. await expect( page.getByTestId("shelf-spine").filter({ hasText: "Origin B" }), ).toBeVisible(); + // An added archive gets its own section, and its own Remove control. + await expect( + page.getByRole("heading", { level: 2, name: "Archives you added" }), + ).toBeVisible(); + await expect( + page.getByRole("button", { name: "Remove Origin B" }), + ).toBeAttached(); }); test("searches across the added origin and returns origin-qualified hits", async ({ diff --git a/export/e2e-hub/official-instances.spec.ts b/export/e2e-hub/official-instances.spec.ts @@ -0,0 +1,132 @@ +import { expect, test, type Page, type Route } from "@playwright/test"; + +// The hub's "Official instances": one card per built-in member +// (hub-sites.json), carrying the homepage's own figures for it when the hub was +// composed with them (hub-summary.json, written by compose-hub from the same +// buildHomepageSummary call the homepage makes). The summary is OPTIONAL — an +// older build or a hub composed with no index ships none — and then the card +// renders with no figures, never an error. + +// A member origin that is never really served: its data is 404 (with CORS so +// the browser reports a clean 404, not a blocked read). The cards come from +// hub-sites.json + hub-summary.json alone, so nothing here needs its data. +const MEMBER = "http://localhost:4598"; + +async function fulfillJson(route: Route, body: unknown) { + await route.fulfill({ + status: 200, + contentType: "application/json", + headers: { "access-control-allow-origin": "*" }, + body: JSON.stringify(body), + }); +} + +async function stubMember(page: Page) { + await page.route("**/hub-sites.json", (r) => + fulfillJson(r, [ + { + siteId: "member", + siteTitle: "Member Archive", + siteUrl: MEMBER, + pwa: false, + contract: 1, + }, + ]), + ); + await page.route(`${MEMBER}/**`, (r) => + r.fulfill({ status: 404, headers: { "access-control-allow-origin": "*" }, body: "" }), + ); +} + +const SUMMARY = { + version: 1, + generatedAt: "2026-09-25T00:00:00.000Z", + official: { + sites: 1, + channels: 12, + recordings: 3456, + transcripts: 2345, + hoursArchived: 7890, + gone: 3, + }, + sites: [ + { + siteId: "member", + siteTitle: "Member Archive", + siteDescription: "Every word the member said.", + siteUrl: MEMBER, + channels: 12, + recordings: 3456, + transcripts: 2345, + hoursArchived: 7890, + gone: 3, + }, + ], +}; + +test.describe("hub official instances", () => { + test("with hub-summary.json, an official card shows its four figures", async ({ + page, + }) => { + await stubMember(page); + await page.route("**/hub-summary.json", (r) => fulfillJson(r, SUMMARY)); + await page.goto("/"); + + await expect( + page.getByRole("heading", { + level: 1, + name: "7,890 hours of speech from every official archive, searchable at once.", + }), + ).toBeVisible(); + await expect( + page.getByRole("heading", { level: 2, name: "Official instances" }), + ).toBeVisible(); + + const card = page + .getByTestId("shelf-spine") + .filter({ hasText: "Member Archive" }); + await expect(card).toBeVisible(); + await expect( + card.getByRole("link", { name: /Member Archive/ }), + ).toHaveAttribute("target", "_blank"); + await expect(card).toContainText("Every word the member said."); + for (const [value, unit] of [ + ["12", "channels"], + ["3,456", "recordings"], + ["2,345", "transcripts"], + ["7,890", "hours"], + ]) { + await expect(card.locator("dd", { hasText: value })).toBeVisible(); + await expect(card.locator("dt", { hasText: unit })).toBeVisible(); + } + // A built-in member is not the visitor's to remove, and there is no + // "Archives you added" section until they add one. + await expect(card.getByRole("button", { name: /^Remove/ })).toHaveCount(0); + await expect( + page.getByRole("heading", { name: "Archives you added" }), + ).toHaveCount(0); + }); + + test("without hub-summary.json, the card renders with no figures", async ({ + page, + }) => { + await stubMember(page); + await page.route("**/hub-summary.json", (r) => + r.fulfill({ status: 404, body: "" }), + ); + await page.goto("/"); + + await expect( + page.getByRole("heading", { + level: 1, + name: "Every official archive, searchable at once.", + }), + ).toBeVisible(); + const card = page + .getByTestId("shelf-spine") + .filter({ hasText: "Member Archive" }); + await expect(card).toBeVisible(); + await expect(card.getByText("Data-only")).toBeVisible(); + await expect(card.locator("dl")).toHaveCount(0); + }); +}); diff --git a/plans/release-9.md b/plans/release-9.md @@ -5,7 +5,10 @@ bugs the release-8 rollout found on the live editor, batched so they cost one ed queued job that ends `failed` after doing its work, a 30 s sleep after every video the download filter declined, YouTube with no request-level pacing, and a `/jobs` page that calls a slow scan stuck and keeps restart-orphaned jobs `queued` forever. The release-8 reviews' optional lows ride -along. Rules: `plans/tools/implementer-rules.md`. Record file: this file. +along. The same release also carries the hub's polish, which the operator asked for before the homepage +links the hub again (`HUB_LINK_ENABLED = false`): C1 brings the hub into the homepage's look and copy +with the homepage's own numbers, and C2 the federated-search UX. Flipping the homepage's hub link is a +later step, not a slice. Rules: `plans/tools/implementer-rules.md`. Record file: this file. ## Record @@ -225,4 +228,152 @@ No e2e covers the boot pass. The test server is cancel-only by design, so the pa reported through `detached`, so a Next upgrade that moves it fails the spec loudly rather than letting it pass vacuously. +### Slice C1, as shipped — the hub in the family's register (2026-09-25) + +Branch `one-core/c1-hub-look` off `main` `9247211e`. **The operator's decision (2026-09-25 evening):** +the hub (`https://archilyzer-hub.pages.dev`, the export app with `INSTANCE_MODE=hub`) gets the +homepage's look and copy, and its shelf becomes "Official instances" showing the SAME per-site numbers +the homepage shows. Scoping: `$T/r9-hub-scope.md` (read-only pass). The federated-search UX +(`SearchDataContext.tsx`, `SearchResults.tsx`, `SearchSessionContext.tsx`) is C2's and untouched. + +**Theme.** `export/app/layout.tsx` branches on `instanceMode() === "hub"` (server-only, the file is a +server component) to `defaultTheme="archilyzer" defaultMode="dark"`, exactly `homepage/app/layout.tsx`'s, +and the hub's browser-chrome `themeColor` is the homepage's `#151b20`. Site builds keep +`base`/`system` and `#2563eb`; `theme.spec.ts`/`theme-family.spec.ts` (site build) stay green. The +built hub `index.html` carries `archilyzer` in its ThemeScript; a render of the built hub reads +`data-theme="archilyzer"` + `.dark` with no stored preference. + +**`--chart-3`** in `[data-theme="archilyzer"]` was the `--state-gone` hex in both modes, so the third +official instance (Bonnellyzer, in `seriesColor` order) drew as "gone" on the homepage chart and card. +It is now a green, lightness-stepped away from the gone red so the two stay apart under CVD simulation +too. Chosen by the dataviz validator (OKLab ΔE ×100, Machado CVD), against the neighbours in the +fixed order: + +| mode | `--chart-3` | vs `--state-gone` normal / protan / deutan | vs chart-2 | vs chart-4 | contrast on `--chart-surface` | +|---|---|---|---|---|---| +| light | `#5a9e3a` (was `#a8412d`) | 25.9 / 21.8 / 11.2 | 26.9 | 30.3 | 3.29:1 on `#ffffff` | +| dark | `#86c86a` (was `#c4553f`) | 29.1 / 27.4 / 17.1 | 20.8 | 26.6 | 8.03:1 on `#1a2229` | + +Hues near magenta read as the gone red at normal vision (ΔE ~10–11); darker greens collapse into it for +deutans (ΔE 2–5). The palette's pre-existing validator failures are unchanged and not this slice's: +chroma floor (`chart-1`, `chart-5` light; `chart-1`, `chart-2` dark), the chart-1↔chart-2 normal-vision +floor (12.4 light / 9.0 dark), and the dark lightness band. Green also sits near `--success`; a chart +series is not chrome state, so that was accepted. + +**The numbers, at build time.** `compose-hub.ts` now also writes `public/hub-summary.json` +(`lib/hubSummary.ts`, version 1): `{version, generatedAt, official, sites[]}`, each site +`{siteId, siteTitle, siteDescription?, siteUrl, channels?, recordings?, transcripts?, hoursArchived?, +gone?, accent?}` — `transcripts` is the homepage summary's `transcribed.total`. It is `toHubSummary()` +of the SAME `buildHomepageSummary` call the homepage makes: the input gathering (buildStats → read the +whole-pool pages → channel→sites map → summarise) moved out of `compose-homepage.ts` into +`common/controller/poolSummary.ts` (`buildPoolSummary`, `readStatsPages`, `channelSitesOf`), and both +composes call it. The hub's whole-pool stats pages go to `<exportIndexDir>/hub-stats` (staging, not +served). **The file is optional:** no index (`paths.lmdbPath` absent) → not written and any stale copy +removed; a failure → logged, removed, the hub build carries on. `compose-site.ts` removes it with +`hub-sites.json`, so a site bundle never ships it. Not in `HUB_CORS_PATHS` (read same-origin); +`builtHubProblem` unchanged. Why build time: a member's `corpus.json` carries only channel/video +counts, so a live sum could never equal the homepage's; the hub is rebuilt alongside the sites. + +**Proved equal, read-only:** a scratch script (`$T/c1-verify.mts`) ran `readStatsPages` over the +primary's `homepage/public/stats` (77,842 records), `listSites` over the primary's `sites/`, and +`buildHomepageSummary` + `toHubSummary`, and compared with `toHubSummary` of the live +`homepage/public/homepage-summary.json`: **MATCH** (official 5 sites / 63 channels / 75,785 recordings +/ 49,767 transcripts / 60,840 h / 480 gone). No LMDB was opened. `compose:hub` itself was NOT run +against the real corpus: `buildStats` writes the index's `statsByPath` sub-DB, so it is not read-only, +and the 15 GB index cannot be copied. In the worktree it ran with no corpus: "0 built-in pool site(s) +…; no index to summarise — hub-summary.json skipped." The first real `hub-summary.json` is written by +the parent's `build-hub`, whose log line says how many instances it covers. + +**The page, top to bottom, with its final copy** (`ArchiveShelf.tsx`, `HubStats.tsx`, +`AddArchive.tsx`, new `useHubSummary.ts`): +1. **H1** `{official.hoursArchived} hours of speech from every official archive, searchable at once.` + (no summary: `Every official archive, searchable at once.`; no summary AND no official instances: + `Every archive you add, searchable at once.`). **Paragraph:** "One search runs across all of them, + in your browser. You can add any other Archilyzer archive to your own search, below. It stays in + this browser until you remove it." +2. **`Official instances`** (h2) + "The archives I run. Anyone can run their own." + one card per + built-in member in `hub-sites.json` order: title linking out (`target="_blank"`, ↗), the summary's + description, and channels · recordings · transcripts · hours, each omitted when absent; the + `Installable`/`Data-only` badge. The stripe is the site's own `accent`; a site with none (all + five today) falls back to `seriesColor(i)` at its index in the summary's `sites` — the homepage's + `ArchiveCards` and chart order — so each hub card wears its homepage card's colour (the card's own + index when there is no summary). Figures match by `siteId`, then by origin. Zero built-ins: "This hub has no official instances. Add an archive to search it." +3. **`Archives you added`** (h2, only with externals) + the same card with the descriptor's channel + count only and the Remove control (`aria-label="Remove {title}"`), stripe its accent or + `var(--brand)`, then the form (the paragraph under the H1 says how adding works, once). The Add button is the + family's CTA (ink on ground, brand on hover), no longer a brand fill. +4. **Live line:** "Searching N archives · N channels · N transcripts right now." — the LIVE federation, + added archives included; the code comment says so, the page does not. +5. `/ask` hub branch: "…for other ways to use every archive on this hub." (was "this archive"). + `siteRegistry.ts` ALREADY: "That archive is already on this hub." (was "on your shelf"; no spec + asserts it). + +**Dropped:** "The shelf" eyebrow, the "Your archives" H1, the Member/Added badges (the two sections +replace them), the empty-shelf sentence. `data-testid="shelf-spine"` stays on every card in both +sections. + +**Changed locators** (same commit as the copy): `getByRole("heading", { name: "Your archives" })` → +`getByRole("heading", { level: 1, name: /searchable at once\./ })` in `e2e-hub/federation.spec.ts`, +`e2e-hub/ask-grounding.spec.ts`, `e2e-2origin/federation.spec.ts`. `e2e-hub/federation.spec.ts` +"adds an archive by URL and shows it on the shelf" is renamed "…shows it under Archives you added" and +also asserts the h2 and `Remove Origin B`. Unchanged: `getByLabel("Archive URL")`, button `Add`, +`getByTestId("shelf-spine")`, `p[role="alert"]`, `role="status"` `Added {title}.`, the `/ask` H1. +New `e2e-hub/official-instances.spec.ts` (+2): with a stubbed `hub-summary.json` the official card +shows its four figures, description and outbound link, the H1 carries the hours, and there is no Remove +and no "Archives you added"; with it 404, the card renders with no `dl` and the fallback H1. + +| sha | what | +|---|---| +| `ec5f8ebe` | `export/app/layout.tsx`: the hub defaults to the archilyzer family, dark, and the homepage's theme colour; sites unchanged | +| `6d847cdc` | `common/styles/tokens.css`: archilyzer `--chart-3` light `#5a9e3a`, dark `#86c86a` | +| `812d1b86` | `common/lib/hubSummary.ts` (+ test, 5), `common/controller/poolSummary.ts`, `compose-hub.ts` writes `hub-summary.json` (+ `compose-hub.test.ts`, 1: no index → no file, stale removed, no index created), `compose-homepage.ts` on `buildPoolSummary`, `compose-site.ts` removes it, `publish/build.ts` doc | +| `5a1bd3c0` | review fixes: `.gitignore` `/export/public/hub-summary.json`; H1 with no official instances; the add sentence once, scoped to this browser; official cards fall back to `seriesColor` in the homepage's order; generic hours in the CHANGELOG | +| `dc9ba0dd` | the hub page (`ArchiveShelf`, `HubStats`, `AddArchive`, `useHubSummary`, `HubHome` comment), `/ask` wording, registry ALREADY copy, the spec locators, `official-instances.spec.ts` | + +**Gates** (on `dc9ba0dd`): tsc clean (per commit). Common tests **1,822/1,822** (1,816 + 6). test:scripts +**161 passed + 1 skipped**; the first run, concurrent with the export builds, had 1 failure in +`scripts/queue-lock.test.mjs` "prints a banner naming the holder while waiting" — a timing test this +slice does not touch; the idle re-run passed. `pnpm --filter export exec next build` (site) ok; +`compose:hub` then `INSTANCE_MODE=hub … next build` ok (11 static routes); both with the one known +Turbopack warning. e2e, queued: **`e2e:hub` 11 passed** (9 + 2), 36 s (after ~6 min in the queue behind +`one-core/r9-fixes`); export `theme.spec.ts theme-family.spec.ts site-branding.spec.ts` **8 passed**, +29 s; **`e2e:2origin`** (`TWO_ORIGIN_REBUILD=1`) **3 passed**, 44 s — the six compose-hub outputs in +`export/public` swapped for copies before and relinked after; the primary's files were untouched +(mtimes 18:09, `hub-sites.json` still the five-site pool). A render of the built hub with the verified +summary and the real `hub-sites.json` (members' origins blocked) was checked at 1280 dark/light and +390 dark. The editor build, editor unit and mcp were not run: nothing they build or test changed +beyond `common/`, which tsc and the common tests cover. **Numbers tools: none.** + +**Review (SHIP AFTER FIXES) re-gate** on `5a1bd3c0`: tsc clean; `hubSummary.test.ts` + +`compose-hub.test.ts` 6/6; `e2e:hub` **11 passed**, 24.5 s. + +**Merged `main` (`ef0978a6`, release 9 slice F)** as `11e185c0`; `plans/release-9.md` and the +CHANGELOG resolved by keeping both. Re-gate on the merged tree: tsc clean; common tests **1,845/1,845** +(1,839 + 6); `pnpm --filter export exec next build` ok; `compose:hub` (worktree, no corpus: skipped) ++ `INSTANCE_MODE=hub … next build` ok; `e2e:hub` **11 passed**, 26.9 s (compose-hub outputs swapped for +copies and relinked, as before). **Projection re-check** with main's `buildHomepageSummary` +(`c714bc14`: official transcripts counted where placed, gone only when held), read-only over the +primary's `homepage/public/stats` (77,842 records, written 18:26): **MATCH** against the live +`homepage-summary.json` (22:26:11Z, written by the pre-change function) — nothing moved. +`official.transcripts` 49,767, `official.gone` 480; per-site gone Jeralyzer 473, Anilyzer 2, +Bonnellyzer 2, Hasanalyzer 3, Rekietalyzer 0 — identical before and after on this stats set. + +**Found and left:** +- The hub build now does what the homepage build does: `buildStats` opens the index LMDB read-write + and walks every channel's `data/`. An unmounted drive therefore undercounts those channels' figures + in `hub-summary.json` with no warning — the same exposure the homepage and a site's `build:stats` + already have, not a new one. +- The branch `one-core/r9-fixes` changes `buildHomepageSummary` (`c714bc14`: official.transcripts + counted where transcripts are placed). `hub-summary.json` is a projection of whatever it returns, so + after both merge the hub and the homepage still agree — as long as both are rebuilt from the same + commit. The MATCH above is against the pre-change function. +- No official site sets `accent`, so every official card wears its homepage series colour. A + `site.json` accent would override it on the hub only (the homepage ignores accents for its cards). +- The live line counts what loaded; while members load it climbs, and a member that fails silently + lowers it with no sign — C2's per-site state is the fix. +- The H1 swaps from the fallback to the hours sentence when `hub-summary.json` arrives (client fetch, + one frame of text change). Embedding it at build would need the hub page to read `public/` at build; + left as a fetch so the e2e can stub it. +- The chart-3 change repaints the homepage's third instance too; it needs a homepage rebuild to show. + ## Rollout