commit 265b55920af9ce9d52ff5b9ef23842cc5cbf0e0b
parent bfcd2fcf0b83266a64ad6d01528ea669fd339a8e
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 25 Sep 2026 12:02:06 -0400
Merge one-core/r7-cli — release 7 slice C: the archilyzer CLI, buildSite/deploySite/buildHub/deployHub entry points, prebuild twins collapsed, hub + homepage deploy path, posts-only manifest fix
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
47 files changed, 1941 insertions(+), 198 deletions(-)
diff --git a/DEPLOY_CLOUDFLARE.md b/DEPLOY_CLOUDFLARE.md
@@ -37,9 +37,12 @@ Oversize archives are staged during compose into `export/.r2-staging/<siteId>/ar
**Build & deploy** actions to `<bucket>/<siteId>/archives/<file>.zip` **before** the
Pages deploy runs, so the manifest URLs resolve immediately.
-> **Note:** only the **editor's** deploy actions upload to R2 (that's where the R2
-> credentials live). A raw `pnpm run build` / `pnpm deploy` from the `export`
-> package only *stages* the files — it does not upload them.
+> **Note:** every deploy path uploads oversize archives to R2 before the Pages
+> deploy: the editor's **Deploy** / **Build & deploy**, `pnpm ops deploy-site`, and
+> `archilyzer deploy site <id>` (which `pnpm run deploy` in `export/` runs, with
+> `SITE_ID`). A build alone (`archilyzer build site`, `pnpm run build`) only
+> *stages* them in `export/.r2-staging/`. The bucket is read from `settings.json`;
+> the R2 credentials come from the environment (see "3. Authenticate" below).
Uploads go through R2's **S3 API** (via the AWS SDK's multipart uploader), not
`wrangler r2 object put` — wrangler caps a single upload at **300 MiB**, and real
diff --git a/SETTINGS.md b/SETTINGS.md
@@ -483,7 +483,7 @@ Default:
## `homepageUrl`
-Absolute public URL of the family hub/homepage (e.g. "https://archilyzer.pages.dev"). Every export site links back to it ("the family" backlink) when set. Empty = no hub link rendered. Normalized to a trailing-slash-free http(s) URL.
+Absolute public URL of the family hub (e.g. "https://archilyzer-hub.pages.dev"). Every export site links back to it ("the family" backlink) when set. Empty = no hub link rendered. Normalized to a trailing-slash-free http(s) URL.
Default: `""`
diff --git a/common/bin/_cli.test.ts b/common/bin/_cli.test.ts
@@ -0,0 +1,120 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { parseArgv } from "./_parseFlags";
+import {
+ argumentProblem,
+ booleanFlags,
+ resolveCommand,
+ runCli,
+ usage,
+ type Command,
+} from "./_cli";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common test
+//
+// The archilyzer CLI's parser and lookup. The command table itself
+// (archilyzer.ts) is not imported: these are the rules every row obeys.
+
+function cmd(path: string[], extra: Partial<Command> = {}): Command {
+ return {
+ path,
+ usage: `the ${path.join(" ")} command`,
+ run: async () => 0,
+ ...extra,
+ };
+}
+
+test("parseArgv keeps positionals in order and reads both flag spellings", () => {
+ assert.deepEqual(
+ parseArgv(["deploy", "site", "anilyzer", "--preview", "tags", "--x=1"]),
+ { positionals: ["deploy", "site", "anilyzer"], flags: { preview: "tags", x: "1" } },
+ );
+});
+
+test("a boolean flag never swallows the positional after it", () => {
+ assert.deepEqual(parseArgv(["build", "site", "--nodata", "jeralyzer"], ["nodata"]), {
+ positionals: ["build", "site", "jeralyzer"],
+ flags: { nodata: true },
+ });
+ // Without the declaration it would — which is why runCli declares them.
+ assert.deepEqual(parseArgv(["--nodata", "jeralyzer"]).flags, { nodata: "jeralyzer" });
+});
+
+test("a lone -- is skipped, so `pnpm run build -- --nodata` still reads the flag", () => {
+ assert.deepEqual(parseArgv(["build", "site", "--", "--nodata"], ["nodata"]), {
+ positionals: ["build", "site"],
+ flags: { nodata: true },
+ });
+});
+
+test("a bare trailing flag and -h are true", () => {
+ assert.deepEqual(parseArgv(["deploy", "--preview"]).flags, { preview: true });
+ assert.deepEqual(parseArgv(["-h"]).flags, { help: true });
+});
+
+test("resolveCommand picks the LONGEST matching path and returns the rest", () => {
+ const table = [cmd(["build"]), cmd(["build", "site"]), cmd(["build", "hub"])];
+ const hit = resolveCommand(table, ["build", "site", "anilyzer"]);
+ assert.deepEqual(hit?.command.path, ["build", "site"]);
+ assert.deepEqual(hit?.rest, ["anilyzer"]);
+ assert.deepEqual(resolveCommand(table, ["build", "all"])?.command.path, ["build"]);
+ assert.equal(resolveCommand(table, ["deploy"]), null);
+ // Table order does not matter.
+ assert.deepEqual(
+ resolveCommand([...table].reverse(), ["build", "hub"])?.command.path,
+ ["build", "hub"],
+ );
+});
+
+test("argumentProblem refuses an unknown flag, a valued boolean, a bare string flag, an extra positional", () => {
+ const c = cmd(["deploy", "site"], {
+ flags: { preview: "string", nodata: "boolean" },
+ maxPositionals: 1,
+ });
+ assert.equal(argumentProblem(c, { preview: "x" }, ["a"]), null);
+ assert.equal(argumentProblem(c, { help: true }, []), null);
+ assert.match(argumentProblem(c, { preveiw: "x" }, [])!, /unknown flag --preveiw \(accepts --preview, --nodata\)/);
+ assert.match(argumentProblem(c, { nodata: "yes" }, [])!, /--nodata takes no value/);
+ assert.match(argumentProblem(c, { preview: true }, [])!, /--preview needs a value/);
+ assert.match(argumentProblem(c, {}, ["a", "b"])!, /unexpected argument "b"/);
+ assert.match(argumentProblem(cmd(["index"]), { x: true }, [])!, /it takes none/);
+});
+
+test("booleanFlags is the union of every boolean flag, plus help", () => {
+ const table = [
+ cmd(["a"], { flags: { nodata: "boolean", preview: "string" } }),
+ cmd(["b"], { flags: { check: "boolean" } }),
+ ];
+ assert.deepEqual(booleanFlags(table).sort(), ["check", "help", "nodata"]);
+});
+
+test("usage lists every command on its own line", () => {
+ const u = usage([cmd(["index"]), cmd(["build", "site"])]);
+ assert.match(u, /^Usage:/);
+ assert.match(u, /archilyzer index\s+the index command/);
+ assert.match(u, /archilyzer build site\s+the build site command/);
+});
+
+test("runCli runs the command with the rest, and refuses with 2 before running", async () => {
+ const seen: unknown[] = [];
+ const table = [
+ cmd(["build", "site"], {
+ flags: { nodata: "boolean" },
+ maxPositionals: 1,
+ run: async (ctx) => {
+ seen.push(ctx.positionals, ctx.flags);
+ return 7;
+ },
+ }),
+ ];
+ const quiet = { log: () => {}, error: () => {} };
+ assert.equal(await runCli(table, ["build", "site", "x", "--nodata"], {}, quiet), 7);
+ assert.deepEqual(seen, [["x"], { nodata: true }]);
+ assert.equal(await runCli(table, ["build", "site", "--bogus"], {}, quiet), 2);
+ assert.equal(await runCli(table, ["nope"], {}, quiet), 2);
+ assert.equal(await runCli(table, [], {}, quiet), 2);
+ assert.equal(await runCli(table, ["--help"], {}, quiet), 0);
+ assert.equal(await runCli(table, ["build", "site", "--help"], {}, quiet), 0);
+ assert.equal(seen.length, 2, "no refused call reached run");
+});
diff --git a/common/bin/_cli.ts b/common/bin/_cli.ts
@@ -0,0 +1,177 @@
+// The `archilyzer` CLI's machinery: a command table, the longest-path lookup
+// over it, and the usage text. Hand-rolled on purpose — there is no
+// commander/yargs in the workspace, and a table of a dozen rows does not earn
+// one. The table itself is `archilyzer.ts`; this file holds nothing that knows
+// what any command does, so it is unit-tested without importing one.
+
+import { realpathSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { parseArgv, type FlagValue } from "./_parseFlags";
+
+// A flag a command accepts. "boolean" never takes a value (`--nodata`),
+// "string" always needs one (`--preview <branch>`).
+export type FlagKind = "boolean" | "string";
+
+export type CommandContext = {
+ positionals: string[];
+ flags: Record<string, FlagValue>;
+ env: NodeJS.ProcessEnv;
+};
+
+export type Command = {
+ // The words that select it: ["build", "site"].
+ path: string[];
+ // One line: the arguments after the path, then a short description.
+ usage: string;
+ // Every flag it accepts. Anything else is refused BEFORE `run`, because a
+ // typo'd flag on a deploy must not quietly mean "production".
+ flags?: Record<string, FlagKind>;
+ // At most this many positionals after the path (default 0).
+ maxPositionals?: number;
+ // The exit code.
+ run: (ctx: CommandContext) => Promise<number>;
+};
+
+/**
+ * The command whose `path` is the LONGEST prefix of `positionals`, and the
+ * positionals left after it — or null when no command matches.
+ */
+export function resolveCommand(
+ table: readonly Command[],
+ positionals: readonly string[],
+): { command: Command; rest: string[] } | null {
+ let best: Command | null = null;
+ for (const c of table) {
+ if (c.path.length > positionals.length) continue;
+ if (!c.path.every((w, i) => positionals[i] === w)) continue;
+ if (!best || c.path.length > best.path.length) best = c;
+ }
+ return best ? { command: best, rest: positionals.slice(best.path.length) } : null;
+}
+
+/** Every boolean flag any command declares — what parseArgv must not feed a value. */
+export function booleanFlags(table: readonly Command[]): string[] {
+ const out = new Set<string>(["help"]);
+ for (const c of table) {
+ for (const [name, kind] of Object.entries(c.flags ?? {})) {
+ if (kind === "boolean") out.add(name);
+ }
+ }
+ return [...out];
+}
+
+/** The usage text: one line per command, in table order. */
+export function usage(table: readonly Command[], bin = "archilyzer"): string {
+ const rows = table.map((c) => [`${bin} ${c.path.join(" ")}`, c.usage] as const);
+ const width = Math.max(...rows.map(([head]) => head.length));
+ return [
+ `Usage:`,
+ ...rows.map(([head, text]) => ` ${head.padEnd(width)} ${text}`),
+ ].join("\n");
+}
+
+/**
+ * Why `flags` / `rest` do not fit `command`, as one sentence — or null.
+ * `help` is always allowed (the caller prints usage for it).
+ */
+export function argumentProblem(
+ command: Command,
+ flags: Record<string, FlagValue>,
+ rest: readonly string[],
+): string | null {
+ const name = command.path.join(" ");
+ const allowed = command.flags ?? {};
+ for (const [key, value] of Object.entries(flags)) {
+ if (key === "help") continue;
+ const kind = allowed[key];
+ if (!kind) {
+ const known = Object.keys(allowed);
+ return `${name}: unknown flag --${key}${
+ known.length ? ` (accepts ${known.map((k) => `--${k}`).join(", ")})` : " (it takes none)"
+ }`;
+ }
+ if (kind === "boolean" && value !== true) {
+ return `${name}: --${key} takes no value`;
+ }
+ if (kind === "string" && value === true) {
+ return `${name}: --${key} needs a value`;
+ }
+ }
+ const max = command.maxPositionals ?? 0;
+ if (rest.length > max) {
+ return `${name}: unexpected argument "${rest[max]}"`;
+ }
+ return null;
+}
+
+/**
+ * Parse argv, resolve the command, check its arguments and run it. Returns the
+ * exit code; every refusal prints to stderr and returns 2 (usage), which is
+ * what a shell script tests for.
+ */
+export async function runCli(
+ table: readonly Command[],
+ argv: readonly string[],
+ env: NodeJS.ProcessEnv = process.env,
+ out: { log: (s: string) => void; error: (s: string) => void } = console,
+): Promise<number> {
+ const { flags, positionals } = parseArgv(argv, booleanFlags(table));
+ if (positionals.length === 0) {
+ (flags.help ? out.log : out.error)(usage(table));
+ return flags.help ? 0 : 2;
+ }
+ const hit = resolveCommand(table, positionals);
+ if (!hit) {
+ out.error(`archilyzer: unknown command "${positionals.join(" ")}"\n\n${usage(table)}`);
+ return 2;
+ }
+ if (flags.help) {
+ out.log(usage([hit.command]));
+ return 0;
+ }
+ const problem = argumentProblem(hit.command, flags, hit.rest);
+ if (problem) {
+ out.error(`${problem}\n\n${usage([hit.command])}`);
+ return 2;
+ }
+ return hit.command.run({ positionals: hit.rest, flags, env });
+}
+
+/**
+ * True when the module at `metaUrl` is the script node was started with — the
+ * `scripts/worktree.mjs` idiom, so a bin can export `main` for the CLI to call
+ * AND keep working as `tsx bin/<name>.ts`. Both sides are realpath'd, because
+ * `import.meta.url` is always the real file and argv[1] may reach it through a
+ * workspace symlink.
+ */
+export function isEntryPoint(metaUrl: string): boolean {
+ const argv1 = process.argv[1];
+ if (!argv1) return false;
+ try {
+ return realpathSync(fileURLToPath(metaUrl)) === realpathSync(argv1);
+ } catch {
+ return false;
+ }
+}
+
+/**
+ * Run `main` when `metaUrl` is the entry point, the way every bin always has:
+ * a thrown error prints and exits 1; a returned number becomes the exit code
+ * without cutting the process short (a bin that returns nothing exits 0 when
+ * its work drains, exactly as before it exported anything).
+ */
+export function runIfEntryPoint(
+ metaUrl: string,
+ main: () => Promise<number | void>,
+): void {
+ if (!isEntryPoint(metaUrl)) return;
+ main().then(
+ (code) => {
+ if (typeof code === "number") process.exitCode = code;
+ },
+ (err: unknown) => {
+ console.error(err);
+ process.exit(1);
+ },
+ );
+}
diff --git a/common/bin/_parseFlags.ts b/common/bin/_parseFlags.ts
@@ -1,6 +1,6 @@
// Tiny argv parser: supports `--key value` and `--key=value`. Unknown
-// positional args are dropped. Good enough for the bin shims; reach for
-// commander/yargs if we ever need subcommands.
+// positional args are dropped. Good enough for the flag-driven bin shims; the
+// `archilyzer` CLI's subcommands use parseArgv below, which keeps positionals.
export function parseFlags(argv: string[]): Record<string, string> {
const out: Record<string, string> = {};
for (let i = 0; i < argv.length; i++) {
@@ -21,3 +21,51 @@ export function parseFlags(argv: string[]): Record<string, string> {
}
return out;
}
+
+// A flag's value: the string it was given, or `true` for a bare `--flag`.
+export type FlagValue = string | true;
+
+/**
+ * The `archilyzer` CLI's argv parser: positionals in order, plus flags.
+ *
+ * `--key=value` always carries its value. `--key value` takes the next word as
+ * the value UNLESS `key` is one of `booleans` (so `build site --nodata jeralyzer`
+ * keeps `jeralyzer` a positional) or the next word is itself a flag; a bare
+ * `--key` is `true`. A lone `--` is skipped rather than treated as the end of
+ * the flags, because `pnpm run build -- --nodata` hands the script its `--`
+ * verbatim and the flag after it must still be read as a flag. `-h` is `--help`.
+ */
+export function parseArgv(
+ argv: readonly string[],
+ booleans: Iterable<string> = [],
+): { flags: Record<string, FlagValue>; positionals: string[] } {
+ const bool = new Set(booleans);
+ const flags: Record<string, FlagValue> = {};
+ const positionals: string[] = [];
+ for (let i = 0; i < argv.length; i++) {
+ const a = argv[i];
+ if (a === "--") continue;
+ if (a === "-h") {
+ flags.help = true;
+ continue;
+ }
+ if (!a.startsWith("--")) {
+ positionals.push(a);
+ continue;
+ }
+ const eq = a.indexOf("=");
+ if (eq > 2) {
+ flags[a.slice(2, eq)] = a.slice(eq + 1);
+ continue;
+ }
+ const key = a.slice(2);
+ const next = argv[i + 1];
+ if (!bool.has(key) && next !== undefined && !next.startsWith("-")) {
+ flags[key] = next;
+ i++;
+ } else {
+ flags[key] = true;
+ }
+ }
+ return { flags, positionals };
+}
diff --git a/common/bin/archilyzer.ts b/common/bin/archilyzer.ts
@@ -0,0 +1,218 @@
+#!/usr/bin/env tsx
+// `archilyzer` — the one command line over the publish layer and the bins.
+//
+// pnpm --filter yt-dlp-transcript-common exec tsx bin/archilyzer.ts <command>
+// (or `tsx ../common/bin/archilyzer.ts …` from export/, as its scripts do)
+//
+// Every row imports its implementation LAZILY, so `archilyzer sync tick` never
+// loads the AWS SDK and `archilyzer settings example` never opens LMDB. The
+// machinery (parser, lookup, usage) is `_cli.ts`; this file is only the table.
+//
+// Not here yet (one-core Phase 4 slice 3): `doctor`, `run <operation>`, `mcp`.
+
+import type { Command } from "./_cli";
+import { runCli, runIfEntryPoint } from "./_cli";
+
+export const COMMANDS: Command[] = [
+ {
+ path: ["index"],
+ usage: "rebuild the LMDB transcript index",
+ run: async () => {
+ await (await import("./build-index")).main();
+ return 0;
+ },
+ },
+ {
+ path: ["compose", "site"],
+ usage: "<id> compose one site's export/public (default: SITE_ID)",
+ maxPositionals: 1,
+ run: async ({ positionals, env }) => {
+ const siteId = siteIdFrom(positionals, env, "compose site");
+ if (!siteId) return 2;
+ await (await import("./compose-site")).main({ siteId });
+ return 0;
+ },
+ },
+ {
+ path: ["compose", "hub"],
+ usage: "compose the hub's export/public (hub-sites.json, corpus.json, …)",
+ run: async () => {
+ await (await import("./compose-hub")).main();
+ return 0;
+ },
+ },
+ {
+ path: ["compose", "homepage"],
+ usage: "compose homepage/public (whole-pool stats + landing summary)",
+ run: async () => {
+ await (await import("./compose-homepage")).main();
+ return 0;
+ },
+ },
+ {
+ path: ["build", "site"],
+ usage:
+ "<id> [--nodata] [--skip-archives] data phase + compose + next build into export/out (default id: SITE_ID)",
+ flags: { nodata: "boolean", "skip-archives": "boolean" },
+ maxPositionals: 1,
+ run: async ({ positionals, flags, env }) => {
+ const siteId = siteIdFrom(positionals, env, "build site");
+ if (!siteId) return 2;
+ // A missing site.json reads as a site of defaults, so a typo would build
+ // the whole data phase before compose noticed. Refuse it up front.
+ const { listSiteIds } = await import("../lib/site");
+ const known = listSiteIds();
+ if (!known.includes(siteId)) {
+ console.error(
+ `build site: no site "${siteId}" (configured: ${known.join(", ") || "none"})`,
+ );
+ return 2;
+ }
+ const { buildSite } = await import("../publish/build");
+ const code = await buildSite(siteId, {
+ signal: interrupted(),
+ skipData: flags.nodata === true,
+ skipArchives: flags["skip-archives"] === true,
+ });
+ if (code !== 0) console.error(`build site ${siteId}: failed (exit ${code})`);
+ return code;
+ },
+ },
+ {
+ path: ["build", "all"],
+ usage:
+ "[--skip-archives] build every site: docker fan-out when an engine answers, else serially on the host",
+ flags: { "skip-archives": "boolean" },
+ run: async ({ flags }) => {
+ const { buildAll, dockerAvailable } = await import("../publish/build");
+ const signal = interrupted();
+ const useDocker = await dockerAvailable(signal);
+ if (!useDocker) {
+ console.log(
+ "[notice] No container engine available — building sites serially on the host.",
+ );
+ }
+ const outcomes = await buildAll({
+ signal,
+ mode: useDocker ? "docker" : "basic",
+ skipArchives: flags["skip-archives"] === true,
+ });
+ const failed = outcomes.filter((o) => o.code !== 0);
+ console.log(`\n=== Summary: ${outcomes.length - failed.length}/${outcomes.length} built ===`);
+ for (const f of failed) console.error(` ${f.siteId}: exit ${f.code}`);
+ return failed.length || signal.aborted ? 1 : 0;
+ },
+ },
+ {
+ path: ["build", "hub"],
+ usage: "compose:hub + INSTANCE_MODE=hub next build into export/out",
+ run: async () => {
+ const { buildHub } = await import("../publish/build");
+ const code = await buildHub({ signal: interrupted() });
+ if (code !== 0) console.error(`build hub: failed (exit ${code})`);
+ return code;
+ },
+ },
+ {
+ path: ["build", "homepage"],
+ usage: "compose + next build in homepage/ (reads the index as it stands)",
+ run: async () => {
+ const { buildHomepage } = await import("../publish/build");
+ const code = await buildHomepage({ signal: interrupted() });
+ if (code !== 0) console.error(`build homepage: failed (exit ${code})`);
+ return code;
+ },
+ },
+ {
+ path: ["deploy", "site"],
+ usage:
+ "<id> [--preview <branch>] ship the site built in export/out to its Pages project (default id: SITE_ID)",
+ flags: { preview: "string" },
+ maxPositionals: 1,
+ run: async ({ positionals, flags, env }) => {
+ const siteId = siteIdFrom(positionals, env, "deploy site");
+ if (!siteId) return 2;
+ const { deploySite } = await import("../publish/build");
+ return refusalsExit(() =>
+ deploySite(siteId, {
+ signal: interrupted(),
+ previewBranch: typeof flags.preview === "string" ? flags.preview : undefined,
+ }),
+ );
+ },
+ },
+ {
+ path: ["deploy", "hub"],
+ usage:
+ "[--preview <branch>] ship the hub built in export/out to homepage.json's Pages project",
+ flags: { preview: "string" },
+ run: async ({ flags }) => {
+ const { deployHub } = await import("../publish/build");
+ return refusalsExit(() =>
+ deployHub({
+ signal: interrupted(),
+ previewBranch: typeof flags.preview === "string" ? flags.preview : undefined,
+ }),
+ );
+ },
+ },
+ {
+ path: ["deploy", "homepage"],
+ usage: "ship homepage/out to the Pages project archilyzer (production branch main)",
+ run: async () => {
+ const { deployHomepage } = await import("../publish/build");
+ return refusalsExit(() => deployHomepage({ signal: interrupted() }));
+ },
+ },
+ {
+ path: ["sync", "tick"],
+ usage: "POST one scheduler tick to the editor (SYNC_TICK_URL, SYNC_TICK_TOKEN)",
+ run: async () => (await import("./sync-tick")).tick(),
+ },
+ {
+ path: ["settings", "example"],
+ usage: "[--check] write settings.json.example + SETTINGS.md from the schema",
+ flags: { check: "boolean" },
+ run: async ({ flags }) =>
+ (await import("./settings-example")).main({ check: flags.check === true }),
+ },
+];
+
+// The site a site command names: its argument, else SITE_ID (which is how
+// export's `build` / `deploy` scripts are called). Prints and returns null when
+// there is neither.
+function siteIdFrom(
+ positionals: string[],
+ env: NodeJS.ProcessEnv,
+ name: string,
+): string | null {
+ const id = (positionals[0] ?? env.SITE_ID ?? "").trim();
+ if (!id) {
+ console.error(`${name}: which site? Pass its id, or set SITE_ID.`);
+ return null;
+ }
+ return id;
+}
+
+// Ctrl-C cancels the step in flight the way the editor's Cancel does: the
+// signal reaches the child, and the entry point stops before its next step.
+function interrupted(): AbortSignal {
+ const ac = new AbortController();
+ process.once("SIGINT", () => ac.abort());
+ process.once("SIGTERM", () => ac.abort());
+ return ac.signal;
+}
+
+// An entry point that THROWS its refusals and failures (the deploys): the
+// sentence goes to stderr and the exit code is 1.
+async function refusalsExit(fn: () => Promise<void>): Promise<number> {
+ try {
+ await fn();
+ return 0;
+ } catch (err) {
+ console.error((err as Error).message);
+ return 1;
+ }
+}
+
+runIfEntryPoint(import.meta.url, () => runCli(COMMANDS, process.argv.slice(2)));
diff --git a/common/bin/build-archives.ts b/common/bin/build-archives.ts
@@ -13,7 +13,7 @@
// so an unchanged channel is reused, not re-zipped — this is a no-op on a warm
// cache and also speeds the basic (non-docker) build by collapsing N per-site
// passes into one union pass.
-import { getPaths } from "../lib/paths";
+import { getPaths, type Paths } from "../lib/paths";
import { getSettings } from "../lib/settings";
import { listSites } from "../lib/site";
import { openChannelSigner } from "../lib/channelSignature";
@@ -22,8 +22,9 @@ import {
archiveTranscripts,
} from "../controller/archiveTranscripts";
import { archiveLiveChat } from "../controller/archiveLiveChat";
+import { runIfEntryPoint } from "./_cli";
-async function main(): Promise<void> {
+export async function main(opts: { paths?: Paths } = {}): Promise<void> {
// Honor the same global/per-build opt-outs the per-site compose respects
// (archivesEnabled in compose-site.ts). The per-site `archives` flag is applied
// below when building the union.
@@ -36,7 +37,7 @@ async function main(): Promise<void> {
return;
}
- const paths = getPaths();
+ const paths = opts.paths ?? getPaths();
const sites = listSites(paths);
// Union of member slugs across sites that ship archives. A channel shared by
@@ -85,7 +86,4 @@ async function main(): Promise<void> {
console.log("[archives] cache warm complete.");
}
-main().catch((err) => {
- console.error(err);
- process.exit(1);
-});
+runIfEntryPoint(import.meta.url, () => main());
diff --git a/common/bin/build-chart-templates.ts b/common/bin/build-chart-templates.ts
@@ -2,19 +2,24 @@
// Bakes each site's editor-authored chart templates into its export staging dir
// as chart-templates.json, composed into the site's bundle by build:site. Falls
// back to built-in presets when a site has authored no templates file.
-import { getPaths } from "../lib/paths";
+import { getPaths, type Paths } from "../lib/paths";
import { listSiteIds } from "../lib/site";
import {
syncTemplatesToExport,
siteTemplatesStagingPath,
} from "../lib/chartsStore";
+import { runIfEntryPoint } from "./_cli";
-const paths = getPaths();
-const siteIds = listSiteIds(paths);
-if (siteIds.length === 0) {
- console.log("No sites configured; nothing to do.");
-}
-for (const siteId of siteIds) {
- syncTemplatesToExport(paths, siteId);
- console.log(`Wrote ${siteTemplatesStagingPath(paths, siteId)}`);
+export async function main(opts: { paths?: Paths } = {}): Promise<void> {
+ const paths = opts.paths ?? getPaths();
+ const siteIds = listSiteIds(paths);
+ if (siteIds.length === 0) {
+ console.log("No sites configured; nothing to do.");
+ }
+ for (const siteId of siteIds) {
+ syncTemplatesToExport(paths, siteId);
+ console.log(`Wrote ${siteTemplatesStagingPath(paths, siteId)}`);
+ }
}
+
+runIfEntryPoint(import.meta.url, () => main());
diff --git a/common/bin/build-index.ts b/common/bin/build-index.ts
@@ -1,8 +1,12 @@
#!/usr/bin/env tsx
-import { getPaths } from "../lib/paths";
+// `archilyzer index` — rebuild the LMDB transcript index. Also runs on its own
+// as `tsx bin/build-index.ts` (export's build:index script).
+import { getPaths, type Paths } from "../lib/paths";
import { buildIndex } from "../controller/buildIndex";
+import { runIfEntryPoint } from "./_cli";
-buildIndex({ paths: getPaths() }).catch((err) => {
- console.error(err);
- process.exit(1);
-});
+export async function main(opts: { paths?: Paths } = {}): Promise<void> {
+ await buildIndex({ paths: opts.paths ?? getPaths() });
+}
+
+runIfEntryPoint(import.meta.url, () => main());
diff --git a/common/bin/build-stats.ts b/common/bin/build-stats.ts
@@ -1,8 +1,12 @@
#!/usr/bin/env tsx
-import { getPaths } from "../lib/paths";
+// Rebuild the stats datasets. Runs as `tsx bin/build-stats.ts` (export's
+// build:stats script); `main` is exported for the archilyzer CLI.
+import { getPaths, type Paths } from "../lib/paths";
import { buildStats } from "../controller/buildStats";
+import { runIfEntryPoint } from "./_cli";
-buildStats({ paths: getPaths() }).catch((err) => {
- console.error(err);
- process.exit(1);
-});
+export async function main(opts: { paths?: Paths } = {}): Promise<void> {
+ await buildStats({ paths: opts.paths ?? getPaths() });
+}
+
+runIfEntryPoint(import.meta.url, () => main());
diff --git a/common/bin/compose-homepage.ts b/common/bin/compose-homepage.ts
@@ -13,7 +13,7 @@
import path from "node:path";
import { mkdir, readFile } from "node:fs/promises";
-import { getPaths } from "../lib/paths";
+import { getPaths, type Paths } from "../lib/paths";
import { writeJsonAtomic as writeJsonAtomicShared } from "../lib/jsonFile-server";
import { buildStats } from "../controller/buildStats";
import { listSites } from "../lib/site";
@@ -26,6 +26,7 @@ import {
buildHomepageSummary,
type ChannelSitesMap,
} from "../lib/homepageSummary";
+import { runIfEntryPoint } from "./_cli";
// Where the homepage Next.js app serves static assets from. Overridable for e2e
// test isolation, mirroring EXPORT_PUBLIC_DIR.
@@ -63,8 +64,8 @@ async function readStatsPages(statsDir: string): Promise<VideoStat[]> {
return out;
}
-async function main(): Promise<void> {
- const paths = getPaths();
+export async function main(opts: { paths?: Paths } = {}): Promise<void> {
+ const paths = opts.paths ?? getPaths();
const publicDir = homepagePublicDir(paths.monorepoRoot);
const statsDir = path.join(publicDir, "stats");
await mkdir(statsDir, { recursive: true });
@@ -106,7 +107,4 @@ async function main(): Promise<void> {
);
}
-main().catch((err) => {
- console.error(err);
- process.exit(1);
-});
+runIfEntryPoint(import.meta.url, () => main());
diff --git a/common/bin/compose-hub.ts b/common/bin/compose-hub.ts
@@ -13,7 +13,7 @@
import path from "node:path";
import { cp, rm, writeFile, access } from "node:fs/promises";
-import { getPaths } from "../lib/paths";
+import { getPaths, type Paths } from "../lib/paths";
import { listSites, resolveHubUrl } from "../lib/site";
import { getHomepageConfig } from "../lib/homepage";
import { SITE_DESCRIPTOR_VERSION } from "../lib/siteDescriptor";
@@ -24,6 +24,7 @@ import {
type HubMemberInput,
} from "../lib/corpus";
import { HUB_CORS_PATHS, renderHeadersFile } from "../lib/archive/headers";
+import { runIfEntryPoint } from "./_cli";
async function exists(p: string): Promise<boolean> {
try {
@@ -34,8 +35,8 @@ async function exists(p: string): Promise<boolean> {
}
}
-async function main(): Promise<void> {
- const paths = getPaths();
+export async function main(opts: { paths?: Paths } = {}): Promise<void> {
+ const paths = opts.paths ?? getPaths();
const publicDir = paths.exportPublicDir;
// Built-in pool: every configured site that publishes a public URL. The entry
@@ -105,7 +106,4 @@ async function main(): Promise<void> {
);
}
-main().catch((err) => {
- console.error(err);
- process.exit(1);
-});
+runIfEntryPoint(import.meta.url, () => main());
diff --git a/common/bin/compose-site.test.ts b/common/bin/compose-site.test.ts
@@ -0,0 +1,108 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { MANIFEST_ONLY_SIGNATURE, reconcileChannelTree } from "./compose-site";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common test
+//
+// reconcileChannelTree materializes a site's member subset of a shared
+// per-channel tree (transcripts, subs, posts, digests) into public/. Importing
+// it runs nothing: compose-site's main() only runs as the entry point.
+//
+// The case that motivated this file: a social channel's transcripts tree is a
+// manifest.json with `pageCount: 0` and nothing else (it never enters the video
+// scan), and the signature ignores manifest.json — so it signed as "" and was
+// DELETED, while corpus.json advertised it. jeralyzer's thequartering-X 404.
+
+function fixture(): { src: string; dest: string; cleanup: () => void } {
+ const root = mkdtempSync(path.join(tmpdir(), "compose-site-"));
+ const src = path.join(root, "shared");
+ const dest = path.join(root, "public");
+ mkdirSync(src, { recursive: true });
+ return { src, dest, cleanup: () => rmSync(root, { recursive: true, force: true }) };
+}
+
+const MANIFEST = JSON.stringify({ pageCount: 0, count: 0 });
+const quiet = () => {};
+
+test("a manifest-only member is copied, not removed", async () => {
+ const { src, dest, cleanup } = fixture();
+ try {
+ mkdirSync(path.join(src, "posts-only"));
+ writeFileSync(path.join(src, "posts-only", "manifest.json"), MANIFEST);
+ const sigs = await reconcileChannelTree("transcripts", src, dest, ["posts-only"], {}, quiet);
+ assert.deepEqual(sigs, { "posts-only": MANIFEST_ONLY_SIGNATURE });
+ assert.equal(
+ readFileSync(path.join(dest, "posts-only", "manifest.json"), "utf8"),
+ MANIFEST,
+ );
+ } finally {
+ cleanup();
+ }
+});
+
+test("an unchanged member is skipped, a changed one re-copied", async () => {
+ const { src, dest, cleanup } = fixture();
+ try {
+ mkdirSync(path.join(src, "chan"));
+ writeFileSync(path.join(src, "chan", "manifest.json"), MANIFEST);
+ writeFileSync(path.join(src, "chan", "page-0000.json"), "[1]");
+ const logs: string[] = [];
+ const log = (m: string) => logs.push(m);
+ const first = await reconcileChannelTree("transcripts", src, dest, ["chan"], {}, log);
+ assert.ok(first.chan && first.chan !== MANIFEST_ONLY_SIGNATURE);
+
+ // Unchanged source: the dest is left alone — a marker written into it
+ // survives, which a re-copy would have wiped.
+ writeFileSync(path.join(dest, "chan", "marker"), "x");
+ const second = await reconcileChannelTree("transcripts", src, dest, ["chan"], first, log);
+ assert.deepEqual(second, first);
+ assert.ok(existsSync(path.join(dest, "chan", "marker")));
+ assert.match(logs[1], /0 copied, 1 unchanged/);
+
+ // The same holds for a manifest-only member on its second compose.
+ mkdirSync(path.join(src, "social"));
+ writeFileSync(path.join(src, "social", "manifest.json"), MANIFEST);
+ const third = await reconcileChannelTree("transcripts", src, dest, ["chan", "social"], second, log);
+ writeFileSync(path.join(dest, "social", "marker"), "x");
+ await reconcileChannelTree("transcripts", src, dest, ["chan", "social"], third, log);
+ assert.ok(existsSync(path.join(dest, "social", "marker")));
+
+ // Pages arriving change the signature, so the tree is re-copied.
+ writeFileSync(path.join(src, "social", "page-0000.json"), "[2]");
+ const fourth = await reconcileChannelTree("transcripts", src, dest, ["chan", "social"], third, log);
+ assert.notEqual(fourth.social, MANIFEST_ONLY_SIGNATURE);
+ assert.ok(existsSync(path.join(dest, "social", "page-0000.json")));
+ assert.ok(!existsSync(path.join(dest, "social", "marker")));
+ } finally {
+ cleanup();
+ }
+});
+
+test("a member with no source (not even a manifest) is removed, and a non-member pruned", async () => {
+ const { src, dest, cleanup } = fixture();
+ try {
+ mkdirSync(path.join(dest, "gone"), { recursive: true });
+ writeFileSync(path.join(dest, "gone", "manifest.json"), MANIFEST);
+ mkdirSync(path.join(dest, "former"), { recursive: true });
+ mkdirSync(path.join(src, "empty-dir"));
+ mkdirSync(path.join(dest, "empty-dir"), { recursive: true });
+ const sigs = await reconcileChannelTree(
+ "transcripts",
+ src,
+ dest,
+ ["gone", "empty-dir"],
+ { gone: "old" },
+ quiet,
+ );
+ assert.deepEqual(sigs, {});
+ assert.ok(!existsSync(path.join(dest, "gone")));
+ assert.ok(!existsSync(path.join(dest, "empty-dir")));
+ assert.ok(!existsSync(path.join(dest, "former")));
+ } finally {
+ cleanup();
+ }
+});
diff --git a/common/bin/compose-site.ts b/common/bin/compose-site.ts
@@ -17,7 +17,7 @@ import path from "node:path";
import { cp, link, mkdir, rm, readdir, access, readFile, writeFile, stat, rename } from "node:fs/promises";
import { createHash } from "node:crypto";
import type { Dirent } from "node:fs";
-import { getPaths } from "../lib/paths";
+import { getPaths, type Paths } from "../lib/paths";
import { getSite, resolveSocialLinks, resolveHubUrl, type Site } from "../lib/site";
import { getSettings } from "../lib/settings";
import {
@@ -60,6 +60,7 @@ import {
type ArchiveManifest,
type ArchiveManifestEntry,
} from "../lib/archiveOptions";
+import { runIfEntryPoint } from "./_cli";
// Emit the public federation contract: /site.json (branding + channels +
// freshness) and the CORS _headers file. Emitted for EVERY site regardless of
@@ -580,7 +581,18 @@ async function dirSignature(
// the previous "rm -rf the whole tree then cp every member" with an in-place
// reconcile: only changed channels are re-copied, and channels no longer members
// are pruned. Returns the new per-slug signature map for the compose cache.
-async function reconcileChannelTree(
+//
+// A MANIFEST-ONLY TREE IS A CHANNEL WITH NOTHING IN IT, NOT A MISSING CHANNEL.
+// The signature ignores manifest.json (it churns), so a member whose source is
+// JUST the manifest — a social channel's transcripts tree, which buildIndex
+// writes with `pageCount: 0` because such a channel never enters the video scan
+// — used to sign as "" and be deleted, while corpus.json still advertised its
+// transcripts manifest to every reader: a 404 (jeralyzer's thequartering-X).
+// The manifest is exactly what "0 transcripts" means, so that tree is copied
+// under a constant signature and the contract stays unconditional.
+export const MANIFEST_ONLY_SIGNATURE = "manifest-only";
+
+export async function reconcileChannelTree(
kind: string,
srcRoot: string,
destRoot: string,
@@ -598,7 +610,10 @@ async function reconcileChannelTree(
const dest = path.join(destRoot, slug);
// Exclude the per-channel manifest.json — its `generatedAt` churns every
// mutation build; the page files capture real content changes.
- const sig = await dirSignature(src, "manifest.json");
+ let sig = await dirSignature(src, "manifest.json");
+ if (sig === "" && (await exists(path.join(src, "manifest.json")))) {
+ sig = MANIFEST_ONLY_SIGNATURE;
+ }
if (sig === "") {
// No source for this member — ensure no stale dest survives.
await rm(dest, { recursive: true, force: true });
@@ -631,13 +646,16 @@ async function reconcileChannelTree(
return next;
}
-async function main(): Promise<void> {
- const siteId = process.env.SITE_ID;
+// `archilyzer compose site <id>` passes the id; the bare bin (export's
+// compose:site script) reads SITE_ID, as it always has.
+export async function main(
+ opts: { siteId?: string; paths?: Paths } = {},
+): Promise<void> {
+ const siteId = opts.siteId?.trim() || process.env.SITE_ID;
if (!siteId) {
- console.error("compose-site: SITE_ID env var is required");
- process.exit(1);
+ throw new Error("compose-site: a site id is required (argument or SITE_ID env var)");
}
- const paths = getPaths();
+ const paths = opts.paths ?? getPaths();
const site = getSite(siteId, paths);
const memberSlugs = site.channels.map((c) => c.slug);
@@ -891,6 +909,11 @@ async function main(): Promise<void> {
// --- federation contract: /site.json descriptor + CORS _headers ---
await emitFederationFiles(site, paths);
+ // A site's bundle is not a hub's. export/public is shared with the hub build,
+ // whose compose writes hub-sites.json; left in place it ships in this site's
+ // out/ and makes the bundle ambiguous to builtHubProblem (and to a site's
+ // own registry, which treats the file as a hub's trusted pool).
+ await rm(path.join(paths.exportPublicDir, "hub-sites.json"), { force: true });
// --- service worker (only when this instance ships a PWA) ---
await composeServiceWorker(site, paths);
@@ -913,7 +936,4 @@ async function main(): Promise<void> {
);
}
-main().catch((err) => {
- console.error(err);
- process.exit(1);
-});
+runIfEntryPoint(import.meta.url, () => main());
diff --git a/common/bin/file-schemas-docs.ts b/common/bin/file-schemas-docs.ts
@@ -20,6 +20,7 @@ import {
renderSiteMarkdown,
} from "../lib/fileSchemaDocs";
import { parseFlags } from "./_parseFlags";
+import { runIfEntryPoint } from "./_cli";
const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
@@ -28,9 +29,9 @@ const FILE_SCHEMA_OUTPUTS: ReadonlyArray<[string, () => string]> = [
["CHANNEL.md", renderChannelMarkdown],
];
-async function main(): Promise<number> {
- const flags = parseFlags(process.argv.slice(2));
- const check = flags.check === "true";
+// `check` writes nothing and returns 1 when a committed file is stale.
+export async function main(opts: { check?: boolean } = {}): Promise<number> {
+ const check = opts.check === true;
let stale = 0;
for (const [name, render] of FILE_SCHEMA_OUTPUTS) {
const file = path.join(REPO, name);
@@ -49,10 +50,6 @@ async function main(): Promise<number> {
return stale > 0 ? 1 : 0;
}
-main().then(
- (code) => process.exit(code),
- (err) => {
- console.error(err);
- process.exit(1);
- },
+runIfEntryPoint(import.meta.url, () =>
+ main({ check: parseFlags(process.argv.slice(2)).check === "true" }),
);
diff --git a/common/bin/settings-example.ts b/common/bin/settings-example.ts
@@ -7,7 +7,7 @@
//
// `--check` writes nothing and exits 1 if either committed file differs from
// what the schema generates (the same claim common/lib/settingsDocs.test.ts
-// makes). Becomes `archilyzer settings example` in one-core phase 4.
+// makes). `archilyzer settings example [--check]` calls main().
//
// Reads no settings.json and writes no settings.json: both outputs are
// functions of the schema alone.
@@ -20,6 +20,7 @@ import {
renderSettingsMarkdown,
} from "../lib/settingsDocs";
import { parseFlags } from "./_parseFlags";
+import { runIfEntryPoint } from "./_cli";
const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
@@ -28,9 +29,9 @@ const OUTPUTS: ReadonlyArray<[string, () => string]> = [
["SETTINGS.md", renderSettingsMarkdown],
];
-async function main(): Promise<number> {
- const flags = parseFlags(process.argv.slice(2));
- const check = flags.check === "true";
+// `check` writes nothing and returns 1 when a committed file is stale.
+export async function main(opts: { check?: boolean } = {}): Promise<number> {
+ const check = opts.check === true;
let stale = 0;
for (const [name, render] of OUTPUTS) {
const file = path.join(REPO, name);
@@ -49,10 +50,6 @@ async function main(): Promise<number> {
return stale > 0 ? 1 : 0;
}
-main().then(
- (code) => process.exit(code),
- (err) => {
- console.error(err);
- process.exit(1);
- },
+runIfEntryPoint(import.meta.url, () =>
+ main({ check: parseFlags(process.argv.slice(2)).check === "true" }),
);
diff --git a/common/bin/sync-tick.ts b/common/bin/sync-tick.ts
@@ -15,11 +15,17 @@
// Exits non-zero on a network/HTTP error so cron surfaces failures (e.g. mails
// the output); a normal tick prints a one-line summary for the cron log.
+import { runIfEntryPoint } from "./_cli";
+
const DEFAULT_URL = "http://127.0.0.1:3001/api/scheduler/tick";
-async function main(): Promise<void> {
- const url = process.env.SYNC_TICK_URL ?? DEFAULT_URL;
- const token = process.env.SYNC_TICK_TOKEN;
+// The exit code: 1 on an HTTP error (cron mails it), 0 otherwise. A network
+// failure throws.
+export async function main(
+ opts: { url?: string; token?: string } = {},
+): Promise<number> {
+ const url = opts.url ?? process.env.SYNC_TICK_URL ?? DEFAULT_URL;
+ const token = opts.token ?? process.env.SYNC_TICK_TOKEN;
const headers: Record<string, string> = { "content-type": "application/json" };
if (token) headers.authorization = `Bearer ${token}`;
@@ -27,8 +33,7 @@ async function main(): Promise<void> {
const text = await res.text();
if (!res.ok) {
console.error(`[sync-tick] ${res.status} ${res.statusText}: ${text}`);
- process.exitCode = 1;
- return;
+ return 1;
}
try {
@@ -47,9 +52,17 @@ async function main(): Promise<void> {
} catch {
console.log(`[sync-tick] ${text}`);
}
+ return 0;
+}
+
+// Same message as ever for a network failure, which throws out of fetch().
+export async function tick(): Promise<number> {
+ try {
+ return await main();
+ } catch (err) {
+ console.error(`[sync-tick] request failed: ${(err as Error).message}`);
+ return 1;
+ }
}
-main().catch((err: unknown) => {
- console.error(`[sync-tick] request failed: ${(err as Error).message}`);
- process.exit(1);
-});
+runIfEntryPoint(import.meta.url, tick);
diff --git a/common/lib/builtExport.test.ts b/common/lib/builtExport.test.ts
@@ -3,7 +3,7 @@ import assert from "node:assert/strict";
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
-import { builtSiteIdIn, builtSiteProblem } from "./builtExport";
+import { builtHubProblem, builtSiteIdIn, builtSiteProblem } from "./builtExport";
function tempOut(siteJson?: string): { dir: string; cleanup: () => void } {
const dir = mkdtempSync(path.join(tmpdir(), "built-export-"));
@@ -84,3 +84,32 @@ test("builtSiteProblem says nothing was built rather than naming a mismatch", ()
cleanup();
}
});
+
+// The hub shares export/out with every site build. A hub bundle carries
+// hub-sites.json and no site.json (the hub build removes it); anything with a
+// site.json is some site's bundle, even when a stale hub-sites.json sits beside
+// it.
+test("builtHubProblem accepts only a hub bundle", () => {
+ const hub = tempOut();
+ const site = tempOut(JSON.stringify({ siteId: "jeralyzer" }));
+ const none = tempOut();
+ try {
+ writeFileSync(path.join(hub.dir, "hub-sites.json"), "[]");
+ assert.equal(builtHubProblem(hub.dir), null);
+
+ writeFileSync(path.join(site.dir, "hub-sites.json"), "[]");
+ assert.equal(
+ builtHubProblem(site.dir),
+ 'export/out holds a build of "jeralyzer", not the hub — build the hub first',
+ );
+
+ assert.equal(
+ builtHubProblem(none.dir),
+ "export/out holds no hub build — build the hub first",
+ );
+ } finally {
+ hub.cleanup();
+ site.cleanup();
+ none.cleanup();
+ }
+});
diff --git a/common/lib/builtExport.ts b/common/lib/builtExport.ts
@@ -13,7 +13,7 @@
// public/ into out/. So the check is a file read, and it is cheap enough to do
// before every deploy.
-import { readFileSync } from "node:fs";
+import { existsSync, readFileSync } from "node:fs";
import path from "node:path";
/**
@@ -58,3 +58,25 @@ export function builtSiteProblem(outDir: string, siteId: string): string | null
}
return null;
}
+
+/**
+ * Why `outDir` may not be deployed as the HUB, as one sentence — or null when
+ * it holds a hub build.
+ *
+ * The hub is the export app built with INSTANCE_MODE=hub into the SAME
+ * export/out a site build uses, so the two overwrite each other. A hub build
+ * names itself by what it carries and what it does not: compose-hub writes
+ * `hub-sites.json` and the hub build removes `site.json` first, while a site's
+ * compose removes `hub-sites.json` and writes `site.json`. A site.json here is
+ * therefore a site's bundle, whatever else is beside it.
+ */
+export function builtHubProblem(outDir: string): string | null {
+ const site = builtSiteIdIn(outDir);
+ if (site !== null) {
+ return `export/out holds a build of "${site}", not the hub — build the hub first`;
+ }
+ if (!existsSync(path.join(outDir, "hub-sites.json"))) {
+ return "export/out holds no hub build — build the hub first";
+ }
+ return null;
+}
diff --git a/common/lib/homepage.ts b/common/lib/homepage.ts
@@ -28,9 +28,10 @@ export type HomepageConfig = {
// SiteSettings.socialLinks; an array (even empty) overrides it. Same semantics
// as Site.socialLinks — resolve with resolveHomepageSocialLinks() at render.
socialLinks?: SocialLink[];
- // Absolute public URL of the deployed hub, e.g. "https://archilyzer.pages.dev".
+ // Absolute public URL of the deployed hub, e.g. "https://archilyzer-hub.pages.dev".
siteUrl?: string;
- // Cloudflare Pages project the hub deploys to.
+ // Cloudflare Pages project the hub deploys to (e.g. "archilyzer-hub"; never
+ // "archilyzer", the homepage's — deployHub refuses it).
cloudflareProject?: string;
// The transcript modal's per-video export controls (Download menu, Copy MD,
// Copy download command) on the hub's Browse and Ask pages — the same switch
diff --git a/common/lib/settingsSchema.ts b/common/lib/settingsSchema.ts
@@ -1515,7 +1515,7 @@ export const siteSettingsSchema = z.object({
"Default social links applied to every site that doesn't define its own. A site inherits these unless its site.json carries an explicit `socialLinks` array — see Site.socialLinks / resolveSocialLinks in common/lib/site.ts. The one presentation field that lives globally so a shared footer doesn't have to be repeated per site.",
),
homepageUrl: settingsField((v): string => normalizeHomepageUrl(v)).describe(
- "Absolute public URL of the family hub/homepage (e.g. \"https://archilyzer.pages.dev\"). Every export site links back to it (\"the family\" backlink) when set. Empty = no hub link rendered. Normalized to a trailing-slash-free http(s) URL.",
+ "Absolute public URL of the family hub (e.g. \"https://archilyzer-hub.pages.dev\"). Every export site links back to it (\"the family\" backlink) when set. Empty = no hub link rendered. Normalized to a trailing-slash-free http(s) URL.",
),
savedVideoBackup: settingsField((v): SavedVideoBackupSettings => sanitizeSavedVideoBackup(v)).describe(
"Backup configuration for the saved-video store (Phase 4 of the video-persistence feature). When enabled with a destination, the store is mirrored there (additively, no deletes) with a per-backup manifest, and the sync scheduler runs the backup on the configured cadence. See common/controller/backupSavedVideos.ts.",
diff --git a/common/package.json b/common/package.json
@@ -45,7 +45,7 @@
"./styles/*": "./styles/*.ts"
},
"scripts": {
- "test": "tsx --test \"*.test.ts\" \"{lib,controller,jobs,social,ytdlp,components,views,publish}/*.test.ts\" \"{lib,controller,jobs,social,ytdlp,components,views,publish}/*/*.test.ts\""
+ "test": "tsx --test \"*.test.ts\" \"{lib,controller,jobs,social,ytdlp,components,views,publish,bin}/*.test.ts\" \"{lib,controller,jobs,social,ytdlp,components,views,publish,bin}/*/*.test.ts\""
},
"dependencies": {
"@aws-sdk/client-s3": "^3.1080.0",
diff --git a/common/publish/build.test.ts b/common/publish/build.test.ts
@@ -2,6 +2,10 @@ import { test } from "node:test";
import assert from "node:assert/strict";
import type { Paths } from "../lib/paths";
import {
+ HOMEPAGE_PAGES_PROJECT,
+ buildHubSteps,
+ buildSiteSteps,
+ hubProjectProblem,
dockerSiteOutDir,
dockerSiteStagingDir,
resolveOutDir,
@@ -17,6 +21,8 @@ import {
const paths = {
exportDir: "/repo/export",
exportBuildsDir: "/repo/export/.export-builds",
+ exportPublicDir: "/repo/export/public",
+ transcriptsDir: "/data/transcripts",
} as Paths;
test("resolveOutDir is export/out whatever the site", () => {
@@ -41,3 +47,82 @@ test("dockerSiteStagingDir nests .r2-staging/<site>/archives under the site's bu
"/repo/export/.export-builds/jeralyzer/.r2-staging/jeralyzer/archives",
);
});
+
+// The basic build of one site, as the children it runs. Pinned because the
+// data phase used to be npm's `prebuild` hook on a script called `build`, and
+// `build:nodata` (the same body under another name) was how it was skipped —
+// both scripts are gone, so this list is now the whole contract.
+test("buildSiteSteps: data phase, compose, next build — all in export/, one env", () => {
+ const steps = buildSiteSteps({
+ siteId: "jeralyzer",
+ paths,
+ baseEnv: { PATH: "/bin" },
+ });
+ const env = {
+ PATH: "/bin",
+ NODE_ENV: "production",
+ TRANSCRIPTS_DIR: "/data/transcripts",
+ EXPORT_PUBLIC_DIR: "/repo/export/public",
+ SITE_ID: "jeralyzer",
+ };
+ assert.deepEqual(steps, [
+ { command: "pnpm", args: ["run", "build:data"], cwd: "/repo/export", env },
+ { command: "pnpm", args: ["run", "compose:site"], cwd: "/repo/export", env },
+ { command: "pnpm", args: ["exec", "next", "build"], cwd: "/repo/export", env },
+ ]);
+});
+
+test("buildSiteSteps: skipData drops the data phase; skipArchives sets BUILD_ARCHIVES=0", () => {
+ const steps = buildSiteSteps({
+ siteId: "anilyzer",
+ paths,
+ skipData: true,
+ skipArchives: true,
+ baseEnv: {},
+ });
+ assert.deepEqual(
+ steps.map((s) => s.args.join(" ")),
+ ["run compose:site", "exec next build"],
+ );
+ for (const s of steps) {
+ assert.equal(s.env.BUILD_ARCHIVES, "0");
+ assert.equal(s.env.SITE_ID, "anilyzer");
+ }
+ assert.equal(
+ buildSiteSteps({ siteId: "a", paths, baseEnv: {} })[0].env.BUILD_ARCHIVES,
+ undefined,
+ );
+});
+
+test("buildHubSteps: compose:hub, then next build with INSTANCE_MODE=hub, in export/", () => {
+ const steps = buildHubSteps({ paths, baseEnv: { PATH: "/bin" } });
+ const env = {
+ PATH: "/bin",
+ NODE_ENV: "production",
+ TRANSCRIPTS_DIR: "/data/transcripts",
+ EXPORT_PUBLIC_DIR: "/repo/export/public",
+ };
+ assert.deepEqual(steps, [
+ { command: "pnpm", args: ["run", "compose:hub"], cwd: "/repo/export", env },
+ {
+ command: "pnpm",
+ args: ["exec", "next", "build"],
+ cwd: "/repo/export",
+ env: { ...env, INSTANCE_MODE: "hub" },
+ },
+ ]);
+});
+
+// The hub and the homepage are two Pages projects. homepage.json's project was
+// "archilyzer" — the homepage's — before the hub could deploy, so that value
+// is refused by name rather than trusted.
+test("hubProjectProblem refuses a missing project and the homepage's", () => {
+ assert.equal(HOMEPAGE_PAGES_PROJECT, "archilyzer");
+ assert.equal(
+ hubProjectProblem(undefined),
+ "The hub has no Cloudflare Pages project configured — set it on /sites under Hub.",
+ );
+ assert.equal(hubProjectProblem(" "), hubProjectProblem(undefined));
+ assert.match(hubProjectProblem("archilyzer")!, /homepage's — set the hub's own project/);
+ assert.equal(hubProjectProblem("archilyzer-hub"), null);
+});
diff --git a/common/publish/build.ts b/common/publish/build.ts
@@ -9,19 +9,22 @@
// forbids non-serializable args). Keep them as plain helpers.
import path from "node:path";
-import { mkdir, readdir, stat } from "node:fs/promises";
+import { mkdir, readdir, rm, stat } from "node:fs/promises";
import { createReadStream, existsSync } from "node:fs";
import { S3Client, HeadObjectCommand } from "@aws-sdk/client-s3";
import { Upload } from "@aws-sdk/lib-storage";
import { runChildIntoLog } from "../jobs/runChild";
+import { builtHubProblem, builtSiteProblem } from "../lib/builtExport";
+import { getHomepageConfig } from "../lib/homepage";
import {
deploymentUrlIn,
pagesDeployArgs,
previewAliasUrl,
+ previewBranchProblem,
} from "../lib/pagesDeploy";
-import type { Paths } from "../lib/paths";
+import { getPaths, type Paths } from "../lib/paths";
import { getSettings } from "../lib/settings";
-import type { Site } from "../lib/site";
+import { getSite, listSites, type Site } from "../lib/site";
// Where the basic (host) build writes the static bundle to deploy: the fixed
// export/out, composed one site at a time. The docker fan-out writes per-site
@@ -43,9 +46,79 @@ export function dockerSiteStagingDir(paths: Paths, siteId: string): string {
return path.join(paths.exportBuildsDir, siteId, ".r2-staging", siteId, "archives");
}
+// One child process of a build: what to run, where, with which environment.
+export type BuildStep = {
+ command: string;
+ args: string[];
+ cwd: string;
+ env: NodeJS.ProcessEnv;
+};
+
+// The basic (host) build of one site, as the child processes it runs, in order:
+// the pool-wide data phase (index + stats + chart templates) unless `skipData`,
+// then compose:site for THIS site, then `next build` — all in export/, all with
+// the same environment.
+//
+// These used to be ONE child, `pnpm run build` (or `build:nodata`), whose data
+// phase was npm's `prebuild` lifecycle hook: two scripts with identical bodies
+// that differed only by name, so the hook fired for one and not the other. The
+// hook is gone and the data phase is an explicit step here, which is what lets
+// export's `build` script BE this function (`archilyzer build site`) without
+// running itself. `skipData` still assumes a prior full build's
+// .export-index staging.
+//
+// `baseEnv` is the environment the steps inherit (process.env by default);
+// passing one is how the test pins the argv without the host's.
+export function buildSiteSteps(opts: {
+ siteId: string;
+ paths: Paths;
+ skipData?: boolean;
+ skipArchives?: boolean;
+ baseEnv?: NodeJS.ProcessEnv;
+}): BuildStep[] {
+ const { paths } = opts;
+ const env: NodeJS.ProcessEnv = {
+ ...(opts.baseEnv ?? process.env),
+ NODE_ENV: "production",
+ TRANSCRIPTS_DIR: paths.transcriptsDir,
+ EXPORT_PUBLIC_DIR: paths.exportPublicDir,
+ SITE_ID: opts.siteId,
+ // Per-build opt-out for the bulk-download archive zips. BUILD_ARCHIVES=0
+ // makes compose-site skip generation this build regardless of the
+ // global/site flags.
+ ...(opts.skipArchives ? { BUILD_ARCHIVES: "0" } : {}),
+ };
+ const step = (args: string[]): BuildStep => ({
+ command: "pnpm",
+ args,
+ cwd: paths.exportDir,
+ env,
+ });
+ return [
+ ...(opts.skipData ? [] : [step(["run", "build:data"])]),
+ step(["run", "compose:site"]),
+ step(["exec", "next", "build"]),
+ ];
+}
+
+// Run a list of child steps in order, streaming into `onLog`, stopping at the
+// first non-zero exit (or a cancel). Returns the exit code.
+async function runSteps(
+ onLog: (line: string) => void,
+ signal: AbortSignal,
+ steps: BuildStep[],
+): Promise<number> {
+ for (const s of steps) {
+ if (signal.aborted) return 1;
+ const code = await runChildIntoLog(onLog, signal, s);
+ if (code !== 0) return code;
+ }
+ return signal.aborted ? 1 : 0;
+}
+
// Run the basic (host) build phase for one site, streaming into `onLog`,
-// returning the exit code. Runs `pnpm run build` in export/ (serialized upstream
-// on the build queue, since the export/ tree is shared). This is the single-site
+// returning the exit code: buildSiteSteps, in export/ (serialized upstream on
+// the build queue, since the export/ tree is shared). This is the single-site
// build for basic mode, and the fallback the all-sites docker action drops to
// when no container engine is available. The parallel per-site container path
// lives in runDockerBuildAllPhase.
@@ -56,11 +129,8 @@ export async function runBuildPhase(
paths: Paths,
opts?: { skipData?: boolean; skipArchives?: boolean },
): Promise<number> {
- // When skipping the data rebuild, run the `build:nodata` script instead of
- // `build`. `build:nodata` has the same body (compose:site + next build) but a
- // different name, so npm's `prebuild` lifecycle hook (which runs build:data =
- // index/stats/templates) does NOT fire — we compose from the existing
- // .export-index staging. Assumes a prior full build produced that staging.
+ // Skipping the data rebuild composes from the existing .export-index staging
+ // (see buildSiteSteps).
const skipData = opts?.skipData === true;
if (skipData) {
onLog(
@@ -68,25 +138,15 @@ export async function runBuildPhase(
"existing .export-index staging.\n",
);
}
- // Per-build opt-out for the bulk-download archive zips. BUILD_ARCHIVES=0 makes
- // compose-site skip generation this build regardless of the global/site flags.
const skipArchives = opts?.skipArchives === true;
if (skipArchives) {
onLog("[notice] Skipping archive-zip generation for this build.\n");
}
- return runChildIntoLog(onLog, signal, {
- command: "pnpm",
- args: ["run", skipData ? "build:nodata" : "build"],
- cwd: paths.exportDir,
- env: {
- ...process.env,
- NODE_ENV: "production",
- TRANSCRIPTS_DIR: paths.transcriptsDir,
- EXPORT_PUBLIC_DIR: paths.exportPublicDir,
- SITE_ID: siteId,
- ...(skipArchives ? { BUILD_ARCHIVES: "0" } : {}),
- },
- });
+ return runSteps(
+ onLog,
+ signal,
+ buildSiteSteps({ siteId, paths, skipData, skipArchives }),
+ );
}
// Where the basic (host) compose staged this site's oversize archives for R2
@@ -580,3 +640,356 @@ async function runWithConcurrency<T, R>(
await Promise.all(runners);
return results;
}
+
+// ---------------------------------------------------------------------------
+// Named entry points (one-core Phase 4 slice 2): one call per thing an operator
+// publishes, over the run* phases above. The editor's actions wrap these in
+// runManagedFunction (a job, a queue, a log); the archilyzer CLI calls them
+// straight, logging to the terminal. Every run* export stays — they are still
+// the parts, and build.test.ts pins some of them.
+// ---------------------------------------------------------------------------
+
+export type PublishOpts = {
+ paths?: Paths;
+ // Default: the terminal.
+ onLog?: (line: string) => void;
+ // Default: never aborted.
+ signal?: AbortSignal;
+};
+
+// runChildIntoLog hands onLog lines WITHOUT their newline and the actions' own
+// notices end in one; the terminal gets exactly one either way.
+export function terminalLog(line: string): void {
+ process.stdout.write(line.endsWith("\n") ? line : `${line}\n`);
+}
+
+function resolved(opts: PublishOpts): {
+ paths: Paths;
+ onLog: (line: string) => void;
+ signal: AbortSignal;
+} {
+ return {
+ paths: opts.paths ?? getPaths(),
+ onLog: opts.onLog ?? terminalLog,
+ signal: opts.signal ?? new AbortController().signal,
+ };
+}
+
+/** Build one site into export/out (basic/host mode). Returns the exit code. */
+export async function buildSite(
+ siteId: string,
+ opts: PublishOpts & { skipData?: boolean; skipArchives?: boolean } = {},
+): Promise<number> {
+ const { paths, onLog, signal } = resolved(opts);
+ return runBuildPhase(onLog, signal, siteId.trim(), paths, {
+ skipData: opts.skipData,
+ skipArchives: opts.skipArchives,
+ });
+}
+
+/**
+ * Deploy the site already built in export/out: its oversize archives to R2,
+ * then the bundle to its Pages project (a PREVIEW with `previewBranch`).
+ * THROWS on every refusal and failure, with the sentences the editor's deploy
+ * job has always ended on; returns quietly on a cancel.
+ *
+ * The refusals are checked here even though the editor action checks them
+ * before it starts the job, because a queued deploy can start after another
+ * site's build has replaced export/out — the action's check is the fast answer,
+ * this one is the last word.
+ */
+export async function deploySite(
+ siteId: string,
+ opts: PublishOpts & { previewBranch?: string } = {},
+): Promise<void> {
+ const { paths, onLog, signal } = resolved(opts);
+ if (opts.previewBranch !== undefined) {
+ const problem = previewBranchProblem(opts.previewBranch);
+ if (problem) throw new Error(problem);
+ }
+ const branch = opts.previewBranch?.trim() || undefined;
+ const site = getSite(siteId.trim(), paths);
+ if (!site.cloudflareProject) {
+ throw new Error(
+ `Site "${site.siteId}" has no Cloudflare Pages project configured.`,
+ );
+ }
+ const outDir = resolveOutDir(site.siteId, paths);
+ const builtProblem = builtSiteProblem(outDir, site.siteId);
+ if (builtProblem) throw new Error(builtProblem);
+ // The production path logs no banner and gains none here: its log has
+ // always opened on wrangler's own first line.
+ if (branch) {
+ onLog(`=== Deploy (preview "${branch}") ===\n`);
+ onLog(PREVIEW_SHARES_ARCHIVES_NOTICE);
+ }
+ // Push oversize archives to R2 first, so the manifest URLs the Pages deploy
+ // publishes resolve immediately. No-op when R2 isn't configured.
+ const uploadCode = await runArchiveUploadIntoLog(onLog, signal, site, paths);
+ if (signal.aborted) return;
+ if (uploadCode !== 0) {
+ throw new Error(`Archive R2 upload failed (exit ${uploadCode}).`);
+ }
+ const code = await runDeployIntoLog(onLog, signal, site, outDir, paths, {
+ previewBranch: branch,
+ });
+ if (signal.aborted) return;
+ if (code !== 0) throw new Error(`Deploy failed (exit ${code}).`);
+}
+
+/**
+ * Build every configured site (or `sites`), without deploying. "docker" is the
+ * parallel per-site container fan-out (runDockerBuildAllPhase, which throws on
+ * an infrastructure failure); "basic" is a serial host loop whose shared
+ * export/out is overwritten per site, so only the last survives. The pool-wide
+ * data phase runs once either way. Per-site failures are returned, not thrown.
+ */
+export async function buildAll(
+ opts: PublishOpts & {
+ mode: "docker" | "basic";
+ skipArchives?: boolean;
+ sites?: Site[];
+ },
+): Promise<SiteBuildOutcome[]> {
+ const { paths, onLog, signal } = resolved(opts);
+ const sites = opts.sites ?? listSites(paths);
+ if (opts.mode === "docker") {
+ return runDockerBuildAllPhase(onLog, signal, sites, paths, {
+ skipArchives: opts.skipArchives,
+ });
+ }
+ const outcomes: SiteBuildOutcome[] = [];
+ for (let i = 0; i < sites.length; i++) {
+ if (signal.aborted) break;
+ const site = sites[i];
+ onLog(`\n=== Build ${site.siteId} (${i + 1}/${sites.length}) ===`);
+ const code = await runBuildPhase(onLog, signal, site.siteId, paths, {
+ skipData: i > 0,
+ skipArchives: opts.skipArchives,
+ });
+ outcomes.push({ siteId: site.siteId, code });
+ }
+ return outcomes;
+}
+
+// ---------------------------------------------------------------------------
+// The hub and the homepage — two apps, two Pages projects (decision
+// 2026-09-25).
+//
+// The HUB is the export app built with INSTANCE_MODE=hub: a federating shell
+// over every site that has a public URL, branded by homepage.json and deployed
+// to homepage.json's `cloudflareProject` (`archilyzer-hub`). It builds into the
+// same export/out as a site — the two overwrite each other, and
+// builtHubProblem / builtSiteProblem refuse to deploy the wrong one.
+//
+// The HOMEPAGE is the `homepage` package: the software's own site (docs, the
+// source download), deployed to the constant project `archilyzer`
+// (https://archilyzer.pages.dev, PROJECT_URL). It is never the hub.
+// ---------------------------------------------------------------------------
+
+/** The homepage package's Pages project. Constant: it is the product's site. */
+export const HOMEPAGE_PAGES_PROJECT = "archilyzer";
+
+/**
+ * Why `project` may not be the hub's deploy target, as one sentence — or null.
+ * The homepage's project is refused by name: homepage.json said `archilyzer`
+ * until the hub got a deploy path, and a hub deployed there would replace the
+ * software's own site.
+ */
+export function hubProjectProblem(project: string | undefined): string | null {
+ const p = project?.trim();
+ if (!p) {
+ return "The hub has no Cloudflare Pages project configured — set it on /sites under Hub.";
+ }
+ if (p === HOMEPAGE_PAGES_PROJECT) {
+ return (
+ `The hub's Cloudflare Pages project is "${p}", which is the Archilyzer ` +
+ `homepage's — set the hub's own project (for example "archilyzer-hub") ` +
+ `on /sites under Hub.`
+ );
+ }
+ return null;
+}
+
+/**
+ * The hub build, as the children it runs: compose:hub (hub-sites.json,
+ * corpus.json, llms.txt, robots.txt, _headers, sw.js into export/public), then
+ * `next build` with INSTANCE_MODE=hub — both in export/.
+ */
+export function buildHubSteps(opts: {
+ paths: Paths;
+ baseEnv?: NodeJS.ProcessEnv;
+}): BuildStep[] {
+ const { paths } = opts;
+ const env: NodeJS.ProcessEnv = {
+ ...(opts.baseEnv ?? process.env),
+ NODE_ENV: "production",
+ TRANSCRIPTS_DIR: paths.transcriptsDir,
+ EXPORT_PUBLIC_DIR: paths.exportPublicDir,
+ };
+ return [
+ { command: "pnpm", args: ["run", "compose:hub"], cwd: paths.exportDir, env },
+ {
+ command: "pnpm",
+ args: ["exec", "next", "build"],
+ cwd: paths.exportDir,
+ env: { ...env, INSTANCE_MODE: "hub" },
+ },
+ ];
+}
+
+/** Compose the hub's export/public, as a child. Returns the exit code. */
+export async function composeHub(opts: PublishOpts = {}): Promise<number> {
+ const { paths, onLog, signal } = resolved(opts);
+ return runSteps(onLog, signal, buildHubSteps({ paths }).slice(0, 1));
+}
+
+/**
+ * Build the hub into export/out. Removes public/site.json first — a site's
+ * compose left it there, and a hub bundle carrying one would read as that
+ * site's (builtExport.ts). Returns the exit code.
+ */
+export async function buildHub(opts: PublishOpts = {}): Promise<number> {
+ const { paths, onLog, signal } = resolved(opts);
+ await rm(path.join(paths.exportPublicDir, "site.json"), { force: true });
+ return runSteps(onLog, signal, buildHubSteps({ paths }));
+}
+
+// One Pages deploy of `outDir` to `project`, streaming into onLog, with the
+// deployment URL (or the preview alias) repeated as the last line on success
+// — runDeployIntoLog's shape, for a bundle that is not a Site.
+async function runPagesDeployIntoLog(
+ onLog: (line: string) => void,
+ signal: AbortSignal,
+ opts: {
+ outDir: string;
+ project: string;
+ cwd: string;
+ previewBranch?: string;
+ extraArgs?: string[];
+ },
+): Promise<number> {
+ let deploymentUrl: string | null = null;
+ const watch = (line: string) => {
+ if (deploymentUrl === null) deploymentUrl = deploymentUrlIn(line, opts.project);
+ onLog(line);
+ };
+ const code = await runChildIntoLog(watch, signal, {
+ command: "pnpm",
+ args: [
+ "dlx",
+ ...pagesDeployArgs({
+ outDir: opts.outDir,
+ project: opts.project,
+ previewBranch: opts.previewBranch,
+ }),
+ ...(opts.extraArgs ?? []),
+ ],
+ cwd: opts.cwd,
+ env: { ...process.env, NODE_ENV: "production" },
+ });
+ if (code === 0) {
+ if (opts.previewBranch) {
+ const alias = previewAliasUrl(opts.project, opts.previewBranch);
+ onLog(
+ `[preview] ${alias}` +
+ (deploymentUrl ? ` (this deployment: ${deploymentUrl})` : "") +
+ "\n",
+ );
+ } else if (deploymentUrl) {
+ onLog(`[deployed] ${deploymentUrl}\n`);
+ }
+ }
+ return code;
+}
+
+/**
+ * Deploy the hub built in export/out to homepage.json's Pages project (a
+ * PREVIEW with `previewBranch`). THROWS every refusal and failure; returns
+ * quietly on a cancel. No R2 step: the hub holds no archives.
+ */
+export async function deployHub(
+ opts: PublishOpts & { previewBranch?: string } = {},
+): Promise<void> {
+ const { paths, onLog, signal } = resolved(opts);
+ if (opts.previewBranch !== undefined) {
+ const problem = previewBranchProblem(opts.previewBranch);
+ if (problem) throw new Error(problem);
+ }
+ const branch = opts.previewBranch?.trim() || undefined;
+ const project = getHomepageConfig(paths).cloudflareProject;
+ const projectProblem = hubProjectProblem(project);
+ if (projectProblem) throw new Error(projectProblem);
+ const outDir = resolveOutDir("", paths);
+ const builtProblem = builtHubProblem(outDir);
+ if (builtProblem) throw new Error(builtProblem);
+ if (branch) onLog(`=== Deploy hub (preview "${branch}") ===\n`);
+ const code = await runPagesDeployIntoLog(onLog, signal, {
+ outDir,
+ project: project!.trim(),
+ cwd: paths.exportDir,
+ previewBranch: branch,
+ });
+ if (signal.aborted) return;
+ if (code !== 0) throw new Error(`Hub deploy failed (exit ${code}).`);
+}
+
+function homepageDir(paths: Paths): string {
+ return path.join(paths.monorepoRoot, "homepage");
+}
+
+function homepageEnv(paths: Paths): NodeJS.ProcessEnv {
+ return {
+ ...process.env,
+ NODE_ENV: "production",
+ TRANSCRIPTS_DIR: paths.transcriptsDir,
+ };
+}
+
+/**
+ * Compose homepage/public (whole-pool stats, channel → sites map, landing
+ * summary), as a child. Reads the LMDB index as it stands: run `archilyzer
+ * index` first when it is stale. Returns the exit code.
+ */
+export async function composeHomepage(opts: PublishOpts = {}): Promise<number> {
+ const { paths, onLog, signal } = resolved(opts);
+ return runSteps(onLog, signal, [
+ { command: "pnpm", args: ["run", "compose"], cwd: homepageDir(paths), env: homepageEnv(paths) },
+ ]);
+}
+
+/** composeHomepage, then `next build` in homepage/ (→ homepage/out). */
+export async function buildHomepage(opts: PublishOpts = {}): Promise<number> {
+ const { paths, onLog, signal } = resolved(opts);
+ const code = await composeHomepage({ paths, onLog, signal });
+ if (code !== 0) return code;
+ return runSteps(onLog, signal, [
+ {
+ command: "pnpm",
+ args: ["exec", "next", "build"],
+ cwd: homepageDir(paths),
+ env: homepageEnv(paths),
+ },
+ ]);
+}
+
+/**
+ * Deploy homepage/out to the homepage's constant project, to its production
+ * branch `main` — exactly what homepage/package.json's hardcoded `deploy` line
+ * ran. THROWS on failure or when there is no build.
+ */
+export async function deployHomepage(opts: PublishOpts = {}): Promise<void> {
+ const { paths, onLog, signal } = resolved(opts);
+ const outDir = path.join(homepageDir(paths), "out");
+ if (!existsSync(path.join(outDir, "index.html"))) {
+ throw new Error("homepage/out holds no build — run archilyzer build homepage first");
+ }
+ const code = await runPagesDeployIntoLog(onLog, signal, {
+ outDir,
+ project: HOMEPAGE_PAGES_PROJECT,
+ cwd: homepageDir(paths),
+ extraArgs: ["--branch", "main"],
+ });
+ if (signal.aborted) return;
+ if (code !== 0) throw new Error(`Homepage deploy failed (exit ${code}).`);
+}
diff --git a/docker/build-site.sh b/docker/build-site.sh
@@ -34,10 +34,11 @@ rm -rf export/.next
ln -s /site/.next export/.next
echo "[build-site] building site '${SITE_ID}'"
-# build:nodata = compose:site + next build, WITHOUT the prebuild data phase
-# (already done on the host). SITE_ID selects the site in compose-site.ts.
-# compose writes straight into the mounted /site/public (EXPORT_PUBLIC_DIR).
-pnpm --filter export run build:nodata
+# `build site --nodata` = compose:site + next build, WITHOUT the data phase
+# (already done on the host). compose writes straight into the mounted
+# /site/public (EXPORT_PUBLIC_DIR). BUILD_ARCHIVES=0 (from `-e`) still reaches
+# compose through the environment.
+pnpm --filter yt-dlp-transcript-common exec tsx bin/archilyzer.ts build site "${SITE_ID}" --nodata
# next build writes export/out as a FRESH real dir (it removes+recreates out, so
# a symlink there wouldn't survive) — publish it into the per-site mount. Reached
diff --git a/docker/publish-site.sh b/docker/publish-site.sh
@@ -6,9 +6,8 @@
#
# Why this exists rather than a baked build: the export site is a static render
# OF A CORPUS, and there is no corpus at image-build time — so the image ships
-# the code and this publishes the output. It is the same `pnpm --filter export
-# run build` a host install runs; the only container-specific part is the copy
-# at the end.
+# the code and this publishes the output. It is the same `archilyzer build site`
+# a host install runs; the only container-specific part is the copy at the end.
#
# The copy is a copy and not a symlink on purpose. `next build` REMOVES and
# recreates export/out (docker/build-site.sh has the same note), so a symlink
@@ -33,9 +32,9 @@ export SITE_ID
cd /repo
echo "[publish-site] building '${SITE_ID}'"
-# The full build: prebuild (shared LMDB index + stats + chart templates), then
-# compose:site for THIS site, then next build.
-pnpm --filter export run build
+# The full build: the data phase (shared LMDB index + stats + chart
+# templates), then compose:site for THIS site, then next build.
+pnpm --filter yt-dlp-transcript-common exec tsx bin/archilyzer.ts build site "${SITE_ID}"
[ -d /repo/export/out ] || { echo "[publish-site] no export/out after build" >&2; exit 1; }
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -8,6 +8,10 @@
- **umtool's report videos can fetch Rumble clips again.** The clip fetch and the source availability check in `umtool/report-to-video` ran yt-dlp without the browser fingerprint Rumble now requires, so every Rumble clip failed with 403 and every Rumble source looked missing. They now pass the same Rumble arguments as the editor, from the same single table.
- **A transcript pulled back from a remote worker is written safely.** It used to be written straight onto `transcript.json`, so a crash part-way through left a truncated transcript; it now goes through the editor's one atomic write (temp file, then rename), like every other file the editor writes.
- **Nothing changes when you build or deploy a site; the code that does it has moved into the shared core.** The site build, the docker per-site fan-out, the R2 archive upload and the Cloudflare Pages deploy used to live inside the editor. They are now `common/publish/build.ts`, with the same log lines, exit codes and output paths, so a later command-line tool can build and deploy without the editor. The editor's Build, Deploy and Build & deploy controls and `pnpm ops build-site` / `build-deploy` / `deploy-site` call them as before. The AWS SDK packages used for the R2 upload moved with the code, from the editor's dependencies to the core's.
+- **`archilyzer`, one command line for building and publishing.** `pnpm --filter yt-dlp-transcript-common exec tsx bin/archilyzer.ts <command>` (export's scripts call it as `tsx ../common/bin/archilyzer.ts`): `index`, `compose site <id>|hub|homepage`, `build site <id> [--nodata] [--skip-archives]`, `build all [--skip-archives]`, `build hub`, `build homepage`, `deploy site <id> [--preview <branch>]`, `deploy hub [--preview <branch>]`, `deploy homepage`, `sync tick` and `settings example [--check]`. `--help` lists them. An unknown flag is refused before anything runs, so a misspelt `--preview` can never turn into a production deploy. The site commands are the same code the editor's Build and Deploy buttons run. Root `pnpm build:index` and `pnpm sync:tick` now go through it. The old `tsx bin/<name>.ts` scripts still work as before.
+- **The hub can be built and deployed.** On `/sites`, under Hub, there is now **Build hub** (tick **Deploy after build** to ship it in the same job) and **Deploy hub**. Both are also `pnpm ops build-hub` / `deploy-hub` and `archilyzer build hub` / `deploy hub`. The hub deploys to the Cloudflare Pages project set in the Hub form, and it refuses when none is set. It also refuses `archilyzer`, because that project is Archilyzer's own homepage. It builds into the same `export/out` as a site, so a deploy checks what is actually there first: deploying the hub refuses a site's build, and deploying a site refuses the hub's. The homepage deploys with `archilyzer deploy homepage` (project `archilyzer`, as before), and its front page shows a **Search all archives** link to the hub once the Hub form has a public URL. The example hub URL throughout is now `https://archilyzer-hub.pages.dev`.
+- **A site build runs its steps itself, and export's `build:nodata` and `prebuild` scripts are gone.** `build:nodata` had the same body as `build`. Only npm's `prebuild` hook told them apart: it ran the data phase for `build` and not for `build:nodata`. The data phase is now an explicit first step (skipped by **Skip data rebuild**, or `--nodata`), followed by compose and `next build`, with the same log notices. `pnpm run build` in `export/` is `archilyzer build site` (the site comes from `SITE_ID`, and `pnpm run build -- --nodata` works). `pnpm run deploy` now deploys to the site's own Pages project, where it used to pass no project at all. The docker build scripts call the same command.
+- **A posts-only channel's transcripts manifest is published again.** A social channel has no videos, so its transcripts folder in the build holds only a manifest saying "0 transcripts". The compose step read that folder as empty and deleted it, while the site's `corpus.json` still listed the manifest, so readers got a 404 (Jeralyzer's `thequartering-X`). That manifest is now copied like any other. The file format is unchanged.
- **Rumble works again, and a Rumble full sweep that gets rate-limited no longer fails the sync.** Every Rumble request had started coming back 403 from Cloudflare unless yt-dlp presents a browser fingerprint (yt-dlp #17496), so Rumble downloads failed and a Rumble channel could not even be added. Every yt-dlp run for a Rumble channel — sync, download, metadata scan, availability check, the clip-window fetch and the new-channel probe — now passes `--impersonate chrome --sleep-requests 1`, from one table in the code; a channel's own extra yt-dlp arguments still come last and still win. Separately, a full sweep that hits HTTP 429 part-way through the listing used to fail the whole sync and try again on the next one, so a large channel (The Quartering on Rumble, 44 days) never synced at all. What it read is now treated as *incomplete* — not a listing, so nothing is flagged missing and the stored playlist is untouched: the job records the platform's rate-limit cooldown, says "sweep incomplete: 429 at page N of the listing, M entries" in its log, does the ordinary newest-first sync instead, and succeeds. Syncs for that platform are then refused until its cooldown ends, and the full sweep is tried again after that. Any other yt-dlp failure still fails the sync as before.
- **A site can turn off its visitors' per-video transcript downloads.** The transcript viewer on a published site has always offered three ways to take a video's text away: a **Download** menu (txt, srt, json), **Copy MD**, and **Copy download command** (a `yt-dlp` line for a marked clip). A site's settings form now has a checkbox for them, *Per-video transcript downloads*, beside the archive zips one. Unticked, the site's next build shows none of the three; **Share** and the clip marks stay. It is on by default, so a site nobody touches is unchanged, and the file stores `"transcriptDownloads": false` only when it is off (`SITE.md` has the key). The site's machine contract (`/corpus.json`, `llms.txt`, the manifests and shards the MCP server and report-to-video read) is published either way. The hub follows the same switch: the hub form on **Sites** has the same checkbox, stored as `"transcriptDownloads": false` in the hub's `homepage.json`, and it hides the three controls on the hub's Browse and Ask pages. The editor's own video pages are unaffected.
- **Channel rows no longer scroll over a group's controls on `/channels`.** Scrolled down and to the right, the pinned Slug column of every row painted over the pinned group header and its five station buttons (Sync, Download, Transcribe, Digest and the speaker lane), and took the clicks. The pinned Slug cell and the group header sat at the same stacking level, and the later rows won. The rack now has one named layer order, kept in one file: the Advanced panel, then the column header, then the group header, then the pinned checkbox and Slug cells. Nothing ties any more. The screenshot audit found four more problems, fixed as well. A group header's name and buttons now stay on screen however far the columns scroll across (they used to scroll off to the left). An Advanced panel opened near the bottom or the right edge scrolls itself into view instead of being cut off. The rule above a pinned group header moves with it instead of leaving a gap the rows showed through. On a phone, the column header no longer paints over the selection bar pinned to the bottom of the screen.
diff --git a/editor/app/api/ops/build-hub/route.ts b/editor/app/api/ops/build-hub/route.ts
@@ -0,0 +1,30 @@
+import { NextResponse } from "next/server";
+import { buildAndDeployHubAction, buildHubAction } from "../../../sites/lib/hubActions";
+import { jobResponse, ops, optBool, optPreviewBranch, OpsInputError } from "../_lib";
+
+export const dynamic = "force-dynamic";
+
+// POST { deploy?: boolean, preview? } -> { ok: true, jobId }
+//
+// Builds the HUB (the export app under INSTANCE_MODE=hub) into export/out — the
+// same directory a site build uses. `deploy: true` makes it the one-job
+// build-then-deploy (`/sites` → Hub, "Deploy after build"), which refuses a
+// missing or wrong Pages project BEFORE building; `preview` needs `deploy`.
+export async function POST(request: Request) {
+ return ops(request, ["deploy", "preview"], async (body) => {
+ const deploy = optBool(body, "deploy") === true;
+ const preview = optPreviewBranch(body);
+ if (preview !== undefined && !deploy) {
+ throw new OpsInputError('"preview" needs "deploy": true — a build alone deploys nothing');
+ }
+ return jobResponse(
+ deploy
+ ? await buildAndDeployHubAction(preview ? { previewBranch: preview } : undefined)
+ : await buildHubAction(),
+ );
+ });
+}
+
+export function GET() {
+ return NextResponse.json({ ok: false, error: "POST only" }, { status: 405 });
+}
diff --git a/editor/app/api/ops/deploy-hub/route.ts b/editor/app/api/ops/deploy-hub/route.ts
@@ -0,0 +1,36 @@
+import { NextResponse } from "next/server";
+import { getHomepageConfig } from "yt-dlp-transcript-common/lib/homepage";
+import { previewAliasUrl } from "yt-dlp-transcript-common/lib/pagesDeploy";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { deployHubAction } from "../../../sites/lib/hubActions";
+import { jobResponse, ops, optPreviewBranch } from "../_lib";
+
+export const dynamic = "force-dynamic";
+
+// POST { preview? } -> { ok: true, jobId, previewUrl? }
+//
+// DEPLOY ONLY — the hub already built in export/out, to homepage.json's
+// Cloudflare Pages project. Refused before any job when there is no project,
+// when the project is the homepage's own ("archilyzer"), or when export/out
+// holds a site rather than the hub. `preview` is a branch name, as on
+// deploy-site; the alias is knowable before the job runs.
+export async function POST(request: Request) {
+ return ops(request, ["preview"], async (body) => {
+ const preview = optPreviewBranch(body);
+ const res = jobResponse(
+ await deployHubAction(preview ? { previewBranch: preview } : undefined),
+ );
+ if (!preview || res.status !== 200) return res;
+ // Only reached once the job started, which the action does only with a
+ // usable project — so this read cannot be what fails.
+ const project = getHomepageConfig(getPaths()).cloudflareProject;
+ const payload = (await res.json()) as Record<string, unknown>;
+ return NextResponse.json(
+ project ? { ...payload, previewUrl: previewAliasUrl(project, preview) } : payload,
+ );
+ });
+}
+
+export function GET() {
+ return NextResponse.json({ ok: false, error: "POST only" }, { status: 405 });
+}
diff --git a/editor/app/settings/components/SettingsForm.tsx b/editor/app/settings/components/SettingsForm.tsx
@@ -45,7 +45,7 @@ export function SettingsForm({ initial }: Props) {
name="homepageUrl"
defaultValue={initial.homepageUrl}
type="url"
- hint="Absolute URL of the family hub/homepage (e.g. https://archilyzer.pages.dev). Every export site links back to it. Leave blank for no hub link."
+ hint="Absolute URL of the family hub (e.g. https://archilyzer-hub.pages.dev). Every export site links back to it. Leave blank for no hub link."
/>
<Field
label="Max transcript page bytes"
diff --git a/editor/app/sites/components/HomepageConfigForm.tsx b/editor/app/sites/components/HomepageConfigForm.tsx
@@ -49,7 +49,7 @@ export function HomepageConfigForm({ config }: { config: HomepageConfig }) {
<input
className={input}
name="siteUrl"
- placeholder="https://archilyzer.pages.dev"
+ placeholder="https://archilyzer-hub.pages.dev"
defaultValue={config.siteUrl ?? ""}
/>
</label>
@@ -58,6 +58,7 @@ export function HomepageConfigForm({ config }: { config: HomepageConfig }) {
<input
className={input}
name="cloudflareProject"
+ placeholder="archilyzer-hub"
defaultValue={config.cloudflareProject ?? ""}
/>
</label>
diff --git a/editor/app/sites/components/HubBuildButtons.tsx b/editor/app/sites/components/HubBuildButtons.tsx
@@ -0,0 +1,84 @@
+"use client";
+
+import { useState } from "react";
+import {
+ buildAndDeployHubAction,
+ buildHubAction,
+ deployHubAction,
+} from "../lib/hubActions";
+import { JobLane } from "./JobLane";
+
+type Lane = { kind: "build" | "build-deploy" | "deploy"; key: number };
+
+const TITLE: Record<Lane["kind"], string> = {
+ build: "Build hub",
+ "build-deploy": "Build & deploy hub",
+ deploy: "Deploy hub",
+};
+
+// The hub's build and deploy — the export app built with INSTANCE_MODE=hub into
+// export/out, then deployed to the Cloudflare Pages project set in the form
+// above. One lane at a time, like BuildAllSitesButton: each launch replaces the
+// previous lane (its job keeps running and stays on /jobs).
+//
+// "Deploy after build" starts unchecked here, unlike the all-sites batch: a
+// hub deploy replaces a public site, and the first one should be a decision.
+export function HubBuildButtons({ project }: { project: string | null }) {
+ const [deploy, setDeploy] = useState(false);
+ const [lane, setLane] = useState<Lane | null>(null);
+ const [run, setRun] = useState(0);
+
+ function launch(kind: Lane["kind"]) {
+ const next = run + 1;
+ setRun(next);
+ setLane({ kind, key: next });
+ }
+
+ return (
+ <div role="group" aria-label="Hub build" className="flex flex-col gap-3">
+ <div className="flex flex-wrap items-center gap-3">
+ <button
+ type="button"
+ onClick={() => launch(deploy ? "build-deploy" : "build")}
+ className="px-3 py-2 rounded-md bg-primary text-primary-foreground text-sm font-medium hover:opacity-90"
+ >
+ Build hub
+ </button>
+ <label className="flex items-center gap-2 text-sm text-muted-foreground">
+ <input
+ type="checkbox"
+ checked={deploy}
+ onChange={(e) => setDeploy(e.target.checked)}
+ />
+ Deploy after build
+ </label>
+ <button
+ type="button"
+ onClick={() => launch("deploy")}
+ className="px-3 py-2 rounded-md border border-border text-sm font-medium hover:bg-muted"
+ >
+ Deploy hub
+ </button>
+ </div>
+ <p className="text-xs text-muted-foreground">
+ {project
+ ? `Builds into export/out (shared with site builds) and deploys to the Pages project "${project}". Deploy hub ships what is already built.`
+ : "Builds into export/out (shared with site builds). Set a Cloudflare Pages project above to deploy."}
+ </p>
+ {lane && (
+ <JobLane
+ key={lane.key}
+ title={TITLE[lane.kind]}
+ subtitle="INSTANCE_MODE=hub · export/out"
+ trigger={() =>
+ lane.kind === "build"
+ ? buildHubAction()
+ : lane.kind === "build-deploy"
+ ? buildAndDeployHubAction()
+ : deployHubAction()
+ }
+ />
+ )}
+ </div>
+ );
+}
diff --git a/editor/app/sites/components/SiteForm.tsx b/editor/app/sites/components/SiteForm.tsx
@@ -248,7 +248,7 @@ export function SiteForm({ initial, channels, allSites, isNew }: Props) {
label="Hub URL"
name="hubUrl"
defaultValue={initial.hubUrl ?? ""}
- hint="The hub this site belongs under (e.g. https://archilyzer.pages.dev). Shows a 'Hub' backlink and lets the hub recognize this site as a member. Leave blank to inherit the family default from Settings."
+ hint="The hub this site belongs under (e.g. https://archilyzer-hub.pages.dev). Shows a 'Hub' backlink and lets the hub recognize this site as a member. Leave blank to inherit the family default from Settings."
/>
<label className="flex items-center gap-2 text-sm">
<input
diff --git a/editor/app/sites/lib/buildAction.ts b/editor/app/sites/lib/buildAction.ts
@@ -23,6 +23,8 @@ import {
} from "yt-dlp-transcript-common/jobs/streamCommand";
import {
PREVIEW_SHARES_ARCHIVES_NOTICE,
+ buildAll,
+ buildSite,
dockerAvailable,
dockerSiteOutDir,
resolveOutDir,
@@ -91,7 +93,10 @@ export async function buildExportAction(
queueKey: queueKey === undefined ? DEFAULT_BUILD_QUEUE : queueKey.trim(),
paths,
fn: async (onLog, signal) => {
- const code = await runBuildPhase(onLog, signal, id, paths, {
+ const code = await buildSite(id, {
+ paths,
+ onLog,
+ signal,
skipData,
skipArchives,
});
@@ -139,7 +144,10 @@ export async function buildAndDeployAction(
paths,
fn: async (onLog, signal) => {
onLog("=== Build ===\n");
- const buildCode = await runBuildPhase(onLog, signal, id, paths, {
+ const buildCode = await buildSite(id, {
+ paths,
+ onLog,
+ signal,
skipArchives,
});
// A cancel mid-build must NOT proceed to deploy.
@@ -338,27 +346,19 @@ export async function buildAllSitesAction(
fn: async (onLog, signal) => {
const useDocker = await dockerAvailable(signal);
if (signal.aborted) return;
- let outcomes: SiteBuildOutcome[];
- if (useDocker) {
- outcomes = await runDockerBuildAllPhase(onLog, signal, sites, paths, {
- skipArchives,
- });
- } else {
+ if (!useDocker) {
onLog(
"[notice] No container engine available — building sites serially on the host.\n",
);
- outcomes = [];
- for (let i = 0; i < sites.length; i++) {
- if (signal.aborted) return;
- const site = sites[i];
- onLog(`\n=== Build ${site.siteId} (${i + 1}/${sites.length}) ===`);
- const code = await runBuildPhase(onLog, signal, site.siteId, paths, {
- skipData: i > 0,
- skipArchives,
- });
- outcomes.push({ siteId: site.siteId, code });
- }
}
+ const outcomes = await buildAll({
+ paths,
+ onLog,
+ signal,
+ mode: useDocker ? "docker" : "basic",
+ skipArchives,
+ sites,
+ });
if (signal.aborted) return;
const ok = outcomes.filter((o) => o.code === 0).length;
const failed = outcomes
diff --git a/editor/app/sites/lib/deployAction.ts b/editor/app/sites/lib/deployAction.ts
@@ -9,10 +9,8 @@ import {
type StreamActionResult,
} from "yt-dlp-transcript-common/jobs/streamCommand";
import {
- PREVIEW_SHARES_ARCHIVES_NOTICE,
+ deploySite,
resolveOutDir,
- runArchiveUploadIntoLog,
- runDeployIntoLog,
} from "yt-dlp-transcript-common/publish/build";
const DEPLOY_QUEUE = "deploy";
@@ -64,30 +62,9 @@ export async function deployExportAction(
kind: "deploy-export",
queueKey: DEPLOY_QUEUE,
paths,
- fn: async (onLog, signal) => {
- // The production path logs no banner and gains none here: its log has
- // always opened on wrangler's own first line.
- if (branch) {
- onLog(`=== Deploy (preview "${branch}") ===\n`);
- onLog(PREVIEW_SHARES_ARCHIVES_NOTICE);
- }
- // Push oversize archives to R2 first, so the manifest URLs the Pages
- // deploy publishes resolve immediately. No-op when R2 isn't configured.
- const uploadCode = await runArchiveUploadIntoLog(onLog, signal, site, paths);
- if (signal.aborted) return;
- if (uploadCode !== 0) {
- throw new Error(`Archive R2 upload failed (exit ${uploadCode}).`);
- }
- const code = await runDeployIntoLog(
- onLog,
- signal,
- site,
- outDir,
- paths,
- { previewBranch: branch },
- );
- if (signal.aborted) return;
- if (code !== 0) throw new Error(`Deploy failed (exit ${code}).`);
- },
+ // The refusals above answer before a job exists; deploySite checks them
+ // again when the job actually starts, and does the upload + deploy.
+ fn: (onLog, signal) =>
+ deploySite(site.siteId, { paths, onLog, signal, previewBranch: branch }),
});
}
diff --git a/editor/app/sites/lib/hubActions.ts b/editor/app/sites/lib/hubActions.ts
@@ -0,0 +1,101 @@
+"use server";
+
+import { builtHubProblem } from "yt-dlp-transcript-common/lib/builtExport";
+import { getHomepageConfig } from "yt-dlp-transcript-common/lib/homepage";
+import { previewBranchProblem } from "yt-dlp-transcript-common/lib/pagesDeploy";
+import { getPaths, type Paths } from "yt-dlp-transcript-common/lib/paths";
+import {
+ runManagedFunction,
+ type StreamActionResult,
+} from "yt-dlp-transcript-common/jobs/streamCommand";
+import {
+ buildHub,
+ deployHub,
+ hubProjectProblem,
+ resolveOutDir,
+} from "yt-dlp-transcript-common/publish/build";
+
+// The hub's build and deploy, as jobs. The hub is the export app built with
+// INSTANCE_MODE=hub into the SAME export/out a site build uses, so its build
+// takes the build queue (it would race a site build for export/) and its
+// deploys take the deploy queue — the queues build-export and deploy-export
+// use (buildAction.ts / deployAction.ts).
+const BUILD_QUEUE = "build";
+const DEPLOY_QUEUE = "deploy";
+
+// Refusals a deploy can give BEFORE any job starts: a bad preview name, no
+// project (or the homepage's), and — for a deploy-only — no hub in export/out.
+// deployHub checks them again when its job runs, because a queued deploy can
+// start after a site build has replaced export/out.
+function deployRefusal(
+ paths: Paths,
+ previewBranch: string | undefined,
+ checkBuilt: boolean,
+): string | null {
+ if (previewBranch !== undefined) {
+ const problem = previewBranchProblem(previewBranch);
+ if (problem) return problem;
+ }
+ const projectProblem = hubProjectProblem(getHomepageConfig(paths).cloudflareProject);
+ if (projectProblem) return projectProblem;
+ if (checkBuilt) return builtHubProblem(resolveOutDir("", paths));
+ return null;
+}
+
+export async function buildHubAction(): Promise<StreamActionResult> {
+ const paths = getPaths();
+ return runManagedFunction({
+ kind: "build-hub",
+ queueKey: BUILD_QUEUE,
+ paths,
+ fn: async (onLog, signal) => {
+ const code = await buildHub({ paths, onLog, signal });
+ if (signal.aborted) return;
+ if (code !== 0) throw new Error(`Hub build failed (exit ${code})`);
+ },
+ });
+}
+
+// Deploy the hub already built in export/out. With `opts.previewBranch` it is
+// a Cloudflare Pages PREVIEW of the hub's project.
+export async function deployHubAction(opts?: {
+ previewBranch?: string;
+}): Promise<StreamActionResult> {
+ const paths = getPaths();
+ const refusal = deployRefusal(paths, opts?.previewBranch, true);
+ if (refusal) return { ok: false, error: refusal };
+ const branch = opts?.previewBranch?.trim();
+ return runManagedFunction({
+ kind: "deploy-hub",
+ queueKey: DEPLOY_QUEUE,
+ paths,
+ fn: (onLog, signal) => deployHub({ paths, onLog, signal, previewBranch: branch }),
+ });
+}
+
+// Build the hub and, only if that succeeds, deploy it — one job, one log, one
+// Cancel, on the deploy queue (as build-deploy is for a site).
+export async function buildAndDeployHubAction(opts?: {
+ previewBranch?: string;
+}): Promise<StreamActionResult> {
+ const paths = getPaths();
+ // Before the build: learning the project is wrong after a build wastes it.
+ const refusal = deployRefusal(paths, opts?.previewBranch, false);
+ if (refusal) return { ok: false, error: refusal };
+ const branch = opts?.previewBranch?.trim();
+ return runManagedFunction({
+ kind: "build-deploy-hub",
+ queueKey: DEPLOY_QUEUE,
+ paths,
+ fn: async (onLog, signal) => {
+ onLog("=== Build hub ===\n");
+ const code = await buildHub({ paths, onLog, signal });
+ if (signal.aborted) return;
+ if (code !== 0) {
+ throw new Error(`Hub build failed (exit ${code}) — not deploying.`);
+ }
+ onLog("\n=== Deploy hub ===\n");
+ await deployHub({ paths, onLog, signal, previewBranch: branch });
+ },
+ });
+}
diff --git a/editor/app/sites/page.tsx b/editor/app/sites/page.tsx
@@ -21,6 +21,7 @@ import { BuildModeToggle } from "./components/BuildModeToggle";
import { BuildSitesPanel } from "./components/BuildSitesPanel";
import { CutReleaseForm } from "./components/CutReleaseForm";
import { HomepageConfigForm } from "./components/HomepageConfigForm";
+import { HubBuildButtons } from "./components/HubBuildButtons";
import { DeleteSiteButton, MigrateButton } from "./components/SiteListActions";
export const dynamic = "force-dynamic";
@@ -29,8 +30,9 @@ export const metadata: Metadata = { title: "Sites" };
// Which live jobs the Pool section lists: the eight kinds the Pool's own
// buttons enqueue (BuildButtons.tsx → sites/lib/buildAction.ts), and only
// those. build-export and build-deploy are a site's Publish tab's, build-all
-// and build-deploy-all are the batch panel's above — each has its own console
-// and is not repeated here.
+// and build-deploy-all are the batch panel's above, build-hub / deploy-hub /
+// build-deploy-hub the Hub section's — each has its own console and is not
+// repeated here.
const BUILD_KINDS = new Set([
"build-index",
"build-stats",
@@ -71,6 +73,7 @@ export default async function SitesPage() {
// Through the one builder, so this list has the same progress bars /jobs does
// (it used to drop `progress`, `tasks`, `drainable` and the reorder bounds).
const activeJobs = await liveJobRows((j) => BUILD_KINDS.has(j.kind));
+ const hubConfig = getHomepageConfig(paths);
return (
<div className="flex flex-col gap-8">
@@ -183,15 +186,19 @@ export default async function SitesPage() {
<div>
<h2 className="text-lg font-semibold">Hub</h2>
<p className="mt-1 text-sm text-muted-foreground">
- The <code>homepage</code> package builds Archilyzer’s own site —
- the marketing home, the documentation and the source download. Its{" "}
- <code>/stats/</code> route is a cross-site dashboard built from the
- whole channel pool. Only the operator-facing bits are edited here: the
- wordmark and page title are the product’s own, but these social
- links and the deploy target are yours.
+ The hub is the export app built in hub mode: one search over every
+ site that has a public URL, reading each archive where it is
+ published. This config names and brands it, and its Public URL and
+ Cloudflare Pages project are the hub’s own (for example{" "}
+ <code>archilyzer-hub</code>). The <code>homepage</code> package —
+ Archilyzer’s own site, with the docs and the source download
+ — shares these social links and links to the hub, but deploys to
+ its own project, <code>archilyzer</code>, with{" "}
+ <code>archilyzer deploy homepage</code>.
</p>
</div>
- <HomepageConfigForm config={getHomepageConfig(paths)} />
+ <HomepageConfigForm config={hubConfig} />
+ <HubBuildButtons project={hubConfig.cloudflareProject ?? null} />
</section>
{/* The shared pool: corpus-wide, no site involved. */}
diff --git a/editor/e2e/ops-api.spec.ts b/editor/e2e/ops-api.spec.ts
@@ -26,7 +26,7 @@
// it is a `finally`, never a trailing line: one failed assertion with the token
// still unset leaves every later /api/ops and /api/worker spec answering 503.
-import { readdir, rm } from "node:fs/promises";
+import { mkdir, readdir, rm, writeFile } from "node:fs/promises";
import { test, expect, type APIRequestContext } from "@playwright/test";
import { baseUrl } from "./baseUrl";
import {
@@ -957,3 +957,47 @@ test("build-site with a bare siteId starts one build-export job", async ({
})
.toBe("build-export");
});
+
+// The hub's deploy path (release 7). Every refusal here is answered BEFORE a
+// job exists, which is what lets a runbook's `pnpm ops deploy-hub --wait` fail
+// fast instead of queueing a deploy that can only fail.
+test("deploy-hub refuses no project, the homepage's project, and a bundle that is not the hub", async ({
+ request,
+}) => {
+ await resetData("title-filter-channel");
+ await settings();
+ const before = await listJobIds();
+
+ // The fixture has no homepage.json at all: no project.
+ const none = await ops(request, "deploy-hub", {});
+ expect(none.status).toBe(400);
+ expect(none.body.error).toBe(
+ "The hub has no Cloudflare Pages project configured — set it on /sites under Hub.",
+ );
+
+ // The homepage's project is refused by name — homepage.json said
+ // "archilyzer" before the hub could deploy, and a hub deployed there would
+ // replace the software's own site.
+ const hubFile = resolvePath("test-transcripts/sites/_homepage/homepage.json");
+ await mkdir(resolvePath("test-transcripts/sites/_homepage"), { recursive: true });
+ await writeFile(hubFile, JSON.stringify({ cloudflareProject: "archilyzer" }));
+ const homepages = await ops(request, "deploy-hub", {});
+ expect(homepages.status).toBe(400);
+ expect(homepages.body.error).toContain('"archilyzer", which is the Archilyzer homepage\'s');
+
+ // A real project, but export/out holds no hub: never built here, or a
+ // site's bundle (shared directory).
+ await writeFile(hubFile, JSON.stringify({ cloudflareProject: "archilyzer-hub" }));
+ const unbuilt = await ops(request, "deploy-hub", { preview: "hub-check" });
+ expect(unbuilt.status).toBe(400);
+ expect(unbuilt.body.error).toMatch(
+ /export\/out holds (no hub build|a build of ".*", not the hub) — build the hub first/,
+ );
+
+ // build-hub: a preview without a deploy is a mistake, not a build.
+ const previewOnly = await ops(request, "build-hub", { preview: "hub-check" });
+ expect(previewOnly.status).toBe(400);
+ expect(previewOnly.body.error).toContain('"preview" needs "deploy": true');
+
+ expect(await listJobIds()).toEqual(before);
+});
diff --git a/export/package.json b/export/package.json
@@ -13,17 +13,15 @@
"detect:duplicates": "NODE_OPTIONS=--max-old-space-size=8192 tsx ../common/bin/duplicate-shorts.ts",
"compose:site": "tsx ../common/bin/compose-site.ts",
"compose:hub": "tsx ../common/bin/compose-hub.ts",
- "prebuild": "pnpm run build:data",
- "build": "pnpm run compose:site && next build",
- "build:nodata": "pnpm run compose:site && next build",
- "build:hub": "pnpm run compose:hub && INSTANCE_MODE=hub next build",
+ "build": "tsx ../common/bin/archilyzer.ts build site",
+ "build:hub": "tsx ../common/bin/archilyzer.ts build hub",
"start": "serve out",
"lint": "eslint",
"e2e": "node ../scripts/queue-lock.mjs --ports EXPORT_E2E_PORT:3020 -- playwright test",
"e2e:hub": "node ../scripts/queue-lock.mjs --ports HUB_PORT:3041 -- playwright test --config playwright.hub.config.ts",
"e2e:2origin": "node ../scripts/queue-lock.mjs --ports ORIGIN_B_PORT:4610,HUB_A_PORT:4611 -- playwright test --config playwright.2origin.config.ts",
"e2e:ui": "playwright test --ui",
- "deploy": "pnpm dlx wrangler pages deploy out"
+ "deploy": "tsx ../common/bin/archilyzer.ts deploy site"
},
"dependencies": {
"@tanstack/react-query": "^5.99.1",
diff --git a/homepage/app/page.tsx b/homepage/app/page.tsx
@@ -3,6 +3,7 @@ import { RecentAdditions } from "./components/RecentAdditions";
import { KpiHeader } from "./components/KpiHeader";
import { ArchiveCards } from "./components/ArchiveCards";
import { loadSummary } from "./lib/summary";
+import { currentHomepage } from "./lib/homepage";
// The project's front page. Its ONE job: make a visitor understand what
// Archilyzer is and download it.
@@ -38,6 +39,11 @@ export default function Home() {
const hours = summary?.totals.hoursArchived ?? null;
const gone = summary?.availability?.byState.deleted ?? 0;
const counted = summary?.availability?.counted ?? 0;
+ // The operator's hub — one search over every published archive — is a
+ // separate app on its own Pages project; homepage.json's siteUrl is its
+ // address (the same field the hub's own build reads). No URL, no link: a
+ // source-only build has no hub to point at.
+ const hubUrl = currentHomepage().siteUrl ?? null;
return (
<>
@@ -87,6 +93,14 @@ export default function Home() {
>
Read the setup guide →
</Link>
+ {hubUrl && (
+ <a
+ href={hubUrl}
+ className="inline-flex items-center gap-2 border border-[var(--border-strong)] px-6 py-3 font-mono text-xs uppercase tracking-[0.16em] text-[var(--foreground)] rounded-[var(--radius)] hover:border-[var(--brand)] hover:text-[var(--brand)] transition-colors"
+ >
+ Search all archives
+ </a>
+ )}
</div>
{/* The rail: real acquisitions, real states. The signature object. */}
diff --git a/homepage/e2e/marketing.spec.ts b/homepage/e2e/marketing.spec.ts
@@ -22,6 +22,18 @@ test("says what it is and offers the source", async ({ page }) => {
).toHaveAttribute("href", "/docs/install/");
});
+// The hub is a separate app on its own Pages project, linked from the hero
+// when homepage.json names its URL. Data-dependent like the rail below, so the
+// assertion is conditional the same way: absent is a legal state (a
+// source-only build has no hub), but PRESENT means an absolute link off-site —
+// never a relative path into this site, which has no search.
+test("the hub link, when there is one, leaves for the hub", async ({ page }) => {
+ const hub = page.getByRole("link", { name: /search all archives/i });
+ if ((await hub.count()) === 0) return;
+ await expect(hub).toHaveCount(1);
+ await expect(hub).toHaveAttribute("href", /^https?:\/\/[^/]/);
+});
+
test("the trust block states what it isn't", async ({ page }) => {
// The cheapest, highest-value block on the page — every claim in it is
// simply true, and each one pre-empts a wrong assumption.
diff --git a/homepage/package.json b/homepage/package.json
@@ -15,7 +15,7 @@
"lint": "eslint",
"e2e": "node ../scripts/queue-lock.mjs --ports HOMEPAGE_E2E_PORT:3040 -- playwright test",
"e2e:ui": "playwright test --ui",
- "deploy": "pnpm dlx wrangler pages deploy out --project-name archilyzer --branch main"
+ "deploy": "tsx ../common/bin/archilyzer.ts deploy homepage"
},
"dependencies": {
"@tanstack/react-query": "^5.99.1",
diff --git a/mcp/README.md b/mcp/README.md
@@ -454,7 +454,7 @@ means "this video's URL" in the output.
```
list_channels # the default corpus
list_channels source="remote:https://rekietalyzer.pages.dev"
-search_transcripts query="k cups" source="hub:https://archilyzer.pages.dev#jeralyzer,rekietalyzer"
+search_transcripts query="k cups" source="hub:https://archilyzer-hub.pages.dev#jeralyzer,rekietalyzer"
resolve_source source="jeralyzer" # → remote:https://jeralyzer.pages.dev
```
@@ -505,7 +505,7 @@ the TypeScript path alias resolves):
pnpm --filter yt-dlp-transcript-mcp exec tsx src/index.ts --remote https://rekietalyzer.pages.dev
# a hub, federating every member site
-pnpm --filter yt-dlp-transcript-mcp exec tsx src/index.ts --hub https://archilyzer.pages.dev
+pnpm --filter yt-dlp-transcript-mcp exec tsx src/index.ts --hub https://archilyzer-hub.pages.dev
# local shards on disk
pnpm --filter yt-dlp-transcript-mcp exec tsx src/index.ts --local ../export/public
diff --git a/package.json b/package.json
@@ -5,8 +5,8 @@
"license": "MIT",
"type": "module",
"scripts": {
- "build:index": "pnpm --filter yt-dlp-transcript-common exec tsx bin/build-index.ts",
- "sync:tick": "pnpm --filter yt-dlp-transcript-common exec tsx bin/sync-tick.ts",
+ "build:index": "pnpm --filter yt-dlp-transcript-common exec tsx bin/archilyzer.ts index",
+ "sync:tick": "pnpm --filter yt-dlp-transcript-common exec tsx bin/archilyzer.ts sync tick",
"build:export": "pnpm --filter export run build",
"build:homepage": "pnpm --filter homepage run build",
"build:homepage:nodata": "pnpm --filter homepage run build:nodata",
diff --git a/plans/release-7.md b/plans/release-7.md
@@ -225,7 +225,9 @@ worktrees shifts port blocks. Reviews: Opus, read-only, `SHIP | SHIP AFTER FIXES
## Rollout at the end (releases 6 + 7 together; ONE editor restart + umtool restart)
Release-5 procedure (`plans/release-5.md` "## Rollout 2026-09-24 (night)") plus:
-1. Final suites on the merge sha in a detached worktree: editor full, export full, `e2e:hub`.
+1. Final suites on the merge sha in a detached worktree: editor full, export full, `e2e:hub`,
+ and `TWO_ORIGIN_REBUILD=1 node scripts/worktree.mjs run -- pnpm --filter export run e2e:2origin`
+ (its globalSetup runs `pnpm run build:hub`, which slice C made `archilyzer build hub`).
2. Primary: `pnpm install --frozen-lockfile` (expected no-op; verify the three node_modules facts
above). MUST precede the editor build.
3. Numbers on the live corpus: settings (1,353 paths, diff empty), files (77 `unknown keys: []`,
@@ -391,4 +393,153 @@ because the fixes touch no file they cover beyond `common/views`, which tsc chec
- **Commit trailers** name `Claude Opus 5.5 (1M context)`, as the release-6 implementers did.
`implementer-rules.md` still names Fable 5.1.
+### Slice C, as shipped — the archilyzer CLI, hub deploy path, posts-only fix (2026-09-25)
+
+Branch `one-core/r7-cli` off `main` `6ee1d336`. There is now one command line, `common/bin/archilyzer.ts`,
+over the publish layer and the bins. `common/publish/build.ts` has named entry points (`buildSite`,
+`deploySite`, `buildAll`, `buildHub`, `deployHub`, `composeHub`, `composeHomepage`, `buildHomepage`,
+`deployHomepage`), and the editor's jobs call the same ones. export's `build` / `build:nodata` twins
+and the `prebuild` hook are gone. The data phase is now an explicit step, so `pnpm run build` in
+`export/` can be the CLI without running itself. The hub now has a build and deploy path: on
+`/sites`, through `pnpm ops`, and through the CLI. The homepage deploys through the CLI. The
+posts-only 404 is fixed in the composer, and the contract is unchanged. Items 1–8 of the spec are
+done. `doctor`, `run` and `mcp` were not started (the cut line).
+
+| sha | what |
+|---|---|
+| `66f138cd` | `_parseFlags.ts` gains `parseArgv(argv, booleans) → {flags, positionals}` beside `parseFlags`. A declared boolean never takes the next word as its value (`build site --nodata jeralyzer`), and a lone `--` is skipped. The skip matters because pnpm 11 hands `pnpm run build -- --nodata` to the script as `-- --nodata`, `--` included (checked in scratch). New `_cli.ts`: `Command = {path, usage, flags?, maxPositionals?, run}`, `resolveCommand` (longest path), `argumentProblem` (unknown flag, a boolean given a value, a string flag given none, extra positionals), `usage`, `runCli` (a refusal exits 2 before `run`). `Command` gained `flags` and `maxPositionals` on top of the spec's `{path, usage, run}`, so a typo such as `--preveiw` is refused before a deploy runs instead of shipping production. The `test` glob gains `bin`. `_cli.test.ts` (9) |
+| `d97b6ad8` | The ten bins (`build-index`, `build-stats`, `build-chart-templates`, `build-archives`, `compose-site`, `compose-hub`, `compose-homepage`, `sync-tick`, `settings-example`, `file-schemas-docs`) `export async function main(opts)` and auto-run through `runIfEntryPoint(import.meta.url, …)`. That is the `worktree.mjs:350` idiom with both sides realpath'd, because `import.meta.url` is always the real file and argv[1] can reach it through a workspace symlink. A returned number becomes `process.exitCode` without cutting the process short, and a throw prints and exits 1, as before. `compose-site` `main({siteId})` falls back to `SITE_ID` and throws when there is neither. It used to `process.exit(1)`. `sync-tick` `main()` returns 1 on an HTTP error, and `tick()` keeps the old `request failed:` line. `archilyzer.ts` is the table, with lazy `import()` per row. Root `build:index` → `archilyzer index`, `sync:tick` → `archilyzer sync tick`. Flag-driven bins (`_parseFlags` users) are untouched |
+| `5dabc262` | `buildSiteSteps({siteId, paths, skipData, skipArchives, baseEnv}) → [{command, args, cwd, env}]` lists the steps: `pnpm run build:data` (unless skipData), `pnpm run compose:site`, then `pnpm exec next build`. All three run in `export/` with the old env block (NODE_ENV, TRANSCRIPTS_DIR, EXPORT_PUBLIC_DIR, SITE_ID, and BUILD_ARCHIVES=0 when archives are skipped). The data phase keeps the env the old `prebuild` inherited, EXPORT_PUBLIC_DIR included, rather than `runHostScript`'s narrower one, which matters for the e2e server's `.export-public`. `runBuildPhase` runs the steps with the same `[notice]` lines. Named entry points: `buildSite`, `deploySite` (the preview, project and built-bundle refusals, then R2, then Pages; it throws the deploy job's exact sentences) and `buildAll({mode: "docker"\|"basic"})` (the serial loop lifted from `buildAllSitesAction`). `build-export`, `build-deploy` (its build half), `deploy-export` and `build-all` call them, and their pre-job refusals stay in the actions. export: `prebuild` and `build:nodata` deleted, `build` → `archilyzer build site`, `deploy` → `archilyzer deploy site` (it names the site's project and refuses without one). CLI gains `build site`, `build all` and `deploy site`. `build site` refuses an id with no `site.json`, because a missing site reads as defaults and would otherwise run the whole data phase first. `build.test.ts` +2 |
+| `e8460a16` | `docker/build-site.sh` → `pnpm --filter yt-dlp-transcript-common exec tsx bin/archilyzer.ts build site "$SITE_ID" --nodata`, `publish-site.sh` → `… build site "$SITE_ID"` |
+| `077fcba4` | Hub and homepage entry points. `buildHubSteps` runs `compose:hub`, then `next build` with `INSTANCE_MODE=hub`, in `export/`. `buildHub` removes `public/site.json` first. `compose-site` now removes `public/hub-sites.json`, the one other line in that bin, so `out/` says which one it holds. `builtHubProblem(outDir)` (`builtExport.ts`, +1 test): a `site.json` means a site's bundle, even with a stale `hub-sites.json` beside it. `hubProjectProblem` refuses a missing project with the spec's sentence, and refuses `archilyzer` (`HOMEPAGE_PAGES_PROJECT`) by name. `deployHub` reads `getHomepageConfig(paths).cloudflareProject` and runs `pagesDeployArgs` through `runChildIntoLog`, with the `deploymentUrlIn` / `[preview]` line of `runDeployIntoLog`. `buildHomepage` is `pnpm run compose` then `next build` in `homepage/`. `deployHomepage` ships `homepage/out` to `archilyzer` with `--branch main`, as the hardcoded `homepage/package.json` line did (refused when there is no `out/index.html`). CLI: `build hub`, `deploy hub [--preview]`, `build homepage`, `deploy homepage`. export `build:hub` and homepage `deploy` call it. `build.test.ts` +2 |
+| `c82aad09` | `editor/app/sites/lib/hubActions.ts`: `buildHubAction` (kind `build-hub`, queue `build`), `deployHubAction` (`deploy-hub`, `deploy`) and `buildAndDeployHubAction` (`build-deploy-hub`, `deploy`). Before any job, they refuse a bad preview, a missing project or the homepage's project, and (deploy-only) a bundle that is not the hub. `HubBuildButtons.tsx` sits under the Hub form, in a group named `Hub build`: button **Build hub**, checkbox **Deploy after build** (unchecked by default, unlike the all-sites batch, because the first hub deploy should be a choice), button **Deploy hub**, lanes **Build hub** / **Build & deploy hub** / **Deploy hub**. The Hub section's copy now says the hub and the homepage are two projects. `/api/ops/build-hub` `{deploy?, preview?}` (a preview without deploy is a 400) and `/api/ops/deploy-hub` `{preview?}` (→ `previewUrl`). `archilyzer-ops.mjs` ACTIONS + usage + test (+1). `ops-api.spec` +1: no project, `archilyzer`, and not-a-hub-build are each refused, and no job starts. Hub-URL hints → `https://archilyzer-hub.pages.dev`: `HomepageConfigForm` placeholder (plus an `archilyzer-hub` placeholder on the project field), `SiteForm` Hub URL hint, `settingsSchema` `homepageUrl` (+ `SETTINGS.md` regenerated through `archilyzer settings example`), `mcp/README.md` :457, :508 |
+| `c66b9d4a` | Homepage hero: **Search all archives** → `homepage.json` `siteUrl` (the field `hubSite()` reads), rendered only when set. `marketing.spec` +1, conditional like the rail test: absent is legal, and a present link must be absolute |
+| `baa7ef45` | Posts-only 404. When the signature is `""` but `src/manifest.json` exists, `reconcileChannelTree` (now exported) copies the tree under the constant `MANIFEST_ONLY_SIGNATURE`. `corpus.ts` is untouched and spec stays 4. New `bin/compose-site.test.ts` (3): a manifest-only tree is copied (then skipped when unchanged, re-copied once pages arrive), an unchanged tree is skipped, and a member with no manifest is removed and a non-member pruned. Two of the three fail with the fix reverted (checked) |
+| `e4b5376c` | this record, four `[Unreleased]` bullets |
+| `bc2d9fb6` | (review fix) `DEPLOY_CLOUDFLARE.md:40-45`: every deploy path uploads to R2 and only a build stages. The review's replacement text said the credentials come from `settings.json`; only the bucket does, and the credentials come from the environment, so the text says that. Settings `homepageUrl` hint and `homepage.ts` comments → `archilyzer-hub` (never `archilyzer`). `compose site` with no id → `siteIdFrom` (one line, exit 2) |
+| `d1ba90e9` | merge `main` `7b79a945` (slice Y at `2497d20b`). Only `plans/release-7.md` conflicted (both records, Y's first). `editor/CHANGELOG.md` auto-merged, and the lockfile did not move |
+| *(this commit)* | (review fix) this record: the Pages-project bullet corrected, the 2origin line, rollout step 1 gains `e2e:2origin`, the found-and-left additions, the re-gate |
+
+**The CLI as shipped** (`pnpm --filter yt-dlp-transcript-common exec tsx bin/archilyzer.ts …`, or
+`tsx ../common/bin/archilyzer.ts …` from `export/` / `homepage/`; `--help` anywhere):
+
+| command | does |
+|---|---|
+| `index` | `buildIndex` (the LMDB index) |
+| `compose site [<id>]` | compose-site `main({siteId})`, id else `SITE_ID` |
+| `compose hub` / `compose homepage` | compose-hub / compose-homepage `main()` (in-process) |
+| `build site [<id>] [--nodata] [--skip-archives]` | `buildSite`: data phase + compose + `next build` into `export/out`; refuses an unknown id |
+| `build all [--skip-archives]` | `buildAll`: docker fan-out when `docker version` answers, else serial host (as the editor's action decides) |
+| `build hub` | `buildHub` → `export/out` |
+| `build homepage` | `buildHomepage` → `homepage/out` (reads the index as it stands) |
+| `deploy site [<id>] [--preview <b>]` | `deploySite`: refusals, R2, Pages |
+| `deploy hub [--preview <b>]` | `deployHub` → `homepage.json` `cloudflareProject` (never `archilyzer`) |
+| `deploy homepage` | `deployHomepage` → project `archilyzer`, branch `main` |
+| `sync tick` | sync-tick (`SYNC_TICK_URL`, `SYNC_TICK_TOKEN`) |
+| `settings example [--check]` | settings-example `main({check})` |
+
+Exit codes: 0 ok, 1 failed or refused by the entry point, 2 usage (unknown command or flag, no site).
+
+**Gates** (worktree root, final tree before this commit). tsc clean at every commit. common
+**1771/1771**: 1754 + 9 `_cli` + 4 `build` + 1 `builtExport` + 3 `compose-site`. Editor unit
+**72/72**. test:scripts **160 pass + 1 skip** (159 + 1 ops). mcp **219/219**.
+`pnpm --filter editor exec next build` ok (compiled in 29.8 s, `/api/ops/build-hub` and
+`/api/ops/deploy-hub` listed). `pnpm --filter export exec next build` ok (12.5 s, no dangling
+`export/public` links). EDITOR e2e `sites-crud site-publish-preview build deploy-page cut-release
+channel-build-toggle ops-api` (all seven exist; `$T/c-specs.txt`): **49 passed, 0 failed, 2.9 min**,
+after 1m33s in the queue behind slice Y. EXPORT e2e in full (`node scripts/worktree.mjs run -- pnpm --filter export run e2e`, no dangling
+links): **192 passed, 0 failed, 7.4 min**.
+`e2e:hub` (`… pnpm --filter export run e2e:hub`): **8 passed, 16 s**.
+Homepage e2e (`… pnpm --filter homepage run e2e`, five specs): **15 passed, 7 skipped, 0 failed,
+33 s**. The 7 skips are `stats.spec.ts`'s `test.skip(noData, …)`: the worktree has no corpus data.
+The new hub-link test took its "absent" branch here, because the worktree has no `homepage.json`.
+Numbers: `plans/tools/phase3-files-numbers.ts` over one frozen copy (`FREEZE_TO`, 71 configs,
+1,763 sidecars; `TMPDIR=$T`), `6ee1d336` against this branch: **diff empty** (3,859 lines each).
+No build, deploy, compose or data build ran against the real corpus. The CLI's refusal paths were
+exercised in the worktree, which has no corpus: an unknown site, `--preview main`, no id, no
+project, no hub project and no homepage build.
+
+**Re-gate after the review fixes and the merge of `main` `7b79a945`** (tree `d1ba90e9` + record;
+`rm -rf editor/.next/dev`, lockfile unchanged). tsc clean. common **1789/1789** (1772 on `main`
++ 17 from this slice). Editor unit **72/72**. test:scripts **160 + 1 skip**. mcp **219/219**.
+`next build`: editor ok (14.7 s), export ok (7.4 s). Numbers (the same frozen copy) against the
+`6ee1d336` run: **diff empty**. EDITOR e2e, the seven specs + `pacing.spec`
+(`$T/c-specs-merged.txt`, `.spec` suffixes): **51 passed, 0 failed, 3.1 min**. EXPORT full:
+**192 passed, 6.0 min**. `e2e:hub`: **8 passed, 15 s**. `e2e:2origin` with
+`TWO_ORIGIN_REBUILD=1`: **3 passed, 40 s** (73 s including two `archilyzer build hub` runs).
+The log shows `$ tsx ../common/bin/archilyzer.ts build hub` and `compose-hub: 0 built-in pool
+site(s)`. The build runs twice because `globalSetup` is called at config load in the runner and
+again in the worker, and `TWO_ORIGIN_REBUILD=1` clears the cache each time. That was already
+true before this slice. The primary's `export/public` files kept their mtimes (checked).
+Homepage: **15 passed, 7 skipped, 25 s**.
+
+**Found and left.**
+- **No export e2e pins the posts-only fix.** No spec reads `corpus.json`, and the fixture site has
+ only `test-youtube`, with no social channel. The composer is pinned by `compose-site.test.ts`,
+ and the index side (a `pageCount: 0` manifest for a channel with no transcripts) by
+ `editor/e2e/build.spec.ts:10`. The live proof is `curl
+ https://jeralyzer.pages.dev/transcripts/thequartering-X/manifest.json` returning 200 after the
+ next jeralyzer build + deploy. That build needs no `--nodata` caveat, because the staging already
+ has the manifest.
+- **`Dockerfile.build` needs nothing.** It installs root + common + export `package.json` (tsx is
+ in common's devDependencies, and no `NODE_ENV=production` is set at install), then
+ `COPY . .`, so the CLI and the new `export/package.json` are baked from source. Every docker
+ fan-out calls `ensureBuildImage` (`docker build`, with cached layers reused) before Phase B, and
+ mounts the host `build-site.sh` over the baked one, so the editor's docker path never runs the
+ new script on an old image. Only an image built BEFORE this slice and run by hand, outside the
+ editor, would find `build:nodata` missing. The runtime `Dockerfile` copies all seven
+ `package.json` files and installs dev dependencies in its build stage, so `publish-site.sh`
+ has tsx too.
+- **A queued deploy now re-checks the bundle when it starts.** `deploySite` / `deployHub`
+ repeat the pre-job refusals inside the job. A deploy queued behind another site's build (which
+ is on the `build` queue, so they do not serialize) used to ship whatever that build left in
+ `export/out`. It now refuses. This is new behaviour, and deliberate.
+- **Two more hub-URL examples still name `archilyzer.pages.dev`**, outside this slice's files:
+ `editor/app/settings/components/SettingsForm.tsx:48` (the `homepageUrl` hint, which now
+ disagrees with `SETTINGS.md`) and the comment at `common/lib/homepage.ts:31`. Both are one-line
+ changes for slice 3. `DEPLOY_CLOUDFLARE.md:41` still says a raw `pnpm deploy` from `export`
+ only stages archives. `pnpm run deploy` is now `archilyzer deploy site`, which uploads them.
+ That doc is due to be absorbed into `PUBLISH.md` in slice 3. `settingsSchema.ts:417,433`
+ ("basic — `pnpm run build` in export/") is still true, since that script is now the CLI.
+- **A Pages project must exist before its first deploy.** *(Corrected after review; the first
+ version of this bullet was wrong.)* wrangler offers to create a missing project only when
+ `process.stdin.isTTY` is set. `runChildIntoLog` spawns it with piped stdin, both from an
+ editor job and from the CLI in a terminal, so it never gets that prompt. It fails at once
+ with wrangler's own "The Pages project … does not exist" sentence, and it never hangs. So the
+ projects are created first, with `pnpm dlx wrangler pages project create <name>
+ --production-branch main`. **Done by the parent on 2026-09-25:** `archilyzer-hub` was created
+ (empty). `archilyzer` already existed, with no deployment. Rollout step 9 therefore needs no
+ project creation. Run `pnpm dlx wrangler pages project list` first to confirm both are there.
+- `build homepage` does not rebuild the index. The homepage's own `pnpm run build` still does,
+ through its `prebuild` (its twins were left as they were, as specified). Run `archilyzer index`
+ first when the index is stale.
+- The `build-deploy` action's deploy half still calls the `run*` phases directly. It has its own
+ banners (a leading newline, no second built-bundle check straight after its own build), which
+ `deploySite` would change.
+- `common/lib/builtExport.test.ts` gained a test. It is the test of an owned file, but not on
+ the ownership list by name.
+- **`export/e2e-2origin/globalSetup.ts:142` runs `pnpm run build:hub`**, which is now
+ `archilyzer build hub`. That command removes `public/site.json` first and sets
+ `NODE_ENV` / `TRANSCRIPTS_DIR` / `EXPORT_PUBLIC_DIR`. The suite caches its hub bundle
+ (it skips the build when `hubA/sw.js` exists), so it was run with `TWO_ORIGIN_REBUILD=1` on
+ the merged tree (see the re-gate below). In a worktree, `export/public` entries are symlinks
+ into the primary, and compose-hub writes through them. For that run, the files the hub build
+ writes or removes (`site.json`, `hub-sites.json`, `corpus.json`, `llms.txt`, `robots.txt`,
+ `_headers`, `sw.js`) were replaced with real copies first, and the symlinks were restored
+ afterwards. The old `build:hub` wrote through them in the same way.
+- **A production `deploy hub` takes its branch from the git checkout** (no `--branch`). This
+ matches `runDeployIntoLog` for sites, and was left as it is by the parent's decision. Run from
+ a non-`main` checkout, it would become a preview. `deploy homepage` passes `--branch main`.
+ Step 9 runs from the primary on `main`.
+- **The homepage hero test was not given a positive fixture (review L7, optional).** The
+ homepage suite has no fixture tree. Its `next dev` reads `getPaths()` with no
+ `TRANSCRIPTS_DIR` override, so seeding a `homepage.json` from a spec would write
+ `transcripts/sites/_homepage/homepage.json`: in the primary, the real corpus's file. It stays
+ conditional. Step 9 checks the link live.
+- **`--preview` takes the next word** (review L6): `deploy site --preview jeralyzer` reads
+ `jeralyzer` as the branch and takes the site from `SITE_ID`. The result is always a preview,
+ never production. The usage text puts `<id>` first.
+- **Commit trailers** name `Claude Opus 5.5 (1M context)`, as in releases 5 and 6.
+
+
## Rollout
diff --git a/scripts/archilyzer-ops.mjs b/scripts/archilyzer-ops.mjs
@@ -34,6 +34,9 @@
// pnpm ops build-site --json '{"siteId":"anilyzer"}' --wait
// pnpm ops build-deploy --json '{"siteIds":["anilyzer","jeralyzer"]}' --wait
// pnpm ops deploy-site --json '{"siteId":"anilyzer","preview":"tags-exclude"}' --wait
+// pnpm ops build-hub --wait
+// pnpm ops build-hub --json '{"deploy":true}' --wait
+// pnpm ops deploy-hub --wait
// pnpm ops get channel the-quartering
// pnpm ops tags --json '{"op":"define","tag":{"id":"eva-collab","label":"Collab"}}'
// pnpm ops tag-videos --file ids.json
@@ -98,6 +101,10 @@ const ACTIONS = [
// one a preview is for: build once, look at the preview, then ship the same
// bundle to production without rebuilding it.
"deploy-site",
+ // The HUB (the export app in hub mode, into the same export/out a site
+ // build uses) and its deploy to homepage.json's Pages project.
+ "build-hub",
+ "deploy-hub",
"relocate",
"relocate-back",
"evict-clips",
@@ -289,6 +296,10 @@ export function usage() {
" alone. The alias is printed after the response. Lowercase letters,",
' digits and dashes, up to 28 characters; "main" is refused.',
"",
+ 'build-hub builds the hub into export/out; {"deploy": true} deploys it',
+ ' after, and deploy-hub ships the one already built. Both deploy to the',
+ " Pages project set on /sites under Hub, and take \"preview\" too.",
+ "",
"Env: ARCHILYZER_EDITOR_URL (default http://localhost:3001), WORKER_TOKEN,",
" ARCHILYZER_AGENT (provenance of a tag write; default \"cli\")",
].join("\n");
diff --git a/scripts/archilyzer-ops.test.mjs b/scripts/archilyzer-ops.test.mjs
@@ -316,3 +316,18 @@ test("--wait-timeout implies --wait", () => {
assert.equal(p.wait, true);
assert.equal(p.waitTimeout, 30);
});
+
+// The hub's pair, added with its deploy path (release 7). A POST each, with
+// the body passed through untouched: the routes judge it.
+test("build-hub and deploy-hub are POSTs to their own routes, named in the usage", () => {
+ const build = parseArgs(["build-hub", "--json", '{"deploy":true}', "--wait"]);
+ assert.equal(build.method, "POST");
+ assert.equal(build.path, "/api/ops/build-hub");
+ assert.deepEqual(build.body, { deploy: true });
+ assert.equal(build.wait, true);
+ const deploy = parseArgs(["deploy-hub"]);
+ assert.equal(deploy.path, "/api/ops/deploy-hub");
+ assert.deepEqual(deploy.body, {});
+ assert.match(usage(), /Actions:.*build-hub, deploy-hub/);
+ assert.match(usage(), /build-hub builds the hub into export\/out/);
+});
