Archilyzer · Source

archilyzer

Archilyzer
git clone https://archilyzer.pages.dev/source/archilyzer.git
Log | Files | Refs | README | LICENSE

commit 40add8319e7ef60697d76ddaf1b4eb70819d19a7
parent 3c062e11f1d145bd044ab38ede181290627e6a13
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date:   Fri, 25 Sep 2026 11:10:07 -0400

publish: buildSite / deploySite / buildAll; export's build twins collapse

common/publish/build.ts gains named entry points over the run* phases:
buildSite (runBuildPhase), deploySite (site + preview + built-bundle
refusals, R2 upload, Pages deploy — the deploy job's body, throwing the same
sentences) and buildAll (docker fan-out or the serial host loop lifted from
buildAllSitesAction). The editor's build-export, build-deploy, deploy-export
and build-all jobs call them; their pre-job refusals stay in the actions.

runBuildPhase no longer spawns pnpm run build|build:nodata: buildSiteSteps
lists the children (build:data unless skipData, compose:site, next build, all
in export/ with the one env block), pinned in build.test.ts. export's
prebuild and build:nodata are deleted; build is archilyzer build site (SITE_ID
from env, --nodata forwards) and deploy is archilyzer deploy site, which
names the site's project and refuses without one. The CLI gains build site,
build all and deploy site.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Diffstat:
Mcommon/bin/archilyzer.ts | 109+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mcommon/publish/build.test.ts | 49+++++++++++++++++++++++++++++++++++++++++++++++++
Mcommon/publish/build.ts | 238+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------
Meditor/app/sites/lib/buildAction.ts | 38+++++++++++++++++++-------------------
Meditor/app/sites/lib/deployAction.ts | 33+++++----------------------------
Mexport/package.json | 6++----
6 files changed, 398 insertions(+), 75 deletions(-)

diff --git a/common/bin/archilyzer.ts b/common/bin/archilyzer.ts @@ -48,6 +48,78 @@ export const COMMANDS: Command[] = [ }, }, { + path: ["build", "site"], + usage: + "<id> [--nodata] [--skip-archives] data phase + compose + next build into export/out (default id: SITE_ID)", + flags: { nodata: "boolean", "skip-archives": "boolean" }, + maxPositionals: 1, + run: async ({ positionals, flags, env }) => { + const siteId = siteIdFrom(positionals, env, "build site"); + if (!siteId) return 2; + // A missing site.json reads as a site of defaults, so a typo would build + // the whole data phase before compose noticed. Refuse it up front. + const { listSiteIds } = await import("../lib/site"); + const known = listSiteIds(); + if (!known.includes(siteId)) { + console.error( + `build site: no site "${siteId}" (configured: ${known.join(", ") || "none"})`, + ); + return 2; + } + const { buildSite } = await import("../publish/build"); + const code = await buildSite(siteId, { + signal: interrupted(), + skipData: flags.nodata === true, + skipArchives: flags["skip-archives"] === true, + }); + if (code !== 0) console.error(`build site ${siteId}: failed (exit ${code})`); + return code; + }, + }, + { + path: ["build", "all"], + usage: + "[--skip-archives] build every site: docker fan-out when an engine answers, else serially on the host", + flags: { "skip-archives": "boolean" }, + run: async ({ flags }) => { + const { buildAll, dockerAvailable } = await import("../publish/build"); + const signal = interrupted(); + const useDocker = await dockerAvailable(signal); + if (!useDocker) { + console.log( + "[notice] No container engine available — building sites serially on the host.", + ); + } + const outcomes = await buildAll({ + signal, + mode: useDocker ? "docker" : "basic", + skipArchives: flags["skip-archives"] === true, + }); + const failed = outcomes.filter((o) => o.code !== 0); + console.log(`\n=== Summary: ${outcomes.length - failed.length}/${outcomes.length} built ===`); + for (const f of failed) console.error(` ${f.siteId}: exit ${f.code}`); + return failed.length || signal.aborted ? 1 : 0; + }, + }, + { + path: ["deploy", "site"], + usage: + "<id> [--preview <branch>] ship the site built in export/out to its Pages project (default id: SITE_ID)", + flags: { preview: "string" }, + maxPositionals: 1, + run: async ({ positionals, flags, env }) => { + const siteId = siteIdFrom(positionals, env, "deploy site"); + if (!siteId) return 2; + const { deploySite } = await import("../publish/build"); + return refusalsExit(() => + deploySite(siteId, { + signal: interrupted(), + previewBranch: typeof flags.preview === "string" ? flags.preview : undefined, + }), + ); + }, + }, + { path: ["sync", "tick"], usage: "POST one scheduler tick to the editor (SYNC_TICK_URL, SYNC_TICK_TOKEN)", run: async () => (await import("./sync-tick")).tick(), @@ -61,4 +133,41 @@ export const COMMANDS: Command[] = [ }, ]; +// The site a site command names: its argument, else SITE_ID (which is how +// export's `build` / `deploy` scripts are called). Prints and returns null when +// there is neither. +function siteIdFrom( + positionals: string[], + env: NodeJS.ProcessEnv, + name: string, +): string | null { + const id = (positionals[0] ?? env.SITE_ID ?? "").trim(); + if (!id) { + console.error(`${name}: which site? Pass its id, or set SITE_ID.`); + return null; + } + return id; +} + +// Ctrl-C cancels the step in flight the way the editor's Cancel does: the +// signal reaches the child, and the entry point stops before its next step. +function interrupted(): AbortSignal { + const ac = new AbortController(); + process.once("SIGINT", () => ac.abort()); + process.once("SIGTERM", () => ac.abort()); + return ac.signal; +} + +// An entry point that THROWS its refusals and failures (the deploys): the +// sentence goes to stderr and the exit code is 1. +async function refusalsExit(fn: () => Promise<void>): Promise<number> { + try { + await fn(); + return 0; + } catch (err) { + console.error((err as Error).message); + return 1; + } +} + runIfEntryPoint(import.meta.url, () => runCli(COMMANDS, process.argv.slice(2))); diff --git a/common/publish/build.test.ts b/common/publish/build.test.ts @@ -2,6 +2,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import type { Paths } from "../lib/paths"; import { + buildSiteSteps, dockerSiteOutDir, dockerSiteStagingDir, resolveOutDir, @@ -17,6 +18,8 @@ import { const paths = { exportDir: "/repo/export", exportBuildsDir: "/repo/export/.export-builds", + exportPublicDir: "/repo/export/public", + transcriptsDir: "/data/transcripts", } as Paths; test("resolveOutDir is export/out whatever the site", () => { @@ -41,3 +44,49 @@ test("dockerSiteStagingDir nests .r2-staging/<site>/archives under the site's bu "/repo/export/.export-builds/jeralyzer/.r2-staging/jeralyzer/archives", ); }); + +// The basic build of one site, as the children it runs. Pinned because the +// data phase used to be npm's `prebuild` hook on a script called `build`, and +// `build:nodata` (the same body under another name) was how it was skipped — +// both scripts are gone, so this list is now the whole contract. +test("buildSiteSteps: data phase, compose, next build — all in export/, one env", () => { + const steps = buildSiteSteps({ + siteId: "jeralyzer", + paths, + baseEnv: { PATH: "/bin" }, + }); + const env = { + PATH: "/bin", + NODE_ENV: "production", + TRANSCRIPTS_DIR: "/data/transcripts", + EXPORT_PUBLIC_DIR: "/repo/export/public", + SITE_ID: "jeralyzer", + }; + assert.deepEqual(steps, [ + { command: "pnpm", args: ["run", "build:data"], cwd: "/repo/export", env }, + { command: "pnpm", args: ["run", "compose:site"], cwd: "/repo/export", env }, + { command: "pnpm", args: ["exec", "next", "build"], cwd: "/repo/export", env }, + ]); +}); + +test("buildSiteSteps: skipData drops the data phase; skipArchives sets BUILD_ARCHIVES=0", () => { + const steps = buildSiteSteps({ + siteId: "anilyzer", + paths, + skipData: true, + skipArchives: true, + baseEnv: {}, + }); + assert.deepEqual( + steps.map((s) => s.args.join(" ")), + ["run compose:site", "exec next build"], + ); + for (const s of steps) { + assert.equal(s.env.BUILD_ARCHIVES, "0"); + assert.equal(s.env.SITE_ID, "anilyzer"); + } + assert.equal( + buildSiteSteps({ siteId: "a", paths, baseEnv: {} })[0].env.BUILD_ARCHIVES, + undefined, + ); +}); diff --git a/common/publish/build.ts b/common/publish/build.ts @@ -14,14 +14,16 @@ import { createReadStream, existsSync } from "node:fs"; import { S3Client, HeadObjectCommand } from "@aws-sdk/client-s3"; import { Upload } from "@aws-sdk/lib-storage"; import { runChildIntoLog } from "../jobs/runChild"; +import { builtSiteProblem } from "../lib/builtExport"; import { deploymentUrlIn, pagesDeployArgs, previewAliasUrl, + previewBranchProblem, } from "../lib/pagesDeploy"; -import type { Paths } from "../lib/paths"; +import { getPaths, type Paths } from "../lib/paths"; import { getSettings } from "../lib/settings"; -import type { Site } from "../lib/site"; +import { getSite, listSites, type Site } from "../lib/site"; // Where the basic (host) build writes the static bundle to deploy: the fixed // export/out, composed one site at a time. The docker fan-out writes per-site @@ -43,9 +45,79 @@ export function dockerSiteStagingDir(paths: Paths, siteId: string): string { return path.join(paths.exportBuildsDir, siteId, ".r2-staging", siteId, "archives"); } +// One child process of a build: what to run, where, with which environment. +export type BuildStep = { + command: string; + args: string[]; + cwd: string; + env: NodeJS.ProcessEnv; +}; + +// The basic (host) build of one site, as the child processes it runs, in order: +// the pool-wide data phase (index + stats + chart templates) unless `skipData`, +// then compose:site for THIS site, then `next build` — all in export/, all with +// the same environment. +// +// These used to be ONE child, `pnpm run build` (or `build:nodata`), whose data +// phase was npm's `prebuild` lifecycle hook: two scripts with identical bodies +// that differed only by name, so the hook fired for one and not the other. The +// hook is gone and the data phase is an explicit step here, which is what lets +// export's `build` script BE this function (`archilyzer build site`) without +// running itself. `skipData` still assumes a prior full build's +// .export-index staging. +// +// `baseEnv` is the environment the steps inherit (process.env by default); +// passing one is how the test pins the argv without the host's. +export function buildSiteSteps(opts: { + siteId: string; + paths: Paths; + skipData?: boolean; + skipArchives?: boolean; + baseEnv?: NodeJS.ProcessEnv; +}): BuildStep[] { + const { paths } = opts; + const env: NodeJS.ProcessEnv = { + ...(opts.baseEnv ?? process.env), + NODE_ENV: "production", + TRANSCRIPTS_DIR: paths.transcriptsDir, + EXPORT_PUBLIC_DIR: paths.exportPublicDir, + SITE_ID: opts.siteId, + // Per-build opt-out for the bulk-download archive zips. BUILD_ARCHIVES=0 + // makes compose-site skip generation this build regardless of the + // global/site flags. + ...(opts.skipArchives ? { BUILD_ARCHIVES: "0" } : {}), + }; + const step = (args: string[]): BuildStep => ({ + command: "pnpm", + args, + cwd: paths.exportDir, + env, + }); + return [ + ...(opts.skipData ? [] : [step(["run", "build:data"])]), + step(["run", "compose:site"]), + step(["exec", "next", "build"]), + ]; +} + +// Run a list of child steps in order, streaming into `onLog`, stopping at the +// first non-zero exit (or a cancel). Returns the exit code. +async function runSteps( + onLog: (line: string) => void, + signal: AbortSignal, + steps: BuildStep[], +): Promise<number> { + for (const s of steps) { + if (signal.aborted) return 1; + const code = await runChildIntoLog(onLog, signal, s); + if (code !== 0) return code; + } + return signal.aborted ? 1 : 0; +} + // Run the basic (host) build phase for one site, streaming into `onLog`, -// returning the exit code. Runs `pnpm run build` in export/ (serialized upstream -// on the build queue, since the export/ tree is shared). This is the single-site +// returning the exit code: buildSiteSteps, in export/ (serialized upstream on +// the build queue, since the export/ tree is shared). This is the single-site // build for basic mode, and the fallback the all-sites docker action drops to // when no container engine is available. The parallel per-site container path // lives in runDockerBuildAllPhase. @@ -56,11 +128,8 @@ export async function runBuildPhase( paths: Paths, opts?: { skipData?: boolean; skipArchives?: boolean }, ): Promise<number> { - // When skipping the data rebuild, run the `build:nodata` script instead of - // `build`. `build:nodata` has the same body (compose:site + next build) but a - // different name, so npm's `prebuild` lifecycle hook (which runs build:data = - // index/stats/templates) does NOT fire — we compose from the existing - // .export-index staging. Assumes a prior full build produced that staging. + // Skipping the data rebuild composes from the existing .export-index staging + // (see buildSiteSteps). const skipData = opts?.skipData === true; if (skipData) { onLog( @@ -68,25 +137,15 @@ export async function runBuildPhase( "existing .export-index staging.\n", ); } - // Per-build opt-out for the bulk-download archive zips. BUILD_ARCHIVES=0 makes - // compose-site skip generation this build regardless of the global/site flags. const skipArchives = opts?.skipArchives === true; if (skipArchives) { onLog("[notice] Skipping archive-zip generation for this build.\n"); } - return runChildIntoLog(onLog, signal, { - command: "pnpm", - args: ["run", skipData ? "build:nodata" : "build"], - cwd: paths.exportDir, - env: { - ...process.env, - NODE_ENV: "production", - TRANSCRIPTS_DIR: paths.transcriptsDir, - EXPORT_PUBLIC_DIR: paths.exportPublicDir, - SITE_ID: siteId, - ...(skipArchives ? { BUILD_ARCHIVES: "0" } : {}), - }, - }); + return runSteps( + onLog, + signal, + buildSiteSteps({ siteId, paths, skipData, skipArchives }), + ); } // Where the basic (host) compose staged this site's oversize archives for R2 @@ -580,3 +639,134 @@ async function runWithConcurrency<T, R>( await Promise.all(runners); return results; } + +// --------------------------------------------------------------------------- +// Named entry points (one-core Phase 4 slice 2): one call per thing an operator +// publishes, over the run* phases above. The editor's actions wrap these in +// runManagedFunction (a job, a queue, a log); the archilyzer CLI calls them +// straight, logging to the terminal. Every run* export stays — they are still +// the parts, and build.test.ts pins some of them. +// --------------------------------------------------------------------------- + +export type PublishOpts = { + paths?: Paths; + // Default: the terminal. + onLog?: (line: string) => void; + // Default: never aborted. + signal?: AbortSignal; +}; + +// runChildIntoLog hands onLog lines WITHOUT their newline and the actions' own +// notices end in one; the terminal gets exactly one either way. +export function terminalLog(line: string): void { + process.stdout.write(line.endsWith("\n") ? line : `${line}\n`); +} + +function resolved(opts: PublishOpts): { + paths: Paths; + onLog: (line: string) => void; + signal: AbortSignal; +} { + return { + paths: opts.paths ?? getPaths(), + onLog: opts.onLog ?? terminalLog, + signal: opts.signal ?? new AbortController().signal, + }; +} + +/** Build one site into export/out (basic/host mode). Returns the exit code. */ +export async function buildSite( + siteId: string, + opts: PublishOpts & { skipData?: boolean; skipArchives?: boolean } = {}, +): Promise<number> { + const { paths, onLog, signal } = resolved(opts); + return runBuildPhase(onLog, signal, siteId.trim(), paths, { + skipData: opts.skipData, + skipArchives: opts.skipArchives, + }); +} + +/** + * Deploy the site already built in export/out: its oversize archives to R2, + * then the bundle to its Pages project (a PREVIEW with `previewBranch`). + * THROWS on every refusal and failure, with the sentences the editor's deploy + * job has always ended on; returns quietly on a cancel. + * + * The refusals are checked here even though the editor action checks them + * before it starts the job, because a queued deploy can start after another + * site's build has replaced export/out — the action's check is the fast answer, + * this one is the last word. + */ +export async function deploySite( + siteId: string, + opts: PublishOpts & { previewBranch?: string } = {}, +): Promise<void> { + const { paths, onLog, signal } = resolved(opts); + if (opts.previewBranch !== undefined) { + const problem = previewBranchProblem(opts.previewBranch); + if (problem) throw new Error(problem); + } + const branch = opts.previewBranch?.trim() || undefined; + const site = getSite(siteId.trim(), paths); + if (!site.cloudflareProject) { + throw new Error( + `Site "${site.siteId}" has no Cloudflare Pages project configured.`, + ); + } + const outDir = resolveOutDir(site.siteId, paths); + const builtProblem = builtSiteProblem(outDir, site.siteId); + if (builtProblem) throw new Error(builtProblem); + // The production path logs no banner and gains none here: its log has + // always opened on wrangler's own first line. + if (branch) { + onLog(`=== Deploy (preview "${branch}") ===\n`); + onLog(PREVIEW_SHARES_ARCHIVES_NOTICE); + } + // Push oversize archives to R2 first, so the manifest URLs the Pages deploy + // publishes resolve immediately. No-op when R2 isn't configured. + const uploadCode = await runArchiveUploadIntoLog(onLog, signal, site, paths); + if (signal.aborted) return; + if (uploadCode !== 0) { + throw new Error(`Archive R2 upload failed (exit ${uploadCode}).`); + } + const code = await runDeployIntoLog(onLog, signal, site, outDir, paths, { + previewBranch: branch, + }); + if (signal.aborted) return; + if (code !== 0) throw new Error(`Deploy failed (exit ${code}).`); +} + +/** + * Build every configured site (or `sites`), without deploying. "docker" is the + * parallel per-site container fan-out (runDockerBuildAllPhase, which throws on + * an infrastructure failure); "basic" is a serial host loop whose shared + * export/out is overwritten per site, so only the last survives. The pool-wide + * data phase runs once either way. Per-site failures are returned, not thrown. + */ +export async function buildAll( + opts: PublishOpts & { + mode: "docker" | "basic"; + skipArchives?: boolean; + sites?: Site[]; + }, +): Promise<SiteBuildOutcome[]> { + const { paths, onLog, signal } = resolved(opts); + const sites = opts.sites ?? listSites(paths); + if (opts.mode === "docker") { + return runDockerBuildAllPhase(onLog, signal, sites, paths, { + skipArchives: opts.skipArchives, + }); + } + const outcomes: SiteBuildOutcome[] = []; + for (let i = 0; i < sites.length; i++) { + if (signal.aborted) break; + const site = sites[i]; + onLog(`\n=== Build ${site.siteId} (${i + 1}/${sites.length}) ===`); + const code = await runBuildPhase(onLog, signal, site.siteId, paths, { + skipData: i > 0, + skipArchives: opts.skipArchives, + }); + outcomes.push({ siteId: site.siteId, code }); + } + return outcomes; +} diff --git a/editor/app/sites/lib/buildAction.ts b/editor/app/sites/lib/buildAction.ts @@ -23,6 +23,8 @@ import { } from "yt-dlp-transcript-common/jobs/streamCommand"; import { PREVIEW_SHARES_ARCHIVES_NOTICE, + buildAll, + buildSite, dockerAvailable, dockerSiteOutDir, resolveOutDir, @@ -91,7 +93,10 @@ export async function buildExportAction( queueKey: queueKey === undefined ? DEFAULT_BUILD_QUEUE : queueKey.trim(), paths, fn: async (onLog, signal) => { - const code = await runBuildPhase(onLog, signal, id, paths, { + const code = await buildSite(id, { + paths, + onLog, + signal, skipData, skipArchives, }); @@ -139,7 +144,10 @@ export async function buildAndDeployAction( paths, fn: async (onLog, signal) => { onLog("=== Build ===\n"); - const buildCode = await runBuildPhase(onLog, signal, id, paths, { + const buildCode = await buildSite(id, { + paths, + onLog, + signal, skipArchives, }); // A cancel mid-build must NOT proceed to deploy. @@ -338,27 +346,19 @@ export async function buildAllSitesAction( fn: async (onLog, signal) => { const useDocker = await dockerAvailable(signal); if (signal.aborted) return; - let outcomes: SiteBuildOutcome[]; - if (useDocker) { - outcomes = await runDockerBuildAllPhase(onLog, signal, sites, paths, { - skipArchives, - }); - } else { + if (!useDocker) { onLog( "[notice] No container engine available — building sites serially on the host.\n", ); - outcomes = []; - for (let i = 0; i < sites.length; i++) { - if (signal.aborted) return; - const site = sites[i]; - onLog(`\n=== Build ${site.siteId} (${i + 1}/${sites.length}) ===`); - const code = await runBuildPhase(onLog, signal, site.siteId, paths, { - skipData: i > 0, - skipArchives, - }); - outcomes.push({ siteId: site.siteId, code }); - } } + const outcomes = await buildAll({ + paths, + onLog, + signal, + mode: useDocker ? "docker" : "basic", + skipArchives, + sites, + }); if (signal.aborted) return; const ok = outcomes.filter((o) => o.code === 0).length; const failed = outcomes diff --git a/editor/app/sites/lib/deployAction.ts b/editor/app/sites/lib/deployAction.ts @@ -9,10 +9,8 @@ import { type StreamActionResult, } from "yt-dlp-transcript-common/jobs/streamCommand"; import { - PREVIEW_SHARES_ARCHIVES_NOTICE, + deploySite, resolveOutDir, - runArchiveUploadIntoLog, - runDeployIntoLog, } from "yt-dlp-transcript-common/publish/build"; const DEPLOY_QUEUE = "deploy"; @@ -64,30 +62,9 @@ export async function deployExportAction( kind: "deploy-export", queueKey: DEPLOY_QUEUE, paths, - fn: async (onLog, signal) => { - // The production path logs no banner and gains none here: its log has - // always opened on wrangler's own first line. - if (branch) { - onLog(`=== Deploy (preview "${branch}") ===\n`); - onLog(PREVIEW_SHARES_ARCHIVES_NOTICE); - } - // Push oversize archives to R2 first, so the manifest URLs the Pages - // deploy publishes resolve immediately. No-op when R2 isn't configured. - const uploadCode = await runArchiveUploadIntoLog(onLog, signal, site, paths); - if (signal.aborted) return; - if (uploadCode !== 0) { - throw new Error(`Archive R2 upload failed (exit ${uploadCode}).`); - } - const code = await runDeployIntoLog( - onLog, - signal, - site, - outDir, - paths, - { previewBranch: branch }, - ); - if (signal.aborted) return; - if (code !== 0) throw new Error(`Deploy failed (exit ${code}).`); - }, + // The refusals above answer before a job exists; deploySite checks them + // again when the job actually starts, and does the upload + deploy. + fn: (onLog, signal) => + deploySite(site.siteId, { paths, onLog, signal, previewBranch: branch }), }); } diff --git a/export/package.json b/export/package.json @@ -13,9 +13,7 @@ "detect:duplicates": "NODE_OPTIONS=--max-old-space-size=8192 tsx ../common/bin/duplicate-shorts.ts", "compose:site": "tsx ../common/bin/compose-site.ts", "compose:hub": "tsx ../common/bin/compose-hub.ts", - "prebuild": "pnpm run build:data", - "build": "pnpm run compose:site && next build", - "build:nodata": "pnpm run compose:site && next build", + "build": "tsx ../common/bin/archilyzer.ts build site", "build:hub": "pnpm run compose:hub && INSTANCE_MODE=hub next build", "start": "serve out", "lint": "eslint", @@ -23,7 +21,7 @@ "e2e:hub": "node ../scripts/queue-lock.mjs --ports HUB_PORT:3041 -- playwright test --config playwright.hub.config.ts", "e2e:2origin": "node ../scripts/queue-lock.mjs --ports ORIGIN_B_PORT:4610,HUB_A_PORT:4611 -- playwright test --config playwright.2origin.config.ts", "e2e:ui": "playwright test --ui", - "deploy": "pnpm dlx wrangler pages deploy out" + "deploy": "tsx ../common/bin/archilyzer.ts deploy site" }, "dependencies": { "@tanstack/react-query": "^5.99.1",