commit a835e96885ddbf6a9817328d914c056f0c5641bd
parent 2484c402c66b17b2bae34b5ea57d05af8b6f885e
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 9 Oct 2026 14:02:16 -0400
Merge r19/integration into Track D (tracks B and C: the docs cli and COMMANDS.md, the heavy slot, reports check/verify-quotes/attach-video, release 22's plan) before the Track D records; the changelog keeps both sides, Track D first
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
64 files changed, 4005 insertions(+), 505 deletions(-)
diff --git a/AGENTS.md b/AGENTS.md
@@ -39,7 +39,15 @@ specs that judge a clip read it through `e2e/capabilities.ts` and skip themselve
capability is a directory that exists, decided once by the builder — not re-guessed per
spec.
-See [WORKTREES.md](WORKTREES.md) for the port scheme, the queue, and the shared-data caveat.
+**Heavy work takes the heavy slot.** e2e, the publish stages' `next build` and video renders
+share ONE machine-global slot and start only above a 6000 MB MemAvailable floor (two OOMs
+took the desktop session down). e2e and the publish builds take it on their own; a render or
+any other heavy command runs as `pnpm heavy -- <cmd>`. `queue-lock: waiting for the heavy slot
+— held by …` or `heavy: waiting for memory …` is the gate working, not a hang. Bypasses:
+`HEAVY=0`, `HEAVY_MIN_FREE_MB=<MB>`, `HEAVY_TIMEOUT=<seconds>`.
+
+See [WORKTREES.md](WORKTREES.md) for the port scheme, the queue, the heavy slot, and the
+shared-data caveat.
# Working this repo with no local corpus
@@ -85,6 +93,12 @@ another markdown file that will drift from it.
Use `/ask` for a question answered in the conversation and `/sweep` for a cited report
written to a file.
+**Running an archive — adding channels, syncing, importing, transcribing, tagging,
+reports, publishing — is [OPERATING.md](OPERATING.md)**: recipes over `pnpm ops` (the
+running editor's actions), `pnpm archilyzer` and the MCP. Every command and action is in
+[COMMANDS.md](COMMANDS.md), generated by `pnpm archilyzer docs cli` from their help; after
+adding an ops action or a CLI row, regenerate it (`--check` is a gate).
+
## Clips and report-to-video
The high-value loop: point the MCP at a public instance, ask about a subject, then pull
diff --git a/COMMANDS.md b/COMMANDS.md
@@ -0,0 +1,137 @@
+# Command reference
+
+<!-- GENERATED by common/bin/cli-docs.ts from the `archilyzer` command table (common/bin/archilyzer.ts) and `pnpm ops --help` (scripts/archilyzer-ops.mjs) — do not edit by hand. Regenerate: `pnpm archilyzer docs cli`. -->
+
+Every `archilyzer` command and every `pnpm ops` action, from their own help. Recipes that chain them: [OPERATING.md](OPERATING.md).
+
+## `pnpm archilyzer`
+
+Run from the repo root (in the container: `docker compose exec editor pnpm archilyzer …`). `--help` after a command prints its line. A command marked *passthrough* parses its own flags.
+
+| command | arguments | what it does |
+|---|---|---|
+| `archilyzer index` | | rebuild the LMDB transcript index |
+| `archilyzer build stats` | | rebuild the stats datasets (reads the index; the data phase's second step) |
+| `archilyzer build templates` | | bake each site's chart templates into its export staging dir (the data phase's third step) |
+| `archilyzer build archives` | | warm the shared archive-zip cache for every enabled site's channels, once |
+| `archilyzer compose site` | `<id> [--allow-missing-media]` | compose one site's export/public (default: SITE\_ID); --allow-missing-media lets a report citation with no prepared media through |
+| `archilyzer compose hub` | | compose the hub's export/public (hub-sites.json, corpus.json, …) |
+| `archilyzer compose homepage` | | compose homepage/public (whole-pool stats + landing summary) |
+| `archilyzer publish index` | | update the index: the LMDB index, the stats datasets and the chart templates in one child (8 GB heap), then the index stamp every build reads |
+| `archilyzer publish build` | `<id\|all> [--runner local\|docker\|auto] [--force] [--skip-archives]` | build a site (or every stale one) into its bundle <exportBuildsDir>/<id>/out from the current index; --runner docker builds every site in containers (host only); a fresh site is a no-op without --force |
+| `archilyzer publish deploy` | `<id\|all> [--preview <branch>] [--to local] [--force]` | ship a site's bundle to its Pages project (a preview with --preview), or with --to local into ARCHILYZER\_SITE\_OUT; a bundle already deployed there is a no-op without --force |
+| `archilyzer publish hub` | `[--deploy \| --deploy-only] [--preview <branch>] [--force]` | build the hub into its bundle <exportBuildsDir>/\_hub/out, then (--deploy) ship it; --deploy-only ships the bundle as built |
+| `archilyzer publish homepage` | `[--deploy \| --deploy-only] [--preview <branch>] [--to local] [--force]` | build homepage/out (source mirror included), then (--deploy) ship it; --deploy-only ships it as built |
+| `archilyzer publish status` | `[--json]` | the publish status: the index, the lane, and per site / hub / homepage its policy and its built, deployed and live chips, then the plan Publish now would run |
+| `archilyzer publish now` | | run what Publish now runs — the index update when it is stale, then each policy target's build and deploy (site.json publish.auto; settings.json publish.hub / publish.homepage), never forced — one stage at a time IN THIS PROCESS under the publish lock, as the editor's queue would (the CLI has no queue); exit 0 when every stage ran or was a no-op |
+| `archilyzer stage` | `<kind> <target> --run-id <id> [--preview <b>] [--to local] [--runner docker] [--force] [--skip-archives] [--allow-missing-media] [--index-after <ms>] [--built-after <ms>]` | INTERNAL: one publish stage, as the editor's job runs it (exit 0 ran/no-op, 1 failed, 2 usage, 3 precondition not met, 130 cancelled) |
+| `archilyzer build site` | `<id> [--nodata] [--skip-archives] [--allow-missing-media]` | alias: publish index (not with --nodata) + publish build <id> --force (default id: SITE\_ID) |
+| `archilyzer build all` | `[--skip-archives]` | alias: publish index + publish build all --runner auto (containers when an engine answers, else serially on the host) |
+| `archilyzer build hub` | | compose:hub + INSTANCE\_MODE=hub next build into export/out — a raw build, unstamped; to deploy the hub build it with `publish hub` |
+| `archilyzer build homepage` | `[--no-source]` | compose + source publish + next build in homepage/ (reads the index as it stands); --no-source removes the published source instead |
+| `archilyzer reports prepare` | `<id>` | cut every clip and copy every post capture the site's published reports cite into its report-media cache, before its build, then export the reports as files (reports export) when nothing is missing (exit 1 when a citation lacks media or an export fails; default id: SITE\_ID) |
+| `archilyzer reports export` | `<id> [--report <reportId>] [--formats html,pdf,md,zip] [--allow-missing-media]` | write the site's published reports as files (report.html, report.pdf, report.md, evidence-pack.zip) into its report-exports staging, where compose publishes them from (exit 1 on a problem; a PDF skipped for want of a browser is a note; default id: SITE\_ID) |
+| `archilyzer reports check` | `<id> [--reports <a,b>] [--allow-missing-media]` | what compose would say about the site's reports, with no build: each report validated, every cited quote checked against its record, every cited moment's prepared media present and current, the report video under the publish limit (exit 1 with the list; --reports checks those, drafts included; default id: SITE\_ID) |
+| `archilyzer reports verify-quotes` | `<report.json> [--json]` | every video, audio and post quote of one report against its record with compose's own check: the best score and track, and the en-orig track's score where the record has one — a quote that matches a served `en` rewrite and not en-orig is reported (exit 1 when any quote drifted) |
+| `archilyzer reports attach-video` | `<report.json> <video> [--poster <image>] [--caption <line>]` | the report's video: remuxed (an H.264 mp4 under the limit) or encoded to fit the 24 MiB publish limit, written beside report.json as video.mp4 with a poster (given, kept, or a frame of the video), and `video` set in report.json |
+| `archilyzer reports convert` | `<sweep\|ask\|manifest> <in> --out <report.json> [--channels-dir <dir>] [--id <id>] [--title <title>]` | a /sweep report (markdown), an /ask answer or a report-to-video manifest as a report.json, written only when it validates (--channels-dir: widen spans from the cues, find posts' channels) |
+| `archilyzer reports to-manifest` | `<report.json> --out <manifest.json> [--channels-dir <dir>] [--site-origin <url>]` | a starter report-to-video manifest from a report (a chapter card per section, a claim's still and clips stamped with its verdict, its posts) |
+| `archilyzer source publish` | `[--force] [--check] [--keep-scratch]` | the scrubbed git mirror, raw tree, history pages (stagit, when installed) and tarball into homepage/public, behind the denied-literal gate (--check: audit and count, write nothing) |
+| `archilyzer source audit` | `[<git dir>]` | the denied-literal gate (+ gitleaks) over a git dir; default the published homepage/public/source/archilyzer.git |
+| `archilyzer deploy site` | `<id> [--preview <branch>]` | alias: publish deploy <id> \[--preview <branch>\] — ship the site's bundle to its Pages project (default id: SITE\_ID) |
+| `archilyzer deploy hub` | `[--preview <branch>]` | alias: publish hub --deploy-only \[--preview <branch>\] — ship the hub's bundle to homepage.json's Pages project |
+| `archilyzer deploy homepage` | `[--preview <branch>]` | alias: publish homepage --deploy-only \[--preview <branch>\] — ship homepage/out to the Pages project archilyzer |
+| `archilyzer run` | `<operation> <channel> [ids…] [--lane local\|remote]` | run one catalogued operation over a channel offline, as the editor's job does (sync, downloads and transcription are refused: they run in the editor; it does not see the editor's lanes, so not beside one on the same channel) |
+| `archilyzer archive-org refresh` | `<slug> [--dry-run]` | bring a channel's archive.org file records up to their provenance: a name-only mirror's title (where the item gives the file none) and upload date from its file name; offline, through the metadata history; prints old → new |
+| `archilyzer wayback refresh` | `<slug> [--titles <json>] [--dry-run]` | bring a channel's Wayback Machine copies up to the Wayback rules: wayback.json (original URL, capture time), the dir renamed to its canonical id through the snapshot's reconcile (roster moved with it), and with --titles (a file of id → {title, upload\_date}) a raw file's title and date; offline, skips a record a live job holds; prints old → new |
+| `archilyzer feeds backfill-metadata` | `<slug> [--feed <url>] [--dry-run]` | complete a podcast channel's records (title, date, description, duration) from its RSS feed: one fetch of the feed (default: the channel's url), no media; --dry-run counts matched / unmatched / already complete and writes nothing |
+| `archilyzer duplicates` | `[--threshold N] [--all-durations] [--blocking title\|duration\|both] [--near F] [--tolerance N] …` | on-demand duplicate detection (after index + stats) *(passthrough)* |
+| `archilyzer posts fetch` | `--slug <channel> [--full \| --older [--floor YYYY-MM-DD] [--from YYYY-MM-DD] [--force]] [--limit N] [--pages N]` | fetch a social channel's posts into its posts corpus (--older: walk back below the oldest archived post; --pages: a forum thread's latest N pages) *(passthrough)* |
+| `archilyzer posts import-html` | `<slug> <file-or-dir>… [--dry-run]` | import forum thread pages saved from a browser ("Save page as", .html) into a forum-thread channel: new posts appended, edited ones updated, nothing fetched |
+| `archilyzer posts check` | `--slug <channel> [--mode stale\|unchecked\|all] [--limit N]` | which archived posts were deleted at the source *(passthrough)* |
+| `archilyzer diarize backfill` | `[--dry-run] [--scope transcribed\|channel:<slug>\|video:<slug>/<id>] [--limit N] [--force] …` | diarize videos whose audio is still on disk *(passthrough)* |
+| `archilyzer digest plan` | `[--lane local\|remote] [--channels a,b] [--top N] [--json] [--census] …` | price the digest backfill; writes nothing *(passthrough)* |
+| `archilyzer digest validate` | `<channel> [<channel> …]` | score digests already on disk *(passthrough)* |
+| `archilyzer reconcile video-dirs` | `[--channel <slug>] [--dry-run] [--verbose]` | rename video dirs to the canonical id layout *(passthrough)* |
+| `archilyzer verify transcripts` | `--channel <slug>` | list duplicate and missing transcripts *(passthrough)* |
+| `archilyzer migrate channel-priority` | `[--dry-run]` | the one-shot channel-priority migration (plans/channel-priority.md, S5) *(passthrough)* |
+| `archilyzer storage migrate-tier` | `<slug>…\|--all [--order smallest] [--include-large] [--dry-run] [--reclaim]` | bring a channel off the retired whole-directory layout onto the media tier, its text home to the corpus disk (editor stopped; --all stops before the three big-text channels) *(passthrough)* |
+| `archilyzer brand media` | `[--out <dir>] [--video-kit]` | render the Archilyzer Media channel's assets *(passthrough)* |
+| `archilyzer mcp` | `[--local <dir>\|--remote <url>\|--hub <url>]` | start the MCP server on stdio (as `pnpm --filter yt-dlp-transcript-mcp exec tsx src/index.ts`) *(passthrough)* |
+| `archilyzer sync tick` | | POST one scheduler tick to the editor (SYNC\_TICK\_URL, SYNC\_TICK\_TOKEN) |
+| `archilyzer docs env` | `[--check]` | write ENVIRONMENT.md from the declared env-var list (lib/envVars.ts) |
+| `archilyzer docs files` | `[--check]` | write SITE.md + CHANNEL.md + REPORT.md + CITATIONS.md from the file schemas |
+| `archilyzer docs cli` | `[--check]` | write COMMANDS.md from this command table and `pnpm ops --help` |
+| `archilyzer settings example` | `[--check]` | write settings.json.example + SETTINGS.md from the schema |
+| `archilyzer doctor` | `[--json]` | read-only report: node, the checkout, the corpus, settings, every tool, the port block; exit 1 on a failure |
+| `archilyzer release show` | `[editor\|export]` | latest release, its date, how many bullets wait under \[Unreleased\] |
+| `archilyzer release cut` | `<editor\|export\|all> <X.Y.Z\|next\|next-minor> [--commit] [--date YYYY-MM-DD]` | \[Unreleased\] -> a dated heading; all = both, one version, two commits |
+
+## `pnpm ops`
+
+Drives a running editor over HTTP (`/api/ops/*`, the same actions its pages run), gated by `WORKER_TOKEN`. A body is `--json '<object>'` or `--file <path>`; `--wait` follows a job to its end.
+
+| action | what it does | example |
+|---|---|---|
+| `channel-priority` | — | `pnpm ops channel-priority --json '{"slugs":["x"],"operation":"download","tier":"paused"}'` |
+| `channel-config` | channel-config changes a channel as its Configure form does: {"slug"} and any of "patch" (form field names; "" clears one), "sites" (the WHOLE membership set: \[{"siteId", "groupId"? \| "newGroupName"?}\], \[\] = on no site; an unknown site id is refused), "excludeFromBuild" and "excludeFromCleanup" (set to the value given, not toggled). | `pnpm ops channel-config --json '{"slug":"x","patch":{"downloadFilterExclude":"rerun"}}'`<br>`pnpm ops channel-config --json '{"slug":"x","sites":[{"siteId":"anilyzer"}]}'`<br>`pnpm ops channel-config --json '{"slug":"x","sites":[],"excludeFromBuild":true}'` |
+| `create-channel` | create-channel is the New channel form: {"fields": {"name", "handling": "youtube"\|"transcribe", "url"?, "platform"?, "sourceKind"?, "postFetcher"?, "socialHandle"?, …}} with channel-config's patch keys; "slug"? (else derived from the name), "sites"? (absent = on no site). "fetchPlaylist", "fetchPostsNow" and "prioritizeDownload" are the form's checkboxes, OFF unless true; a job they start comes back as jobId(s), so --wait follows it. | `pnpm ops create-channel --json '{"fields":{"name":"Example (X)","handling":"transcribe","url":"https://x.com/example"}}'` |
+| `rename-channel` | rename-channel moves a channel to a new slug, as Danger → Rename does: {"slug", "newSlug"}. Refused while the channel is busy (a job, a lane unit, media in transition) or when the new slug is taken. Old links break. | `pnpm ops rename-channel --json '{"slug":"old-slug","newSlug":"new-slug"}'` |
+| `delete-channel` | delete-channel removes a channel's whole directory, as Danger → Delete does: {"slug", "confirm"} — "confirm" must repeat the slug. No undo outside the transcripts/ repo's own history. | `pnpm ops delete-channel --json '{"slug":"x","confirm":"x"}'` |
+| `metadata-scan` | — | `pnpm ops metadata-scan --json '{"slug":"the-quartering"}'` |
+| `refresh-metadata` | refresh-metadata re-reads ONE video's metadata.info.json from its source (no subtitles, no media) on the platform's queue: {"slug", "id"}. The job's log ends with what the source now says — live\_status, formats, audio-only formats and whether any is non-fragmented, English captions, the keys that changed. An id with no data/<id>/ is refused (a refresh re-reads a video already archived), as are archive.org and Wayback records. | `pnpm ops refresh-metadata --json '{"slug":"the-quartering","id":"<videoId>"}' --wait` |
+| `import-video` | — | `pnpm ops import-video --json '{"slug":"demo-archive","url":"https://archive.org/details/example-item"}'` |
+| `import-archive-org` | — | `pnpm ops import-archive-org --json '{"slug":"demo-archive","item":"example-item","match":"\\.mp4$"}' --wait` |
+| `feed-metadata` | — | `pnpm ops feed-metadata --json '{"slug":"demo-podcast","dryRun":true}' --wait` |
+| `refresh-report` | — | `pnpm ops refresh-report --json '{"all":true}'` |
+| `sync` | — | `pnpm ops sync --json '{"slug":"the-quartering"}' --wait` |
+| `download-missing` | — | |
+| `retry-bucket` | retry-bucket runs one bucket of a channel's report as one job, past any lane hold: {"slug", "bucket"}. "ids": \[...\] runs only those videos, and every one must be in the bucket (a stray id is refused, named); a job run with ids is not replayable, as a checkbox selection in the UI is not. | |
+| `transcribe-bucket` | transcribe-bucket transcribes a channel's "downloaded, not transcribed" bucket on the transcription queue, as the channel page's Transcribe button does: {"slug"}. "ids": \[...\] narrows it the same way as on retry-bucket. | |
+| `fetch-posts` | fetch-posts fetches a social channel's new posts: {"slug"}. "full": true re-walks the whole timeline; "older": true walks back from the oldest archived post through search (X; needs a login), saving its place for the next run, down to "floor": "YYYY-MM-DD" when given; "from": "YYYY-MM-DD" starts the walk afresh there, replacing its saved place (and a "complete") — for a gap above one surviving old post. "limit": N caps the posts one run reads; "pages": N caps the pages (a forum thread: its latest N pages). "full" and "older" together are refused. An older walk over an account that shows no posts (nothing archived, and the last timeline fetch read none) is refused unless "force": true. A drained fetch stops at its next resume point and the next run resumes. | |
+| `capture-posts` | capture-posts captures archived posts of a social channel (X, forum): a screenshot of each through the connected X profile (a forum thread: its host's forum profile), and its attached media through gallery-dl (a forum thread: the same profile), into the channel's posts-media/<id>/: {"slug", "ids": \[...\]}. Every id must be in the channel's posts archive. "shots": false or "media": false skips that half; posts already captured are skipped unless "force": true. A post that links to an X Article also gets the article (article.json, .md, .png and its images) unless "articles": false; both halves off with "articles": true reads only the articles. Paced like a post fetch, on its queue. | |
+| `publish` | publish runs publish stages on the editor's publish queue, one at a time, under one run id: {"verb": …}. "index" updates the index; "build" builds "siteId"/"siteIds" (forced; the index first when stale; "runner": "docker" builds every site in containers); "deploy" ships their built bundles (production, "preview": "<branch>", or "to": "local"; "force" redeploys a bundle already shipped there); "hub" / "homepage" build them, {"deploy": true} deploys after; "now" is Publish now (the stale index, then each policy target); "stale" builds every stale site. The answer lists every job ({target, kind, jobId}) and --wait follows them all. A site never built is refused: "no build of <id> in <dir> — archilyzer publish build <id>". `get publish` is the status. | |
+| `build-index`, `build-site`, `build-deploy`, `deploy-site`, `build-hub`, `deploy-hub`, `build-homepage`, `deploy-homepage` | build-index, build-site, build-deploy, deploy-site, build-hub, deploy-hub, build-homepage and deploy-homepage are publish's aliases, with their old bodies and answers ("skipData" is accepted and ignored). | `pnpm ops build-deploy --json '{"siteIds":["anilyzer","jeralyzer"]}' --wait`<br>`pnpm ops build-site --json '{"siteId":"anilyzer"}' --wait`<br>`pnpm ops deploy-site --json '{"siteId":"anilyzer","preview":"tags-exclude"}' --wait`<br>`pnpm ops build-hub --wait`<br>`pnpm ops build-hub --json '{"deploy":true}' --wait`<br>`pnpm ops deploy-hub --wait`<br>`pnpm ops build-homepage --json '{"deploy":true}' --wait`<br>`pnpm ops deploy-homepage --json '{"preview":"refresh"}' --wait` |
+| `build-site`, `build-deploy`, `deploy-site` | build-site, build-deploy and deploy-site all take "siteId" (one) or "siteIds" (a list). | |
+| `build-hub` | build-hub builds the hub into its bundle; {"deploy": true} deploys it after, and deploy-hub ships the one already built. Both deploy to the Pages project set on /sites under Hub, and take "preview" too. | |
+| `build-homepage` | build-homepage builds the homepage package into homepage/out; {"deploy": true} deploys it after (only if the build succeeded), and deploy-homepage ships the one already built. Both deploy to the Pages project archilyzer (https://archilyzer.pages.dev), production unless "preview" is given. | |
+| `relocate` | — | `pnpm ops relocate --json '{"slugs":["x"],"locationId":"platter"}'` |
+| `relocate-back` | — | |
+| `evict-clips` | — | |
+| `reports-prepare` | reports-prepare cuts every clip and copies every post capture a site's published reports cite into its report-media cache, before its build: {"siteId"}. The job fails, naming each one, when a citation lacks media. When nothing is missing it then exports the reports, as reports-export. | `pnpm ops reports-prepare --json '{"siteId":"demo-site"}' --wait` |
+| `reports-export` | reports-export writes each published report as report.html, report.pdf, report.md and evidence-pack.zip for the site's build to publish: {"siteId", "reportId"?, "formats"?: \["html","pdf","md","zip"\]}. | `pnpm ops reports-export --json '{"siteId":"demo-site","formats":["html","md"]}' --wait` |
+| `lane` | — | `pnpm ops lane --json '{"lane":"download","held":true}'` |
+| `tags` | — | `pnpm ops tags --json '{"op":"define","tag":{"id":"eva-collab","label":"Collab"}}'` |
+| `tag-videos` | — | `pnpm ops tag-videos --file ids.json` |
+| `keep-videos` | — | |
+| `persist-videos` | persist-videos saves specific videos, across channels, to the saved-video store: {"items": \[{"slug", "id"}, ...\]}. "format": "original" \| "video\_720" (default: each channel's own). "replace": "above-height" also re-fetches a saved one whose height is unknown or above that quality (default "never"). "gapMs" pauses between downloads (default the batch gap), "minFreeMemMb" waits for that much free memory before each. "dryRun": true answers with the buckets (saved, wrongHeight, toFetch, noUrl, unknown) and starts nothing. One job per channel, on its download queue; a low disk or a rate limit stops it, and running the same body again resumes — saved videos are skipped. | `pnpm ops persist-videos --file list.json --wait` |
+| `fetch-windows` | fetch-windows fetches clip windows, one paced job per platform queue (YouTube and Rumble side by side): {"siteId"} fetches every window the site's published reports cite and the disk does not hold; {"items": \[{"slug", "id", "from", "to", "clipId"?, "reason"?, "pad"?, "webpageUrl"?}, ...\], "requestedBy", "manifest"?} fetches a list. "maxHeight" caps the source height (default 720). "dryRun": true lists the windows per platform, the ones already on disk ("cached") and the ones no fetch can fill ("unfetchable": deleted, off the site) and starts nothing. A platform cooling down or held is refused for its group; a 429, or two 403s in a row, backs the platform off and stops its job. Running the same body again resumes — fetched windows are cached. | `pnpm ops fetch-windows --json '{"siteId":"demo-site","dryRun":true}'`<br>`pnpm ops fetch-windows --file windows.json --wait` |
+| `cut-release` | cut-release turns a changelog's \[Unreleased\] into "## \[<version>\] - <date>": {"workspace": "editor" \| "export" \| "all", "version": "X.Y.Z" \| "next" \| "next-minor", "commit": boolean (default false), "date": "YYYY-MM-DD" (default today)}. "all" cuts both with ONE version and commits each ("Release <workspace> <version>") — or neither: every check runs before either file is written. Only an editor built from release 10 or later has the route (an older one answers 404); with no editor running, `archilyzer release cut` does the same locally. | `pnpm ops cut-release --json '{"workspace":"all","version":"next","commit":true}'` |
+| `transcribe` | transcribe runs ONE local file through a local transcription worker, as a job: {"path": "/abs/file"} (audio or video), "start"/"end" (seconds) for a window, "workerId" (a settings worker id; default: the one auto-transcribe would get), "out" (an absolute path for the result JSON, never inside the corpus). The result is {path, window, worker: {id, appId, model, device}, cues: \[{start, end, text}\], text, ...}, cue times on the file's own clock. "words": true adds words: \[{w, start, end, conf?}\] on the same clock -- from parakeet, which keeps its word timestamps; \[\] from an engine that does not. With --wait it is printed on stdout (the response and the log go to stderr), so `pnpm ops transcribe ... --wait \| jq -r .text` works. | `pnpm ops transcribe --json '{"path":"/abs/clip.mp4","start":120,"end":150}' --wait`<br>`pnpm ops transcribe --json '{"path":"/abs/a.wav","workerId":"parakeet-cpu","out":"/tmp/a.json"}' --wait` |
+
+### Usage, flags and environment
+
+```text
+Usage: pnpm ops <action> [--json '<body>' | --file <path>] [--wait]
+ [--wait-timeout <seconds>] [--quiet]
+ pnpm ops get channel <slug> [--counts]
+ pnpm ops get channels
+ pnpm ops get tags [<tagId>]
+ pnpm ops get publish
+ pnpm ops list
+
+--wait follows the job's log and survives a poll that fails (a busy
+ in-process build starves the server): it backs off and, after three
+ failures, asks /api/jobs/active whether the job is still there.
+--wait-timeout <seconds> gives up and exits 1 instead of waiting forever.
+ Default: no timeout — the queue may legitimately hold a job for hours.
+
+"preview": "<branch>" on deploy-site or build-deploy makes it a Cloudflare
+ Pages PREVIEW instead of production: the same bundle goes to a branch
+ alias, https://<branch>.<project>.pages.dev, and the live site is left
+ alone. The alias is printed after the response. Lowercase letters,
+ digits and dashes, up to 28 characters; "main" is refused.
+
+Env: ARCHILYZER_EDITOR_URL (default http://localhost:3001), WORKER_TOKEN,
+ ARCHILYZER_AGENT (provenance of a tag write; default "cli")
+```
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -146,16 +146,10 @@ The same controllers the editor uses are one command line, `common/bin/archilyze
which is how you drive the pipeline headlessly or from cron. `pnpm archilyzer
<command>` from the repo root is the short form of `pnpm --filter
yt-dlp-transcript-common exec tsx bin/archilyzer.ts <command>`; `pnpm archilyzer
---help` lists every command.
-
-```bash
-pnpm archilyzer doctor # read-only: can this machine do what it is configured to?
-pnpm archilyzer index # the LMDB index
-pnpm archilyzer run diarization <channel> [ids…] # one catalogued operation, offline, as the editor's job
-pnpm archilyzer build site <id> # publish: see PUBLISH.md
-pnpm archilyzer verify transcripts --channel <slug>
-pnpm archilyzer mcp # the MCP server on stdio
-```
+--help` lists every command. Every command and every `pnpm ops` action is in
+[COMMANDS.md](COMMANDS.md), generated by `pnpm archilyzer docs cli` from the two help
+texts (`--check` fails when it is stale, or when [OPERATING.md](OPERATING.md) names a
+command that does not exist); recipes that chain them are in OPERATING.md.
The table is `archilyzer.ts`; the machinery (parser, lookup, usage) is `_cli.ts`. Every
file in `common/bin/` is reachable from a row — a test fails otherwise. A bin that
diff --git a/ENVIRONMENT.md b/ENVIRONMENT.md
@@ -111,6 +111,9 @@ Tokens, credentials and knobs a running process reads. Most configuration is not
| `PARAKEET_DECODER` | parakeet-cli's | `ctc` or `tdt`, passed through to parakeet-cli. | scripts/parakeet-stitch.mjs |
| `PARAKEET_LANG` | parakeet-cli's | A locale, passed through to parakeet-cli. | scripts/parakeet-stitch.mjs |
| `PARAKEET_DEVICE` | parakeet-cli's | Compute device (`cpu`, `CUDA0`, `Vulkan1`, …), exported to parakeet-cli. | scripts/parakeet-stitch.mjs |
+| `HEAVY` | on | `0` skips the heavy slot AND the memory floor: the machine-wide one-at-a-time gate that `pnpm heavy -- <cmd>`, every e2e entry point and the publish stages' `next build` go through. | scripts/queue-lock.mjs |
+| `HEAVY_MIN_FREE_MB` | `6000` | The memory floor: a heavy job, once it holds the slot, waits until /proc/meminfo's MemAvailable is at least this many MB. `0` turns the floor off; a machine whose MemTotal is under it runs without waiting. | scripts/queue-lock.mjs |
+| `HEAVY_TIMEOUT` | wait forever | Seconds a `pnpm heavy` run waits for the slot, and then for the floor, before giving up (exit 3). An e2e run uses `E2E_QUEUE_TIMEOUT` for both. | scripts/queue-lock.mjs |
## Ports
@@ -187,6 +190,10 @@ Read only by a test harness, a fake binary or a test-mode branch. Never set one
| `E2E_PORT_GRACE_MS` | `3000` | How long the port check waits for a just-freed port. | scripts/queue-lock.mjs |
| `E2E_QUEUE_LOCK_FILE` | one per machine | The queue's lock file; the queue's own tests point it elsewhere. | scripts/queue-lock.mjs |
| `QUEUE_LOCK_HELD` | — | Set by the queue for the command it runs, so a nested wrapper passes through. | scripts/queue-lock.mjs |
+| `HEAVY_HELD` | — | Set by the heavy slot for the command it runs, so a heavy command inside it (a `pnpm heavy -- pnpm e2e`, a build stage under an e2e suite's editor) passes through. | scripts/queue-lock.mjs |
+| `HEAVY_LOCK_FILE` | one per machine | The heavy slot's lock file; the gate's own tests point it elsewhere. | scripts/queue-lock.mjs |
+| `HEAVY_MEMINFO_FILE` | `/proc/meminfo` | Where the memory floor reads MemAvailable; the gate's tests hand it a fake. | scripts/queue-lock.mjs |
+| `HEAVY_POLL_MS` | `5000` | How often a run waiting for the memory floor re-reads it. | scripts/queue-lock.mjs |
| `PLAYWRIGHT_BASE_URL` | `http://localhost:<PORT>` | The editor test server's URL; the worktree injector sets it. | editor/playwright.config.ts, editor/e2e/baseUrl.ts |
| `E2E_AUDIO_CHECK_INTERVAL_MS` | the real cadence | Shrinks the mid-download audio check so the e2e suite sees it fire. | common/ytdlp/audioCheckedDownload.ts |
| `E2E_AUDIO_CHECK_SIZE_GATE` | the real gate | Likewise, the size gate. | common/ytdlp/audioCheckedDownload.ts |
diff --git a/OPERATING.md b/OPERATING.md
@@ -0,0 +1,160 @@
+# Operating an archive
+
+Recipes for running an archive from a shell or an agent. Every command named here is in
+[COMMANDS.md](COMMANDS.md), generated from the commands' own help.
+
+## The three surfaces
+
+| surface | what it is | needs |
+|---|---|---|
+| `pnpm ops <action>` | the running editor's actions over HTTP (`/api/ops/*`): the same code a click runs, as jobs on the editor's queues | a running editor; `ARCHILYZER_EDITOR_URL` (default `http://localhost:3001`) and `WORKER_TOKEN` (the editor's own, from `editor/.env`) |
+| `pnpm archilyzer <command>` | the core's CLI: index, compose, publish stages, reports, offline refreshes, doctor | the checkout and its `transcripts/`; in the container, `docker compose exec editor pnpm archilyzer …` |
+| the MCP server | reads a published archive (search, transcripts, reports); `fetch_clip` asks the editor for clip media | registered as `archilyzer` ([AGENTS.md](AGENTS.md), [mcp/README.md](mcp/README.md)) |
+
+- A job-starting `pnpm ops` action answers with a `jobId` once the job is queued. `--wait` follows its log to the
+ end and exits with its status; a platform queue may hold a job for hours.
+- What is there: `pnpm ops get channels`, `pnpm ops get channel <slug> --counts`, `pnpm ops get publish`,
+ `pnpm archilyzer doctor`.
+- Every fetch goes through the editor (paced per platform, cookie-aware, provenanced). Never run yt-dlp, whisper or
+ parakeet by hand, and never edit `transcripts/**` by hand: each file has one writer, named below.
+
+## Add a channel
+
+```sh
+pnpm ops create-channel --json '{"fields":{"name":"Example","handling":"youtube","url":"https://www.youtube.com/@example"}}'
+pnpm ops channel-config --json '{"slug":"example","patch":{"downloadFilterExclude":"#shorts"}}'
+pnpm ops channel-config --json '{"slug":"example","sites":[{"siteId":"jeralyzer"}]}'
+pnpm ops get channel example
+```
+
+- `handling`: `"youtube"` fetches the platform's captions; `"transcribe"` downloads audio and transcribes it here.
+ `fields` takes the New channel form's field names, the same as `channel-config`'s `patch`
+ ([RUNNING_IN_DOCKER.md](RUNNING_IN_DOCKER.md), "Driving the editor without a browser"). Every `config.json` key:
+ [CHANNEL.md](CHANNEL.md).
+- `sites` is the whole membership set; absent or `[]` keeps the channel on no site.
+- `"fetchPlaylist": true` on `create-channel` stores the playlist at once (the form's "Fetch playlist now"); a social channel's is `"fetchPostsNow": true`.
+- Rename or delete: `pnpm ops rename-channel --json '{"slug":"old","newSlug":"new"}'`,
+ `pnpm ops delete-channel --json '{"slug":"x","confirm":"x"}'` — both refused while the channel is busy.
+
+## Sync and download
+
+```sh
+pnpm ops sync --json '{"slug":"example"}' --wait
+pnpm ops metadata-scan --json '{"slug":"example"}' --wait
+pnpm ops download-missing --json '{"slug":"example"}' --wait
+pnpm ops retry-bucket --json '{"slug":"example","bucket":"<bucket>"}' --wait
+pnpm ops refresh-metadata --json '{"slug":"example","id":"<videoId>"}' --wait
+pnpm ops persist-videos --json '{"items":[{"slug":"example","id":"<videoId>"}],"dryRun":true}'
+```
+
+- The lanes do this unattended: `pnpm ops lane --json '{"lane":"download","enabled":true,"action":"start"}'`;
+ `"held": true` pauses dispatch and keeps the runner's place. Lanes: `transcription`, `download`, `digest`,
+ `backfill`.
+- A channel's priority: `pnpm ops channel-priority --json '{"slugs":["example"],"tier":"low"}'`
+ (`"operation"` pins one operation's tier).
+- Bucket names and sizes: `pnpm ops get channel example`.
+- Keep videos out of cleanup by title or description: `pnpm ops keep-videos --json '{"slug":"example","match":"interview","dryRun":true}'`.
+
+## Import from archive.org, Odysee, BitChute and the Wayback Machine
+
+```sh
+pnpm ops import-archive-org --json '{"slug":"example-archive","item":"<item>","match":"\\.mp4$","dryRun":true}' --wait
+pnpm ops import-archive-org --json '{"slug":"example-archive","item":"<item>","match":"\\.mp4$"}' --wait
+pnpm ops import-video --json '{"slug":"example","url":"https://www.bitchute.com/video/<id>/"}' --wait
+pnpm ops import-video --json '{"slug":"example","url":"https://odysee.com/@example:0/<video>:0"}' --wait
+pnpm ops import-video --json '{"slug":"example","url":"https://web.archive.org/web/<timestamp>/<original-url>"}' --wait
+```
+
+- `import-archive-org` takes one item's files (`files` exact, or `match` a regex), one at a time on archive.org's
+ queue; files already held are skipped. archive.org is fetched over BitTorrent when it can be, never by yt-dlp.
+- A one-off Odysee or BitChute import is paced like that platform's own downloads. A whole Odysee or BitChute
+ channel is a channel with that URL, then `sync`.
+- A Wayback capture is named by what it copies; `wayback.json` beside it records the capture.
+- Bring records up to the current rules, offline (dry run first):
+ `pnpm archilyzer archive-org refresh <slug> --dry-run`, `pnpm archilyzer wayback refresh <slug> --dry-run`,
+ `pnpm archilyzer feeds backfill-metadata <slug> --dry-run` (or `pnpm ops feed-metadata` on the editor).
+
+## Fetch and capture posts
+
+```sh
+pnpm ops create-channel --json '{"fields":{"name":"Example (X)","handling":"transcribe","url":"https://x.com/example"}}'
+pnpm ops fetch-posts --json '{"slug":"example-x"}' --wait
+pnpm ops fetch-posts --json '{"slug":"example-x","older":true}' --wait
+pnpm ops capture-posts --json '{"slug":"example-x","ids":["<postId>"]}' --wait
+```
+
+- An X, Bluesky or XenForo URL makes a social channel (its handle derived from the URL). `"older": true` walks back below the oldest
+ archived post and resumes from its saved place; `"full": true` re-walks the timeline.
+- `capture-posts` saves a screenshot and the attached media of named posts (and a linked X Article).
+- Forum pages saved from a browser: `pnpm archilyzer posts import-html <slug> <file-or-dir> --dry-run`.
+- Which archived posts were deleted at the source: `pnpm archilyzer posts check --slug <slug>`.
+
+## Transcribe
+
+```sh
+pnpm ops lane --json '{"lane":"transcription","enabled":true,"action":"start"}'
+pnpm ops transcribe-bucket --json '{"slug":"example"}' --wait
+pnpm ops transcribe --json '{"path":"/abs/clip.mp4","start":120,"end":150,"words":true}' --wait
+```
+
+- The transcription lane takes every channel's downloaded, untranscribed audio; `transcribe-bucket` runs one
+ channel's now.
+- `transcribe` runs one local file (or a window of it) through the corpus's own engine and model; the result JSON
+ (cues, text, and with `"words": true` the word timings parakeet keeps) is printed on stdout with `--wait`.
+ `"out"` writes it to a file outside the corpus.
+
+## Tag
+
+```sh
+pnpm ops get tags
+pnpm ops tags --json '{"op":"define","tag":{"id":"example-collab","label":"Collab"}}'
+pnpm ops tag-videos --json '{"tag":"example-collab","op":"add","videos":[{"slug":"example","id":"<videoId>"}]}'
+pnpm ops tag-videos --file ids.json
+```
+
+- `transcripts/tags.json` has one writer, and `tags` and `tag-videos` go through it, recording who asked
+ (`ARCHILYZER_AGENT`). `remove` unpins; `suppress` rejects a rule's hit.
+- A tag's `sites` limits the sites it exists on.
+- Read side: the MCP's `list_tags`, and `tags` on `search_transcripts` / `enumerate_matches`.
+
+## Prepare and export reports
+
+```sh
+pnpm archilyzer reports convert sweep <sweep.md> --out <report.json>
+pnpm ops fetch-windows --json '{"siteId":"example-site","dryRun":true}'
+pnpm ops fetch-windows --json '{"siteId":"example-site"}' --wait
+pnpm ops reports-prepare --json '{"siteId":"example-site"}' --wait
+pnpm ops reports-export --json '{"siteId":"example-site","formats":["html","md"]}' --wait
+```
+
+- A report is `transcripts/sites/<site>/reports/<id>/report.json` ([REPORT.md](REPORT.md),
+ [CITATIONS.md](CITATIONS.md)); `reports convert` makes one from a `/sweep` report, an `/ask` answer or a
+ report-to-video manifest.
+- `fetch-windows` with `siteId` fetches every clip window the site's published reports cite that the disk does not
+ hold, one paced job per platform; a re-run resumes.
+- `reports-prepare` cuts the cited clips and copies the cited post captures, then exports; it fails naming each
+ citation that lacks media. Offline: `pnpm archilyzer reports prepare <siteId>`,
+ `pnpm archilyzer reports export <siteId>`.
+- One cited moment's media, from an agent: the MCP's `fetch_clip`.
+
+## Publish
+
+```sh
+pnpm ops get publish
+pnpm ops publish --json '{"verb":"now"}' --wait
+pnpm ops publish --json '{"verb":"build","siteId":"example-site"}' --wait
+pnpm ops publish --json '{"verb":"deploy","siteId":"example-site","preview":"check"}' --wait
+pnpm archilyzer publish status
+```
+
+- Publishing is stages on one queue: update the index, build a site into its bundle, deploy the bundle, each
+ checked live. `now` runs what Publish now runs (the stale index, then each site's policy). Offline, under the same
+ lock: `pnpm archilyzer publish index`, `publish build <id>`, `publish deploy <id> --preview <branch>`.
+- Details, policies and the hub and homepage: [PUBLISH.md](PUBLISH.md).
+
+## Research with the MCP
+
+- `/ask` answers a question with citations in the conversation; `/sweep` writes a cited report to a file.
+- Search first, then pull only the cited seconds with `fetch_clip`. The editor fetches only for a channel it
+ already archives.
+- Tools and their arguments: [mcp/README.md](mcp/README.md).
diff --git a/PLAN.md b/PLAN.md
@@ -688,6 +688,6 @@ The phases above are the AI track. Later work is planned and recorded one releas
| 19 | agents run the archive: ops/CLI/MCP (Track A), machine safety and tooling (Track B), OPERATING.md and the docs (Track C) | [`plans/release-19.md`](plans/release-19.md) | in flight |
| 20 | the data model and the index: recorded dates, Twitch ids, the caption-track bug closed | [`plans/release-20.md`](plans/release-20.md) | planned, after 19 |
| 21 | playable archives: local media attached to held videos (clips cut locally), per-video torrents played in the page, a home seeder of last resort behind a VPN; pilot TISM on jeralyzer-private | [`plans/release-21.md`](plans/release-21.md) | planned 2026-10-09 |
-| 22 | — | — | not yet planned |
+| 22 | one article, two shapes: slides from the same report.json (authoring fields + a derived default), a reader switch Article / Slides / Overview, the isometric overview pairing sections with slides, `slides.html`/`slides.pdf` exports | [`plans/release-22.md`](plans/release-22.md) | planned 2026-10-09 |
Work that landed on `main` without a plan of its own: [`plans/landed-2026-10.md`](plans/landed-2026-10.md).
diff --git a/README.md b/README.md
@@ -640,6 +640,8 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) to work on the code.
| [SETUP.md](SETUP.md) | Full per-OS install, transcription backends. |
| [ENVIRONMENT.md](ENVIRONMENT.md) | Every environment variable, by audience (generated). |
| [CONTRIBUTING.md](CONTRIBUTING.md) | Workspace layout, tests, the `archilyzer` CLI, internals. |
+| [OPERATING.md](OPERATING.md) | Running an archive from a shell or an agent: recipes over `pnpm ops`, `archilyzer` and the MCP. |
+| [COMMANDS.md](COMMANDS.md) | Every `archilyzer` command and `pnpm ops` action (generated). |
| [RUNNING_IN_DOCKER.md](RUNNING_IN_DOCKER.md) | `docker compose up` for the whole stack: exposure model, auth, GPU. |
| [PUBLISH.md](PUBLISH.md) | Building and deploying sites: Pages + R2, cost-abuse protection, parallel builds in containers. |
| [SCHEDULED_SYNC.md](SCHEDULED_SYNC.md) | Unattended per-channel syncing. |
diff --git a/RUNNING_IN_DOCKER.md b/RUNNING_IN_DOCKER.md
@@ -413,6 +413,9 @@ pnpm ops get channels # every channel, its kind and its sites
pnpm ops list # every action name
```
+These are examples. Every action and getter, with its body, is in [COMMANDS.md](COMMANDS.md) (generated from
+`pnpm ops --help` and `pnpm archilyzer --help`); recipes that chain them are in [OPERATING.md](OPERATING.md).
+
Four things to know before you script against it:
- **A job-starting action returns a `jobId` and does not stream.** The job may
diff --git a/WORKTREES.md b/WORKTREES.md
@@ -141,12 +141,71 @@ audio checks run at production pace, which reads as real failures.
| Variable | Effect |
|---|---|
-| `E2E_QUEUE=0` | Skip the queue entirely (the port preflight still runs) |
+| `E2E_QUEUE=0` | Skip the queue and the heavy slot (the port preflight and the memory floor still run) |
| `E2E_PORT_CHECK=0` | Skip the port preflight, reusing whatever servers are up (a hand-started editor test server needs the config's `E2E_SERVER_ENV`, above) |
| `E2E_QUEUE_TIMEOUT=<seconds>` | Give up waiting after N seconds (default: wait forever) |
Verify the queue with `pnpm test:scripts`.
+## The heavy slot (`pnpm heavy`)
+
+An e2e suite, a `next build` and a video render each want several GB, and two of them at
+once is how this machine OOMed (taking the desktop session with it). So all three go
+through ONE more machine-global lock, the **heavy slot**, and start only once
+`/proc/meminfo`'s MemAvailable is at least a floor (6000 MB):
+
+```sh
+pnpm heavy -- <cmd…> # any heavy command, by hand
+pnpm heavy -- node umtool/report-to-video/build-video.mjs <manifest> … # a render
+```
+
+Who takes it, and in what order:
+
+- **Every e2e entry point** (the same ones the e2e queue covers): the heavy slot FIRST,
+ then the e2e queue. One order everywhere, so nothing can deadlock — and a heavy command
+ that starts another (`pnpm heavy -- pnpm e2e`, a build stage run by an e2e suite's
+ editor) passes straight through (`HEAVY_HELD`), as a nested e2e run always has.
+- **The publish stages' `next build`** — a site's, the hub's, the homepage's
+ (`common/publish/build.ts`, `heavyGated`). The wait shows in the stage's log, and a
+ Cancel still stops the build (the gate forwards SIGTERM). In a docker-runner container
+ the slot is the container's own; the floor still reads the host's memory, which
+ throttles a fan-out when the host runs low.
+- **A render**, by hand, as above.
+
+The slot is taken before the floor is waited for, so nobody slips in while the holder
+waits for memory. A waiter is told what it waits behind:
+
+```
+queue-lock: waiting for the heavy slot — held by feature-x (feature-x, pid 31337) for 2m10s: playwright test
+heavy: waiting for memory — 4210 MB available, the floor is 6000 MB
+```
+
+The lock is `<git-common-dir>/heavy-queue.lock`, released by the kernel like the e2e one.
+A machine whose MemTotal is under the floor is told so and runs; with no usable `flock` the
+gate warns and runs on the floor alone — it is a safety net, not a correctness lock.
+
+| Variable | Effect |
+|---|---|
+| `HEAVY=0` | Skip the heavy slot AND the memory floor |
+| `HEAVY_MIN_FREE_MB=<MB>` | Move the floor (`0` turns it off) |
+| `HEAVY_TIMEOUT=<seconds>` | Give up waiting (slot, then floor) after N seconds; an e2e run uses `E2E_QUEUE_TIMEOUT` |
+
+### A render and the transcription lane
+
+A render competes with the transcription lane for memory and CPU, and the lane is not a heavy
+slot holder.
+Hold the lane for the render's duration, and release it whatever the render's outcome:
+
+```sh
+pnpm ops lane --json '{"lane":"transcription","held":true}'
+pnpm heavy -- node umtool/report-to-video/build-video.mjs <manifest> … ; \
+ pnpm ops lane --json '{"lane":"transcription","held":false}'
+```
+
+A hold stops new dispatches, not a transcription already running; and the release
+resumes the lane even if someone else held it for another reason — check `/operations`
+first.
+
## Data directories
By default each worktree is **fully isolated**: `common/lib/paths.ts` resolves
diff --git a/common/bin/archilyzer.ts b/common/bin/archilyzer.ts
@@ -284,6 +284,61 @@ export const COMMANDS: Command[] = [
},
},
{
+ path: ["reports", "check"],
+ usage:
+ "<id> [--reports <a,b>] [--allow-missing-media] what compose would say about the site's reports, with no build: each report validated, every cited quote checked against its record, every cited moment's prepared media present and current, the report video under the publish limit (exit 1 with the list; --reports checks those, drafts included; default id: SITE_ID)",
+ flags: { reports: "string", "allow-missing-media": "boolean" },
+ maxPositionals: 1,
+ run: async ({ positionals, flags, env }) => {
+ const siteId = siteIdFrom(positionals, env, "reports check");
+ if (!siteId) return 2;
+ const reports =
+ typeof flags.reports === "string"
+ ? flags.reports.split(",").map((r) => r.trim()).filter(Boolean)
+ : undefined;
+ return (await import("./reports-check")).checkMain({
+ siteId,
+ ...(reports ? { reports } : {}),
+ allowMissingMedia: flags["allow-missing-media"] === true,
+ });
+ },
+ },
+ {
+ path: ["reports", "verify-quotes"],
+ usage:
+ "<report.json> [--json] every video, audio and post quote of one report against its record with compose's own check: the best score and track, and the en-orig track's score where the record has one — a quote that matches a served `en` rewrite and not en-orig is reported (exit 1 when any quote drifted)",
+ flags: { json: "boolean" },
+ maxPositionals: 1,
+ run: async ({ positionals, flags }) => {
+ const [file] = positionals;
+ if (!file) {
+ console.error("reports verify-quotes: give <report.json>");
+ return 2;
+ }
+ return (await import("./reports-check")).verifyQuotesMain({ file, json: flags.json === true });
+ },
+ },
+ {
+ path: ["reports", "attach-video"],
+ usage:
+ "<report.json> <video> [--poster <image>] [--caption <line>] the report's video: remuxed (an H.264 mp4 under the limit) or encoded to fit the 24 MiB publish limit, written beside report.json as video.mp4 with a poster (given, kept, or a frame of the video), and `video` set in report.json",
+ flags: { poster: "string", caption: "string" },
+ maxPositionals: 2,
+ run: async ({ positionals, flags }) => {
+ const [reportFile, video] = positionals;
+ if (!reportFile || !video) {
+ console.error("reports attach-video: give <report.json> <video>");
+ return 2;
+ }
+ return (await import("./reports-attach-video")).attachVideoMain({
+ reportFile,
+ video,
+ ...(typeof flags.poster === "string" ? { poster: flags.poster } : {}),
+ ...(typeof flags.caption === "string" ? { caption: flags.caption } : {}),
+ });
+ },
+ },
+ {
path: ["reports", "convert"],
usage:
"<sweep|ask|manifest> <in> --out <report.json> [--channels-dir <dir>] [--id <id>] [--title <title>] a /sweep report (markdown), an /ask answer or a report-to-video manifest as a report.json, written only when it validates (--channels-dir: widen spans from the cues, find posts' channels)",
@@ -618,6 +673,13 @@ export const COMMANDS: Command[] = [
(await import("./file-schemas-docs")).main({ check: flags.check === true }),
},
{
+ path: ["docs", "cli"],
+ usage: "[--check] write COMMANDS.md from this command table and `pnpm ops --help`",
+ flags: { check: "boolean" },
+ run: async ({ flags }) =>
+ (await import("./cli-docs")).main({ check: flags.check === true }),
+ },
+ {
path: ["settings", "example"],
usage: "[--check] write settings.json.example + SETTINGS.md from the schema",
flags: { check: "boolean" },
diff --git a/common/bin/cli-docs.test.ts b/common/bin/cli-docs.test.ts
@@ -0,0 +1,140 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import { test } from "node:test";
+import { fileURLToPath } from "node:url";
+import {
+ COMMANDS_FILE,
+ generateCommandsMarkdown,
+ mdCell,
+ opsActions,
+ opsExamples,
+ opsParagraphs,
+ RECIPES_FILE,
+ renderCommandsMarkdown,
+ splitUsage,
+ unknownCommandsIn,
+ unknownRecipeCommands,
+} from "./cli-docs";
+
+const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
+
+const OPS_USAGE = [
+ "Usage: pnpm ops <action> [--json '<body>'] [--wait]",
+ " pnpm ops list",
+ "",
+ "Actions: sync, build-site, deploy-site, lane",
+ "",
+ "--wait follows the job's log.",
+ "",
+ 'build-site and deploy-site take "siteId" or',
+ ' "siteIds" (a list).',
+ "",
+ 'deploy-site ships a built bundle: {"siteId"}. See `a | b`.',
+ "",
+ "Env: WORKER_TOKEN",
+].join("\n");
+
+test("splitUsage splits at the first double space; a usage with none is all text", () => {
+ assert.deepEqual(splitUsage("<id> [--force] build it now"), {
+ args: "<id> [--force]",
+ text: "build it now",
+ });
+ assert.deepEqual(splitUsage("rebuild the index"), { args: "", text: "rebuild the index" });
+});
+
+test("mdCell escapes markup outside code and only the pipe inside it", () => {
+ assert.equal(mdCell("a <id> | *b* _c_ [x]"), "a <id> \\| \\*b\\* \\_c\\_ \\[x\\]");
+ assert.equal(mdCell("run `x | y <z>` then"), "run `x \\| y <z>` then");
+});
+
+test("opsActions reads the Actions line, and refuses a usage without one", () => {
+ assert.deepEqual(opsActions(OPS_USAGE), ["sync", "build-site", "deploy-site", "lane"]);
+ assert.throws(() => opsActions("Usage: nothing"), /no `Actions:` line/);
+});
+
+test("opsParagraphs attributes a paragraph to every action it opens with", () => {
+ const ps = opsParagraphs(OPS_USAGE, opsActions(OPS_USAGE));
+ assert.deepEqual(
+ ps.map((p) => p.actions),
+ [[], [], ["build-site", "deploy-site"], ["deploy-site"], []],
+ );
+ assert.ok(!ps.some((p) => p.text.startsWith("Actions: ")));
+});
+
+test("opsExamples takes header lines of known actions only, padding collapsed", () => {
+ const source = [
+ "// pnpm ops sync --json '{\"slug\":\"x\"}' --wait",
+ "// pnpm ops get channel x",
+ "// pnpm ops lane",
+ "// pnpm ops sync --json '{\"slug\":\"y\"}'",
+ "const x = 1; // pnpm ops sync not a header line",
+ ].join("\n");
+ const ex = opsExamples(source, ["sync", "lane"]);
+ assert.deepEqual(ex.get("sync"), [
+ "pnpm ops sync --json '{\"slug\":\"x\"}' --wait",
+ "pnpm ops sync --json '{\"slug\":\"y\"}'",
+ ]);
+ assert.deepEqual(ex.get("lane"), ["pnpm ops lane"]);
+ assert.equal(ex.has("get"), false);
+});
+
+test("the rendered reference lists every command and every action exactly once as a row head", () => {
+ const md = renderCommandsMarkdown(
+ [
+ { path: ["index"], usage: "rebuild the LMDB transcript index" },
+ { path: ["publish", "build"], usage: "<id|all> [--force] build a site" },
+ { path: ["mcp"], usage: "[--local <dir>] start the MCP server", passthrough: true },
+ ],
+ OPS_USAGE,
+ new Map([["deploy-site", ["pnpm ops deploy-site --json '{}'"]]]),
+ );
+ assert.match(md, /^# Command reference\n\n<!-- GENERATED by common\/bin\/cli-docs\.ts/);
+ assert.match(md, /\| `archilyzer index` \| \| rebuild the LMDB transcript index \|/);
+ assert.match(md, /\| `archilyzer publish build` \| `<id\\\|all> \[--force\]` \| build a site \|/);
+ assert.match(md, /\| `archilyzer mcp` \| .* \*\(passthrough\)\* \|/);
+ // An action with no paragraph still has its row.
+ assert.match(md, /\| `sync` \| — \| \|/);
+ assert.match(md, /\| `lane` \| — \| \|/);
+ // The shared paragraph is one row naming both; deploy-site's example sits on
+ // that first row, not on its own paragraph's.
+ assert.match(
+ md,
+ /\| `build-site`, `deploy-site` \| build-site and deploy-site take "siteId" or "siteIds" \(a list\)\. \| `pnpm ops deploy-site --json '\{\}'` \|/,
+ );
+ assert.match(md, /\| `deploy-site` \| deploy-site ships a built bundle: \{"siteId"\}\. See `a \\\| b`\. \| \|/);
+ // The general paragraphs are printed as they stand, in a text block.
+ assert.match(md, /```text\nUsage: pnpm ops <action>[^]*--wait follows the job's log\.\n\nEnv: WORKER_TOKEN\n```\n$/);
+ assert.doesNotMatch(md, /Actions: sync/);
+});
+
+test("unknownCommandsIn names what no action, getter or command answers to", () => {
+ const cli = [{ path: ["publish", "build"], usage: "" }, { path: ["doctor"], usage: "" }];
+ const md = [
+ "`pnpm ops <action>` and `pnpm archilyzer <command>` are placeholders.",
+ "pnpm ops sync --json '{}' --wait",
+ "pnpm ops list",
+ "pnpm ops get channel x and pnpm ops get bogus",
+ "pnpm ops frobnicate --wait",
+ "`pnpm archilyzer publish build <id>`, `pnpm archilyzer doctor`.",
+ "pnpm archilyzer publish destroy x",
+ ].join("\n");
+ const usage = OPS_USAGE.replace("pnpm ops list", "pnpm ops list\n pnpm ops get channel <slug>");
+ assert.deepEqual(unknownCommandsIn(md, cli, usage), [
+ "pnpm ops get bogus",
+ "pnpm ops frobnicate",
+ "pnpm archilyzer publish destroy x",
+ ]);
+});
+
+test(`every command ${RECIPES_FILE} names exists`, async () => {
+ assert.deepEqual(await unknownRecipeCommands(), []);
+});
+
+test(`${COMMANDS_FILE} is what the command table and pnpm ops --help generate`, async () => {
+ assert.equal(
+ readFileSync(path.join(REPO, COMMANDS_FILE), "utf8"),
+ await generateCommandsMarkdown(),
+ `${COMMANDS_FILE} is stale: run \`archilyzer docs cli\``,
+ );
+});
diff --git a/common/bin/cli-docs.ts b/common/bin/cli-docs.ts
@@ -0,0 +1,274 @@
+// WRITE COMMANDS.md FROM THE TWO COMMAND SURFACES' OWN HELP.
+//
+// archilyzer docs cli [--check]
+//
+// The `archilyzer` rows come from the command table (`COMMANDS`, archilyzer.ts):
+// each row's path and its one-line usage. The `pnpm ops` rows come from
+// `usage()` in scripts/archilyzer-ops.mjs — the text `pnpm ops --help` prints —
+// split into paragraphs: a paragraph that opens with action names documents
+// those actions; every other paragraph (the usage lines, --wait, preview,
+// the env) is printed as it stands. Each action also gets the examples the
+// script's header comment gives it (`// pnpm ops <action> …` lines). An
+// action with neither is still listed, so the reference never omits one.
+//
+// `--check` writes nothing and returns 1 when the committed file differs from
+// what the two sources generate (the same claim cli-docs.test.ts makes). Both
+// modes then check OPERATING.md: every `pnpm ops …` / `pnpm archilyzer …` its
+// recipes name must exist. The sibling of env-docs.ts (ENVIRONMENT.md) and
+// file-schemas-docs.ts.
+
+import { readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { runIfEntryPoint } from "./_cli";
+
+const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
+
+export const COMMANDS_FILE = "COMMANDS.md";
+const OPS_SCRIPT = path.join("scripts", "archilyzer-ops.mjs");
+
+// What the renderer needs of a CLI row — structural, so the tests never import
+// the table.
+export type CliRow = { path: readonly string[]; usage: string; passthrough?: boolean };
+
+/** A usage line split at its first double space: the arguments, then the text. */
+export function splitUsage(usage: string): { args: string; text: string } {
+ const at = usage.indexOf(" ");
+ if (at < 0) return { args: "", text: usage.trim() };
+ return { args: usage.slice(0, at).trim(), text: usage.slice(at).trim() };
+}
+
+/**
+ * Plain help text as one Markdown table cell: backtick spans are kept as code,
+ * and everything else is escaped so `<id>`, `*`, `_` and `[x]` print as typed.
+ * A `|` is escaped everywhere, code included (GFM reads it as a cell edge).
+ */
+export function mdCell(text: string): string {
+ return text
+ .split(/(`[^`]*`)/)
+ .map((part, i) =>
+ i % 2 === 1
+ ? part.replace(/\|/g, "\\|")
+ : part
+ .replace(/\\/g, "\\\\")
+ .replace(/\|/g, "\\|")
+ .replace(/</g, "<")
+ .replace(/>/g, ">")
+ .replace(/([*_[\]])/g, "\\$1"),
+ )
+ .join("");
+}
+
+function codeCell(text: string): string {
+ return text ? `\`${text.replace(/\|/g, "\\|")}\`` : "";
+}
+
+/** The action list `usage()` prints on its `Actions: a, b, c` line. */
+export function opsActions(opsUsage: string): string[] {
+ const line = opsUsage.split("\n").find((l) => l.startsWith("Actions: "));
+ if (!line) throw new Error("pnpm ops usage: no `Actions:` line");
+ return line
+ .slice("Actions: ".length)
+ .split(",")
+ .map((a) => a.trim())
+ .filter(Boolean);
+}
+
+export type OpsParagraph = { actions: string[]; text: string };
+
+/**
+ * The usage text's paragraphs, each with the actions it opens with
+ * ("build-site, build-deploy and deploy-site all take …" → three). A paragraph
+ * that opens with no action name has `actions: []`. The `Actions:` line itself
+ * is left out: the table replaces it.
+ */
+export function opsParagraphs(opsUsage: string, actions: readonly string[]): OpsParagraph[] {
+ const known = new Set(actions);
+ const out: OpsParagraph[] = [];
+ for (const raw of opsUsage.split(/\n\s*\n/)) {
+ const text = raw.replace(/\s+$/, "");
+ if (!text.trim() || text.startsWith("Actions: ")) continue;
+ const opened: string[] = [];
+ for (const word of text.trim().split(/[\s,]+/)) {
+ if (known.has(word)) opened.push(word);
+ else if (word === "and" && opened.length > 0) continue;
+ else break;
+ }
+ out.push({ actions: opened, text });
+ }
+ return out;
+}
+
+/**
+ * The examples in the ops script's header comment, per action: every
+ * `// pnpm ops <action> …` line whose action is a known one, its padding
+ * collapsed. (`get` and `list` lines are not actions; the usage block has them.)
+ */
+export function opsExamples(source: string, actions: readonly string[]): Map<string, string[]> {
+ const known = new Set(actions);
+ const out = new Map<string, string[]>();
+ for (const line of source.split("\n")) {
+ const m = /^\/\/\s+pnpm ops ([a-z][a-z-]*)(\s+.*)?$/.exec(line);
+ if (!m || !known.has(m[1])) continue;
+ const rest = m[2] ? ` ${m[2].trim().replace(/\s{2,}/g, " ")}` : "";
+ out.set(m[1], [...(out.get(m[1]) ?? []), `pnpm ops ${m[1]}${rest}`]);
+ }
+ return out;
+}
+
+const oneLine = (s: string) => s.replace(/\s+/g, " ").trim();
+
+export function renderCommandsMarkdown(
+ cli: readonly CliRow[],
+ opsUsage: string,
+ examples: ReadonlyMap<string, readonly string[]> = new Map(),
+): string {
+ const actions = opsActions(opsUsage);
+ const paragraphs = opsParagraphs(opsUsage, actions);
+ const lines: string[] = [
+ "# Command reference",
+ "",
+ "<!-- GENERATED by common/bin/cli-docs.ts from the `archilyzer` command table (common/bin/archilyzer.ts) and `pnpm ops --help` (scripts/archilyzer-ops.mjs) — do not edit by hand. Regenerate: `pnpm archilyzer docs cli`. -->",
+ "",
+ "Every `archilyzer` command and every `pnpm ops` action, from their own help. Recipes that chain them: [OPERATING.md](OPERATING.md).",
+ "",
+ "## `pnpm archilyzer`",
+ "",
+ "Run from the repo root (in the container: `docker compose exec editor pnpm archilyzer …`). `--help` after a command prints its line. A command marked *passthrough* parses its own flags.",
+ "",
+ "| command | arguments | what it does |",
+ "|---|---|---|",
+ ];
+ for (const row of cli) {
+ const { args, text } = splitUsage(row.usage);
+ const note = row.passthrough ? " *(passthrough)*" : "";
+ lines.push(
+ `| \`archilyzer ${row.path.join(" ")}\` | ${codeCell(args)} | ${mdCell(oneLine(text))}${note} |`,
+ );
+ }
+ lines.push(
+ "",
+ "## `pnpm ops`",
+ "",
+ "Drives a running editor over HTTP (`/api/ops/*`, the same actions its pages run), gated by `WORKER_TOKEN`. A body is `--json '<object>'` or `--file <path>`; `--wait` follows a job to its end.",
+ "",
+ "| action | what it does | example |",
+ "|---|---|---|",
+ );
+ // One row per help paragraph that opens with actions, in the order of their
+ // first action; an action no paragraph opens with gets a row of its own. An
+ // action's examples go on the first row that names it.
+ type Row = { actions: string[]; text: string; examples: string[] };
+ const rows: Row[] = [];
+ const emitted = new Set<OpsParagraph>();
+ for (const action of actions) {
+ const mine = paragraphs.filter((p) => p.actions.includes(action));
+ if (mine.length === 0) rows.push({ actions: [action], text: "", examples: [] });
+ for (const p of mine) {
+ if (emitted.has(p)) continue;
+ emitted.add(p);
+ rows.push({ actions: p.actions, text: oneLine(p.text), examples: [] });
+ }
+ }
+ for (const action of actions) {
+ const row = rows.find((r) => r.actions.includes(action));
+ row?.examples.push(...(examples.get(action) ?? []));
+ }
+ for (const r of rows) {
+ const names = r.actions.map((a) => `\`${a}\``).join(", ");
+ const ex = r.examples.map(codeCell).join("<br>");
+ lines.push(`| ${names} | ${r.text ? mdCell(r.text) : "—"} | ${ex} |`);
+ }
+ lines.push("", "### Usage, flags and environment", "", "```text");
+ for (const p of paragraphs) {
+ if (p.actions.length === 0) lines.push(p.text, "");
+ }
+ if (lines[lines.length - 1] === "") lines.pop();
+ lines.push("```", "");
+ return lines.join("\n");
+}
+
+/**
+ * Every `pnpm ops …` and `pnpm archilyzer …` a document names that is not a
+ * real action, getter or command — what keeps OPERATING.md's recipes runnable.
+ * A placeholder (`pnpm ops <action>`) is not a name.
+ */
+export function unknownCommandsIn(
+ markdown: string,
+ cli: readonly CliRow[],
+ opsUsage: string,
+): string[] {
+ const actions = new Set(opsActions(opsUsage));
+ const nouns = new Set([...opsUsage.matchAll(/pnpm ops get ([a-z][a-z-]*)/g)].map((m) => m[1]));
+ const bad = new Set<string>();
+ for (const m of markdown.matchAll(/pnpm ops ([^\s`'"]+)(?:[ \t]+([^\s`'"]+))?/g)) {
+ const [, word, next] = m;
+ if (word.startsWith("<") || word === "list") continue;
+ if (word === "get") {
+ if (!next || !nouns.has(next)) bad.add(`pnpm ops get ${next ?? ""}`.trim());
+ continue;
+ }
+ if (!actions.has(word)) bad.add(`pnpm ops ${word}`);
+ }
+ for (const m of markdown.matchAll(/pnpm archilyzer((?:[ \t]+[a-z][a-z-]*)+)/g)) {
+ const words = m[1].trim().split(/\s+/);
+ if (!cli.some((r) => r.path.every((w, i) => words[i] === w))) {
+ bad.add(`pnpm archilyzer ${words.join(" ")}`);
+ }
+ }
+ return [...bad];
+}
+
+// The recipes the reference backs.
+export const RECIPES_FILE = "OPERATING.md";
+
+/** `usage()` of scripts/archilyzer-ops.mjs — what `pnpm ops --help` prints. */
+export async function loadOpsUsage(repo = REPO): Promise<string> {
+ const mod = (await import(pathToFileURL(path.join(repo, OPS_SCRIPT)).href)) as {
+ usage?: () => string;
+ };
+ if (typeof mod.usage !== "function") {
+ throw new Error(`${OPS_SCRIPT} exports no usage()`);
+ }
+ return mod.usage();
+}
+
+export async function generateCommandsMarkdown(repo = REPO): Promise<string> {
+ const { COMMANDS } = await import("./archilyzer");
+ const opsUsage = await loadOpsUsage(repo);
+ const source = await readFile(path.join(repo, OPS_SCRIPT), "utf8");
+ return renderCommandsMarkdown(COMMANDS, opsUsage, opsExamples(source, opsActions(opsUsage)));
+}
+
+/** OPERATING.md's names that no command answers to (see unknownCommandsIn). */
+export async function unknownRecipeCommands(repo = REPO): Promise<string[]> {
+ const { COMMANDS } = await import("./archilyzer");
+ const recipes = await readFile(path.join(repo, RECIPES_FILE), "utf8");
+ return unknownCommandsIn(recipes, COMMANDS, await loadOpsUsage(repo));
+}
+
+// Writes COMMANDS.md (or, with `check`, compares it), then checks that every
+// command OPERATING.md names exists. 1 when either is wrong.
+export async function main(opts: { check?: boolean } = {}): Promise<number> {
+ const file = path.join(REPO, COMMANDS_FILE);
+ const want = await generateCommandsMarkdown();
+ let code = 0;
+ if (opts.check) {
+ const have = await readFile(file, "utf8").catch(() => "");
+ if (have !== want) {
+ console.error(`${COMMANDS_FILE} is stale — regenerate it with \`archilyzer docs cli\``);
+ code = 1;
+ }
+ } else {
+ await writeFile(file, want);
+ console.log(`wrote ${COMMANDS_FILE}`);
+ }
+ const unknown = await unknownRecipeCommands();
+ if (unknown.length > 0) {
+ console.error(`${RECIPES_FILE} names commands that do not exist: ${unknown.join("; ")}`);
+ code = 1;
+ }
+ return code;
+}
+
+runIfEntryPoint(import.meta.url, () => main({ check: process.argv.includes("--check") }));
diff --git a/common/bin/reports-attach-video.ts b/common/bin/reports-attach-video.ts
@@ -0,0 +1,49 @@
+// `archilyzer reports attach-video <report.json> <video> [--poster <image>]
+// [--caption <line>]` — the video as the report's: encoded or remuxed to fit
+// the publish limit, written beside report.json as video.mp4 with a poster,
+// and `video` set in report.json (publish/reportVideo.ts).
+//
+// Exit 0 when attached, 1 when it could not be (the reason printed: too long
+// to fit, not a report, ffmpeg's error), 2 for usage.
+
+import { getPaths } from "../lib/paths";
+import { AttachVideoError, attachReportVideo } from "../publish/reportVideo";
+
+type Out = { log: (s: string) => void; error: (s: string) => void };
+
+export async function attachVideoMain(
+ opts: {
+ reportFile: string;
+ video: string;
+ poster?: string;
+ caption?: string;
+ limitBytes?: number;
+ ffmpegBin?: string;
+ ffprobeBin?: string;
+ },
+ out: Out = console,
+): Promise<number> {
+ const paths = getPaths();
+ try {
+ const done = await attachReportVideo({
+ reportFile: opts.reportFile,
+ video: opts.video,
+ ...(opts.poster !== undefined ? { poster: opts.poster } : {}),
+ ...(opts.caption !== undefined ? { caption: opts.caption } : {}),
+ ...(opts.limitBytes !== undefined ? { limitBytes: opts.limitBytes } : {}),
+ ffmpegBin: opts.ffmpegBin ?? paths.ffmpegBin,
+ ffprobeBin: opts.ffprobeBin ?? paths.ffprobeBin,
+ onLog: out.log,
+ });
+ out.log(
+ `reports attach-video: ${opts.reportFile} video = ${JSON.stringify(done.video)} ` +
+ `(${done.mode === "remux" ? "remuxed" : `encoded, ${done.attempts} pass(es)`}, ${(done.bytes / (1024 * 1024)).toFixed(2)} MiB)`,
+ );
+ return 0;
+ } catch (err) {
+ const e = err as Error & { stderr?: string };
+ const why = err instanceof AttachVideoError ? e.message : (e.stderr || e.message).trim().split("\n").slice(-3).join(" ");
+ out.error(`reports attach-video: ${why}`);
+ return 1;
+ }
+}
diff --git a/common/bin/reports-check.test.ts b/common/bin/reports-check.test.ts
@@ -0,0 +1,286 @@
+// `archilyzer reports check`, `reports verify-quotes` and `reports
+// attach-video`, over a temp corpus.
+//
+// One channel with three records: one whose `en` track has the quote, one
+// whose served `en` track is a REWRITE (the quote matches it) while
+// `en-orig` has the words as spoken, and one the quote does not match at all.
+// A site publishing a report on the first; drafts on the others.
+//
+// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test bin/reports-check.test.ts
+
+import { after, test } from "node:test";
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
+const ROOT = mkdtempSync(path.join(tmpdir(), "reports-check-"));
+Object.assign(process.env, {
+ TRANSCRIPTS_DIR: path.join(ROOT, "transcripts"),
+ SITES_DIR: path.join(ROOT, "transcripts", "sites"),
+ SETTINGS_FILE: path.join(ROOT, "settings.json"),
+ EXPORT_PUBLIC_DIR: path.join(ROOT, "public"),
+ EXPORT_INDEX_DIR: path.join(ROOT, ".export-index"),
+ EXPORT_BUILDS_DIR: path.join(ROOT, ".export-builds"),
+});
+after(() => rmSync(ROOT, { recursive: true, force: true }));
+
+const { getPaths } = await import("../lib/paths");
+const { checkMain, verifyQuotesMain } = await import("./reports-check");
+const { attachVideoMain } = await import("./reports-attach-video");
+const { encodePlan, encodeArgs } = await import("../publish/reportVideo");
+
+const paths = getPaths();
+const CHAN = "chan";
+
+const writeJson = (file: string, value: unknown) => {
+ mkdirSync(path.dirname(file), { recursive: true });
+ writeFileSync(file, JSON.stringify(value, null, 2));
+};
+const writeText = (file: string, text: string) => {
+ mkdirSync(path.dirname(file), { recursive: true });
+ writeFileSync(file, text);
+};
+const ts = (s: number) => new Date(s * 1000).toISOString().slice(11, 23);
+const vtt = (cues: [number, number, string][]) =>
+ "WEBVTT\nKind: captions\nLanguage: en\n\n" +
+ cues.map(([a, b, text]) => `${ts(a)} --> ${ts(b)} align:start position:0%\n${text}<${ts(a)}><c></c>\n`).join("\n");
+
+function record(id: string, tracks: Record<string, [number, number, string][]>) {
+ const dir = path.join(paths.channelsDir, CHAN, "data", id);
+ writeJson(path.join(dir, "metadata.info.json"), {
+ id,
+ title: `Stream ${id}`,
+ channel: CHAN,
+ upload_date: "20260110",
+ duration: 600,
+ webpage_url: `https://www.youtube.com/watch?v=${id}`,
+ extractor_key: "Youtube",
+ });
+ for (const [track, cues] of Object.entries(tracks)) writeText(path.join(dir, `transcript.${track}.vtt`), vtt(cues));
+}
+
+const span = (id: string, quote: string) => ({ kind: "video", channel: CHAN, id, start: 10, end: 20, quote });
+
+function report(id: string, citations: Record<string, unknown>) {
+ return {
+ format: "archilyzer-report",
+ version: 1,
+ id,
+ kind: "sweep",
+ title: `Report ${id}`,
+ citations,
+ sections: [
+ {
+ id: "s",
+ title: "S",
+ body: Object.keys(citations)
+ .map((c) => `Said [this](cite:${c}).`)
+ .join(" "),
+ },
+ ],
+ };
+}
+
+const reportFile = (siteId: string, id: string) => path.join(paths.sitesDir, siteId, "reports", id, "report.json");
+
+writeJson(paths.settingsFile, {});
+writeJson(path.join(paths.channelsDir, CHAN, "config.json"), {
+ handling: "youtube",
+ name: "Chan",
+ url: "https://www.youtube.com/@chan/videos",
+});
+record("good1", { en: [[10, 20, "The bridge opened in the spring, I was there for it."]] });
+record("rewr1", {
+ en: [[10, 20, "We will never agree to the deal on the table."]],
+ "en-orig": [[10, 20, "Honestly we might take whatever they offer us now."]],
+});
+record("miss1", { en: [[10, 20, "Something else entirely about the weather today."]] });
+writeJson(path.join(paths.sitesDir, "demo", "site.json"), {
+ siteId: "demo",
+ siteTitle: "Demo",
+ siteDescription: "fixture",
+ headerTitle: "demo",
+ homeTagline: "",
+ socialLinks: [],
+ groups: [{ id: "default", name: "All channels", selectedByDefault: true }],
+ defaultGroupId: "default",
+ channels: [{ slug: CHAN, groupId: "default" }],
+ siteUrl: "https://demo.example.test",
+ archives: false,
+ reports: ["pub1"],
+});
+writeJson(reportFile("demo", "pub1"), report("pub1", { c1: span("good1", "The bridge opened in the spring, I was there for it.") }));
+writeJson(
+ reportFile("demo", "draft1"),
+ report("draft1", { c1: span("miss1", "The bridge opened in the spring, I was there for it.") }),
+);
+writeJson(
+ reportFile("demo", "mixed"),
+ report("mixed", {
+ c1: span("good1", "The bridge opened in the spring, I was there for it."),
+ c2: span("rewr1", "We will never agree to the deal on the table."),
+ c3: span("miss1", "The bridge opened in the spring, I was there for it."),
+ c4: span("ghost1", "nothing here"),
+ }),
+);
+
+function capture() {
+ const lines: string[] = [];
+ return {
+ lines,
+ text: () => lines.join("\n"),
+ out: { log: (s: string) => lines.push(s), error: (s: string) => lines.push(s) },
+ };
+}
+
+test("check: compose's problems with no build — the published report lacks its prepared media", async () => {
+ const c = capture();
+ assert.equal(await checkMain({ siteId: "demo", paths }, c.out), 1);
+ assert.match(c.text(), /compose would fail/);
+ assert.match(c.text(), /missing-media: chan\/good1\/10\.00-20\.00: no prepared media/);
+ // Nothing was composed.
+ assert.throws(() => statSync(path.join(paths.exportPublicDir, "reports")));
+});
+
+test("check --allow-missing-media passes the text, and says what it let through", async () => {
+ const c = capture();
+ assert.equal(await checkMain({ siteId: "demo", paths, allowMissingMedia: true }, c.out), 0);
+ assert.match(c.text(), /allowed \(--allow-missing-media\): missing-media/);
+ assert.match(c.text(), /pub1: 1 quote\(s\) checked, lowest 1\.00/);
+ assert.match(c.text(), /compose would pass/);
+});
+
+test("check --reports checks a draft not in site.json: a drifted quote fails it", async () => {
+ const c = capture();
+ assert.equal(await checkMain({ siteId: "demo", paths, reports: ["draft1"], allowMissingMedia: true }, c.out), 1);
+ assert.match(c.text(), /quote-drift: draft1#c1: the quote matches \d+% of what the record says there/);
+});
+
+test("check: an unknown site is a usage error", async () => {
+ const c = capture();
+ assert.equal(await checkMain({ siteId: "nope", paths }, c.out), 2);
+});
+
+test("verify-quotes: each quote's best track, en-orig where there is one, and what failed", async () => {
+ const c = capture();
+ const code = await verifyQuotesMain({ file: reportFile("demo", "mixed"), json: true, paths, now: "2026-10-09T00:00:00Z" }, c.out);
+ assert.equal(code, 1);
+ const { results } = JSON.parse(c.text()) as {
+ results: { citation: string; status: string; score?: number; track?: string; enOrig?: number }[];
+ };
+ const by = Object.fromEntries(results.map((r) => [r.citation, r]));
+ assert.equal(by.c1.status, "ok");
+ assert.equal(by.c1.score, 1);
+ assert.equal(by.c1.track, "transcript.en.vtt");
+ // Compose would pass c2 (its best track, the served `en`, matches) — but
+ // the words as spoken do not.
+ assert.equal(by.c2.status, "en-orig-drift");
+ assert.equal(by.c2.score, 1);
+ assert.equal(by.c2.track, "transcript.en.vtt");
+ assert.ok(by.c2.enOrig !== undefined && by.c2.enOrig < 0.6, String(by.c2.enOrig));
+ assert.equal(by.c3.status, "drift");
+ assert.equal(by.c4.status, "missing-record");
+});
+
+test("verify-quotes: a clean report exits 0, in words", async () => {
+ const c = capture();
+ assert.equal(await verifyQuotesMain({ file: reportFile("demo", "pub1"), paths }, c.out), 0);
+ assert.match(c.text(), /ok\s+c1 {2}video chan\/good1 10–20 s {2}1\.00 transcript\.en\.vtt/);
+ assert.match(c.text(), /1 quote\(s\): 1 ok/);
+});
+
+// ─── attach-video ───
+
+test("encodePlan: remux an H.264 mp4 under the limit; encode the rest to fit; refuse what cannot", () => {
+ const mp4 = { durationSec: 120, formatName: "mov,mp4,m4a,3gp,3g2,mj2", videoCodec: "h264", audioCodec: "aac", width: 1920 };
+ const MiB = 1024 * 1024;
+ assert.deepEqual(encodePlan(mp4, 10 * MiB), { mode: "remux" });
+ // Over the limit: 24 MiB × 0.96 over 120 s ≈ 1610 kb/s in all.
+ const over = encodePlan(mp4, 80 * MiB);
+ assert.equal(over.mode, "encode");
+ if (over.mode === "encode") {
+ assert.equal(over.audioKbps, 128);
+ assert.ok(over.videoKbps > 1300 && over.videoKbps < 1500, String(over.videoKbps));
+ assert.equal(over.maxWidth, 1280);
+ }
+ // A VP9 webm is encoded whatever its size.
+ assert.equal(encodePlan({ ...mp4, formatName: "matroska,webm", videoCodec: "vp9", audioCodec: "opus" }, MiB).mode, "encode");
+ // Two hours do not fit 24 MiB watchably.
+ const long = encodePlan({ ...mp4, durationSec: 7200 }, 900 * MiB);
+ assert.equal(long.mode, "refuse");
+ if (long.mode === "refuse") assert.match(long.reason, /120\.0 min .* trim it/);
+ assert.equal(encodePlan({ ...mp4, videoCodec: null }, MiB).mode, "refuse");
+ const args = encodeArgs("in.webm", "out.mp4", { mode: "encode", videoKbps: 800, audioKbps: 64, maxWidth: 854 });
+ assert.deepEqual(args.slice(args.indexOf("-b:v"), args.indexOf("-b:v") + 6), ["-b:v", "800k", "-maxrate", "1200k", "-bufsize", "1600k"]);
+ assert.ok(args.includes("scale='min(854,iw)':-2"));
+ assert.equal(args.at(-2), "+faststart");
+});
+
+const hasFfmpeg = (() => {
+ try {
+ execFileSync("ffmpeg", ["-version"], { stdio: "ignore" });
+ execFileSync("ffprobe", ["-version"], { stdio: "ignore" });
+ return true;
+ } catch {
+ return false;
+ }
+})();
+
+// A 3 s 640×360 clip with a tone: an H.264/AAC mp4, or an MPEG-4 Part 2 MKV.
+function makeClip(file: string, container: "mp4" | "mkv") {
+ execFileSync("ffmpeg", [
+ "-v", "error", "-y",
+ "-f", "lavfi", "-i", "testsrc=size=640x360:rate=25",
+ "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=44100",
+ "-t", "3",
+ ...(container === "mp4"
+ ? ["-c:v", "libx264", "-preset", "ultrafast", "-crf", "8", "-c:a", "aac"]
+ : ["-c:v", "mpeg4", "-q:v", "2", "-c:a", "aac"]),
+ file,
+ ]);
+}
+
+test("attach-video: an H.264 mp4 under the limit is remuxed, a poster drawn, report.json gets `video`", { skip: !hasFfmpeg && "no ffmpeg" }, async () => {
+ const file = reportFile("demo", "pub1");
+ const src = path.join(ROOT, "clip.mp4");
+ makeClip(src, "mp4");
+ const c = capture();
+ assert.equal(await attachVideoMain({ reportFile: file, video: src, caption: "The stream, cut." }, c.out), 0, c.text());
+ const dir = path.dirname(file);
+ const doc = JSON.parse(readFileSync(file, "utf8")) as { video: unknown; citations: unknown };
+ assert.deepEqual(doc.video, { src: "video.mp4", poster: "poster.jpg", caption: "The stream, cut." });
+ assert.ok(statSync(path.join(dir, "video.mp4")).size > 0);
+ assert.ok(statSync(path.join(dir, "poster.jpg")).size > 0);
+ assert.match(c.text(), /remuxed/);
+ // The rest of the report is untouched.
+ assert.deepEqual(Object.keys(doc.citations as object), ["c1"]);
+});
+
+test("attach-video: anything else is encoded under the limit (re-encoded smaller on an overshoot); the poster and caption are kept", { skip: !hasFfmpeg && "no ffmpeg" }, async () => {
+ const file = reportFile("demo", "pub1");
+ const src = path.join(ROOT, "clip.mkv");
+ makeClip(src, "mkv");
+ const limit = 200_000;
+ assert.ok(statSync(src).size > limit, "the fixture must start over the limit");
+ const c = capture();
+ assert.equal(await attachVideoMain({ reportFile: file, video: src, limitBytes: limit }, c.out), 0, c.text());
+ const dir = path.dirname(file);
+ const size = statSync(path.join(dir, "video.mp4")).size;
+ assert.ok(size > 0 && size <= limit, `video.mp4 is ${size} bytes, over ${limit}`);
+ const probe = execFileSync("ffprobe", ["-v", "error", "-show_entries", "stream=codec_name", "-of", "csv=p=0", path.join(dir, "video.mp4")], { encoding: "utf8" });
+ assert.deepEqual(probe.trim().split("\n").sort(), ["aac", "h264"]);
+ const doc = JSON.parse(readFileSync(file, "utf8")) as { video: unknown };
+ assert.deepEqual(doc.video, { src: "video.mp4", poster: "poster.jpg", caption: "The stream, cut." });
+ assert.match(c.text(), /encoded/);
+});
+
+test("attach-video refuses a file that is not a report, and writes nothing", { skip: !hasFfmpeg && "no ffmpeg" }, async () => {
+ const notReport = path.join(ROOT, "nope", "report.json");
+ writeJson(notReport, { hello: "world" });
+ const c = capture();
+ assert.equal(await attachVideoMain({ reportFile: notReport, video: path.join(ROOT, "clip.mp4") }, c.out), 1);
+ assert.match(c.text(), /is not a report/);
+ assert.throws(() => statSync(path.join(ROOT, "nope", "video.mp4")));
+});
diff --git a/common/bin/reports-check.ts b/common/bin/reports-check.ts
@@ -0,0 +1,276 @@
+// `archilyzer reports check <site>` and `archilyzer reports verify-quotes
+// <report.json>` — what compose would say about a site's reports, without a
+// build; and one report's quotes against the transcripts, track by track.
+//
+// CHECK is the reports stage of compose with nothing written:
+// resolveSiteReports (publish/composeReports.ts) — every report parsed and
+// validated, every cited quote verified against its record, every cited
+// moment's prepared media present and current, the report's video on disk and
+// under the publish limit. Its problems are compose's, word for word. With
+// `--reports a,b` it checks those reports (drafts included: a report need not
+// be in site.json yet); `--allow-missing-media` checks the text before
+// `reports prepare` has cut anything.
+//
+// VERIFY-QUOTES runs compose's own quote check (checkSpanQuote, the post
+// check) over every video, audio and post citation of ONE report.json,
+// published or not, and prints each one's best score and track — and, where
+// the record has an `en-orig` track, that track's score. A served `en` track
+// can be a rewrite of what was said; a quote that matches it and not
+// `en-orig` is not what the speaker said, and is reported as such, even
+// though compose (which takes the best track) would pass it.
+//
+// Exit 0 when there is nothing to report, 1 with the list, 2 for usage (an
+// unknown site, an unreadable file).
+
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { getPaths, type Paths } from "../lib/paths";
+import { getSite, listSiteIds } from "../lib/site";
+import { readChannelConfig } from "../controller/channels";
+import { assertChannelTextReadable } from "../lib/channelMedia";
+import { readAllPosts } from "../lib/posts-server";
+import { parseReport } from "../lib/report/validate";
+import type { Report } from "../lib/report/schema";
+import { reportCitationNumbers } from "../lib/report/uses";
+import { QUOTE_DRIFT_THRESHOLD, quoteDrifted, quoteVerification, roundScore } from "../lib/citations/verify";
+import {
+ ComposeReportsError,
+ checkSpanQuote,
+ formatComposeReportsProblems,
+ quoteDriftMessage,
+ readCitedRecord,
+ resolveSiteReports,
+} from "../publish/composeReports";
+
+type Out = { log: (s: string) => void; error: (s: string) => void };
+
+// ─── reports check ───
+
+export async function checkMain(
+ opts: {
+ siteId: string;
+ reports?: string[];
+ allowMissingMedia?: boolean;
+ paths?: Paths;
+ settings?: { social?: { x?: { visibility?: unknown } } };
+ },
+ out: Out = console,
+): Promise<number> {
+ const paths = opts.paths ?? getPaths();
+ if (!listSiteIds(paths).includes(opts.siteId)) {
+ out.error(`reports check: no site "${opts.siteId}" (sites/${opts.siteId}/site.json)`);
+ return 2;
+ }
+ const site = getSite(opts.siteId, paths);
+ const ids = opts.reports ?? site.reports ?? [];
+ if (ids.length === 0) {
+ out.log(`reports check ${opts.siteId}: the site publishes no reports — nothing to check.`);
+ return 0;
+ }
+ try {
+ const resolved = await resolveSiteReports({
+ paths,
+ site: { ...site, reports: ids },
+ allowMissingMedia: opts.allowMissingMedia === true,
+ ...(opts.settings ? { settings: opts.settings } : {}),
+ });
+ for (const line of formatComposeReportsProblems(resolved.allowed)) {
+ out.log(` allowed (--allow-missing-media): ${line}`);
+ }
+ for (const r of resolved.reports) {
+ const scores = Object.values(r.citations ?? {})
+ .map((c) => (c.kind === "video" || c.kind === "audio" || c.kind === "post" ? c.verification?.quoteScore : undefined))
+ .filter((s): s is number => typeof s === "number");
+ const low = scores.length ? Math.min(...scores) : null;
+ out.log(
+ ` ${r.id}: ${scores.length} quote(s) checked` + (low === null ? "" : `, lowest ${low.toFixed(2)}`),
+ );
+ }
+ out.log(
+ `reports check ${opts.siteId}: ${resolved.reports.length} report(s), ${resolved.moments.length} moment(s) — compose would pass.`,
+ );
+ return 0;
+ } catch (err) {
+ if (!(err instanceof ComposeReportsError)) {
+ out.error(`reports check ${opts.siteId}: ${(err as Error).message}`);
+ return 1;
+ }
+ out.error(`reports check ${opts.siteId}: ${err.problems.length} problem(s) — compose would fail:`);
+ for (const line of formatComposeReportsProblems(err.problems)) out.error(` ${line}`);
+ return 1;
+ }
+}
+
+// ─── reports verify-quotes ───
+
+export type QuoteStatus =
+ | "ok"
+ | "drift"
+ | "en-orig-drift"
+ | "missing-record"
+ | "no-cues"
+ | "missing-post"
+ | "unreadable";
+
+export type QuoteResult = {
+ citation: string;
+ kind: "video" | "audio" | "post";
+ channel: string;
+ id: string;
+ start?: number;
+ end?: number;
+ cited: boolean;
+ status: QuoteStatus;
+ // The best score and the track it came from (a post: its text).
+ score?: number;
+ track?: string;
+ // Every track's score; and the en-orig track's, when the record has one.
+ tracks?: { name: string; score: number }[];
+ enOrig?: number;
+ message?: string;
+};
+
+export const EN_ORIG_TRACK = "transcript.en-orig.vtt";
+
+// Every video, audio and post citation of a report, checked against the
+// corpus at `paths.channelsDir` — cited or not (an uncited one is marked).
+export async function verifyReportQuotes(
+ report: Report,
+ opts: { paths: Paths; now?: string },
+): Promise<QuoteResult[]> {
+ const { paths } = opts;
+ const now = opts.now ?? new Date().toISOString();
+ const used = new Set(reportCitationNumbers(report).keys());
+ const results: QuoteResult[] = [];
+ const unreadable = new Map<string, string | null>();
+ const textProblem = async (slug: string) => {
+ if (!unreadable.has(slug)) {
+ try {
+ await assertChannelTextReadable(paths, slug, await readChannelConfig(paths, slug).catch(() => null));
+ unreadable.set(slug, null);
+ } catch (e) {
+ unreadable.set(slug, (e as Error).message);
+ }
+ }
+ return unreadable.get(slug) ?? null;
+ };
+ const posts = new Map<string, Map<string, string>>();
+ for (const [cid, c] of Object.entries(report.citations ?? {})) {
+ if (c.kind !== "video" && c.kind !== "audio" && c.kind !== "post") continue;
+ const base = {
+ citation: cid,
+ kind: c.kind,
+ channel: c.channel,
+ id: c.id,
+ cited: used.has(cid),
+ ...(c.kind === "post" ? {} : { start: c.start, end: c.end }),
+ };
+ const text = await textProblem(c.channel);
+ if (text) {
+ results.push({ ...base, status: "unreadable", message: text });
+ continue;
+ }
+ if (c.kind === "post") {
+ if (!posts.has(c.channel)) {
+ const all = await readAllPosts(path.join(paths.channelsDir, c.channel)).catch(() => []);
+ posts.set(c.channel, new Map(all.map((p) => [p.id, p.text])));
+ }
+ const postText = posts.get(c.channel)!.get(c.id);
+ if (postText === undefined) {
+ results.push({ ...base, status: "missing-post", message: `no post ${c.id} in the posts archive of "${c.channel}"` });
+ continue;
+ }
+ const v = quoteVerification(c.quote, postText, now);
+ results.push({
+ ...base,
+ score: v.quoteScore,
+ track: "post",
+ status: quoteDrifted(v) ? "drift" : "ok",
+ ...(quoteDrifted(v) ? { message: quoteDriftMessage(v.quoteScore) } : {}),
+ });
+ continue;
+ }
+ const record = await readCitedRecord(paths.channelsDir, c.channel, c.id);
+ if (!record) {
+ results.push({ ...base, status: "missing-record", message: `no record ${c.channel}/${c.id} (no metadata or transcript in its data dir)` });
+ continue;
+ }
+ if (record.cues.length === 0) {
+ results.push({ ...base, status: "no-cues", message: `${c.channel}/${c.id} has no transcript cues to check the quote against` });
+ continue;
+ }
+ const checked = checkSpanQuote(record, c, now);
+ const score = checked.verification.quoteScore ?? 0;
+ const best = checked.tracks.reduce<{ name: string; score: number } | null>(
+ (b, t) => (!b || t.score > b.score ? t : b),
+ null,
+ );
+ const enOrig = checked.tracks.find((t) => t.name === EN_ORIG_TRACK)?.score;
+ let status: QuoteStatus = "ok";
+ let message: string | undefined;
+ if (quoteDrifted(checked.verification)) {
+ status = "drift";
+ message = quoteDriftMessage(score);
+ } else if (enOrig !== undefined && enOrig < QUOTE_DRIFT_THRESHOLD) {
+ status = "en-orig-drift";
+ message =
+ `the quote matches ${best?.name ?? "a track"} (${score.toFixed(2)}) but only ${Math.round(enOrig * 100)}% of ` +
+ `the en-orig track, the words as spoken — a served \`en\` track can rewrite them: quote en-orig, or check the audio`;
+ }
+ results.push({
+ ...base,
+ score: roundScore(score),
+ ...(best ? { track: best.name } : {}),
+ tracks: checked.tracks,
+ ...(enOrig !== undefined ? { enOrig } : {}),
+ status,
+ ...(message ? { message } : {}),
+ });
+ }
+ return results;
+}
+
+function describe(r: QuoteResult): string {
+ const where = r.kind === "post" ? `${r.channel}/${r.id}` : `${r.channel}/${r.id} ${r.start}–${r.end} s`;
+ const score = r.score === undefined ? "" : ` ${r.score.toFixed(2)}${r.track ? ` ${r.track}` : ""}`;
+ const orig = r.enOrig === undefined || r.track === EN_ORIG_TRACK ? "" : ` · en-orig ${r.enOrig.toFixed(2)}`;
+ const label = r.status === "ok" ? "ok" : r.status.toUpperCase();
+ return ` ${label.padEnd(14)} ${r.citation}${r.cited ? "" : " (not cited)"} ${r.kind} ${where}${score}${orig}` +
+ (r.message ? `\n${" ".repeat(17)}${r.message}` : "");
+}
+
+export async function verifyQuotesMain(
+ opts: { file: string; json?: boolean; paths?: Paths; now?: string },
+ out: Out = console,
+): Promise<number> {
+ const paths = opts.paths ?? getPaths();
+ let raw: unknown;
+ try {
+ raw = JSON.parse(await readFile(opts.file, "utf8"));
+ } catch (err) {
+ out.error(`reports verify-quotes: ${opts.file} is not readable JSON (${(err as Error).message})`);
+ return 2;
+ }
+ const parsed = parseReport(raw);
+ if (!parsed.ok) {
+ out.error(`reports verify-quotes: ${opts.file} is not a report:`);
+ for (const p of parsed.problems) out.error(` ${p.path}: ${p.message}`);
+ return 1;
+ }
+ const results = await verifyReportQuotes(parsed.value, { paths, ...(opts.now ? { now: opts.now } : {}) });
+ const bad = results.filter((r) => r.status !== "ok");
+ if (opts.json) {
+ out.log(JSON.stringify({ file: opts.file, problems: parsed.problems, results }, null, 2));
+ } else {
+ for (const p of parsed.problems) out.error(` invalid: ${p.path}: ${p.message}`);
+ for (const r of results) (r.status === "ok" ? out.log : out.error)(describe(r));
+ const counts = new Map<QuoteStatus, number>();
+ for (const r of results) counts.set(r.status, (counts.get(r.status) ?? 0) + 1);
+ out.log(
+ `reports verify-quotes: ${results.length} quote(s): ` +
+ [...counts].map(([s, n]) => `${n} ${s}`).join(", ") +
+ (parsed.problems.length ? `; ${parsed.problems.length} validation problem(s)` : ""),
+ );
+ }
+ return bad.length > 0 || parsed.problems.length > 0 ? 1 : 0;
+}
diff --git a/common/controller/transcribeFile.test.ts b/common/controller/transcribeFile.test.ts
@@ -10,6 +10,7 @@ import {
symlink,
writeFile,
} from "node:fs/promises";
+import { existsSync, statSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import type { Worker } from "../lib/workers";
@@ -130,7 +131,10 @@ const {
enqueueTranscribeFile,
offsetCues,
parseTranscribeFileBody,
+ transcribeFileTier,
transcribeWorkerFilter,
+ URGENT_MAX_AUDIO_SEC,
+ wavSeconds,
windowOf,
windowWavArgs,
wordsFromTranscript,
@@ -461,3 +465,90 @@ test("the guards answer before any job exists", async () => {
assert.match((await runJob({ path: MEDIA, workerId: "off" })).error!, /"off" is disabled/);
assert.equal(await readFile(SETTINGS_FILE, "utf8"), SETTINGS_TEXT);
});
+
+// --- the wait, and the tier ----------------------------------------------------
+
+test("a cut's length decides its tier: up to 15 minutes of audio is urgent", () => {
+ assert.equal(wavSeconds(44), 0);
+ assert.equal(wavSeconds(44 + 32_000 * 90), 90);
+ assert.equal(URGENT_MAX_AUDIO_SEC, 900);
+ assert.equal(transcribeFileTier(0.1), "urgent");
+ assert.equal(transcribeFileTier(900), "urgent");
+ assert.equal(transcribeFileTier(901), "foreground");
+});
+
+const { getWorkerPool } = await import("../jobs/workerPool");
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+const onGpu = (w: Worker) => w.id === "gpu";
+
+// Enqueue a job and resolve once it is PARKED on the pool (its log says it is
+// waiting for a worker) — a fixed sleep raced the fake ffmpeg's start on a
+// loaded machine. `finish` reads the result as runJob does.
+async function parkedJob(body: Record<string, unknown>) {
+ const res = await enqueueTranscribeFile(body, { paths });
+ if (!res.ok) throw new Error(`refused: ${res.error}`);
+ void res.stream.cancel();
+ const logFile = path.join(paths.jobsDir, `${res.jobId}.log`);
+ const t0 = Date.now();
+ while (!/Waiting for worker/.test(await readFile(logFile, "utf8").catch(() => ""))) {
+ if (Date.now() - t0 > 20_000) throw new Error("the job never parked");
+ await sleep(20);
+ }
+ const finish = async () => {
+ const done = await res.done;
+ const log = await readFile(logFile, "utf8");
+ const line = log.split("\n").find((l) => l.startsWith(TRANSCRIBE_RESULT_MARKER));
+ return {
+ status: done.status,
+ log,
+ result: line ? JSON.parse(line.slice(TRANSCRIBE_RESULT_MARKER.length)) : null,
+ };
+ };
+ return { finish };
+}
+
+test("durationMs is the engine's time; the wait for a busy worker is waitedMs", async () => {
+ const pool = getWorkerPool();
+ // Something else holds the GPU worker's one slot.
+ const held = await pool.acquire(undefined, { only: onGpu });
+ const job = await parkedJob({ path: MEDIA, workerId: "gpu" });
+ await sleep(500);
+ const released = Date.now();
+ held.release();
+ const run = await job.finish();
+ assert.equal(run.status, "done", run.log);
+ const r = run.result;
+ assert.ok(r.waitedMs >= 450, `waitedMs ${r.waitedMs}`);
+ assert.ok(r.durationMs >= 0, `durationMs ${r.durationMs}`);
+ // The engine's clock starts when the worker is taken — after the release.
+ assert.ok(r.durationMs <= Date.now() - released, `durationMs ${r.durationMs} counts the wait`);
+ assert.match(run.log, /ahead of queued transcriptions/);
+});
+
+test("a short file goes ahead of parked transcriptions — the lane's and a manual batch's", async () => {
+ const pool = getWorkerPool();
+ const held = await pool.acquire(undefined, { only: onGpu });
+ const t0 = Date.now();
+ await sleep(20); // so an engine run after this is visibly newer than t0
+ // Did the file job's engine run before this slot was granted?
+ const engineRan = () => existsSync(ENGINE_ARGS) && statSync(ENGINE_ARGS).mtimeMs >= t0;
+ const order: string[] = [];
+ // Parked first: an auto-lane unit (background) and a manual batch's next
+ // video (foreground), both waiting for the GPU worker.
+ const lane = pool.acquire(undefined, { background: true, only: onGpu }).then((l) => {
+ order.push(`lane${engineRan() ? " after the file" : ""}`);
+ l.release();
+ });
+ const batch = pool.acquire(undefined, { only: onGpu }).then((l) => {
+ order.push(`batch${engineRan() ? " after the file" : ""}`);
+ l.release();
+ });
+ const job = await parkedJob({ path: MEDIA, workerId: "gpu" });
+ held.release();
+ const run = await job.finish();
+ await Promise.all([lane, batch]);
+ assert.equal(run.status, "done", run.log);
+ // The file job, parked LAST, took the freed slot first; the two parked
+ // before it got it after, in their own order (manual before the lane).
+ assert.deepEqual(order, ["batch after the file", "lane after the file"]);
+});
diff --git a/common/controller/transcribeFile.ts b/common/controller/transcribeFile.ts
@@ -54,6 +54,7 @@ import { writeJsonAtomic } from "../lib/jsonFile-server";
import { getSettings } from "../lib/settings";
import { getWorkerPool, type WorkerFilter } from "../jobs/workerPool";
import { makeTaskTracker } from "../jobs/taskHooks";
+import type { SchedulerTier } from "../jobs/jobKinds";
import {
runManagedFunction,
type JobRunContext,
@@ -80,6 +81,27 @@ export const TRANSCRIBE_FILE_BODY_KEYS = [
const AUDIO_NAME = "audio.wav";
// A WAV header with no samples after it: the window held no audio.
const EMPTY_WAV_BYTES = 44;
+// The cut is 16 kHz mono s16 (windowWavArgs): 32,000 bytes a second.
+const WAV_BYTES_PER_SEC = 16_000 * 2;
+
+// A file transcription at most this long (seconds of audio) waits for a worker
+// in the pool's "urgent" tier, ahead of every parked transcription — the
+// lane's background units and a manual batch's next video alike. It is the
+// quote check an agent is waiting on; one more long transcription in the
+// queue is not. Longer files keep the manual ("foreground") tier: still ahead
+// of the lane, behind batches queued before them. The tier only orders
+// WAITERS: a transcription already running is never interrupted.
+export const URGENT_MAX_AUDIO_SEC = 15 * 60;
+
+/** Seconds of audio in the cut WAV, from its size. */
+export function wavSeconds(bytes: number): number {
+ return Math.max(0, bytes - EMPTY_WAV_BYTES) / WAV_BYTES_PER_SEC;
+}
+
+/** The worker-pool tier a file transcription of `audioSec` seconds asks for. */
+export function transcribeFileTier(audioSec: number): SchedulerTier {
+ return audioSec <= URGENT_MAX_AUDIO_SEC ? "urgent" : "foreground";
+}
export type TranscribeFileRequest = {
path: string;
@@ -112,7 +134,11 @@ export type TranscribeFileResult = {
worker: TranscribeWorkerInfo;
transcriptFormat: TranscriptOutputFormat;
transcribedAt: string;
+ // The engine's own time: from the moment a worker took the job to the
+ // transcript. The wait for a free worker is `waitedMs`, never in here.
durationMs: number;
+ // How long the job waited for a free worker (the pool's queue).
+ waitedMs: number;
cues: Cue[];
text: string;
// Present only when the request asked for words: [] when the engine has none.
@@ -441,7 +467,6 @@ export async function runTranscribeFile(
opts: RunTranscribeFileOpts,
): Promise<TranscribeFileResult> {
const { request: req, onLog, paths } = opts;
- const started = Date.now();
const scratch = await mkdtemp(path.join(os.tmpdir(), "archilyzer-transcribe-"));
try {
const wav = path.join(scratch, AUDIO_NAME);
@@ -465,12 +490,18 @@ export async function runTranscribeFile(
}
// Set by onWorker; a holder, so the closure's write is seen after the await.
- const used: { worker?: Worker } = {};
+ // `startedAt` is the moment a worker took it: the engine's clock starts
+ // there (the last attempt's, when a transport failure moved it).
+ const used: { worker?: Worker; startedAt?: number } = {};
+ const audioSec = wavSeconds(wavStat.size);
+ const tier = transcribeFileTier(audioSec);
onLog(
- req.workerId
- ? `Waiting for worker ${req.workerId}…`
- : "Waiting for a free local worker…",
+ `${req.workerId ? `Waiting for worker ${req.workerId}` : "Waiting for a free local worker"}` +
+ (tier === "urgent"
+ ? ` (${Math.round(audioSec)}s of audio: ahead of queued transcriptions)…`
+ : "…"),
);
+ const asked = Date.now();
const label = `${path.basename(req.path)}${windowOf(req) ? " (window)" : ""}`;
const outcome = await transcribeWithWorker({
paths,
@@ -484,12 +515,15 @@ export async function runTranscribeFile(
onLog,
signal: opts.signal,
only: transcribeWorkerFilter(req.workerId),
+ tier,
onWorker: (w) => {
used.worker = w;
+ used.startedAt = Date.now();
},
skipInlineDiarization: true,
...(req.words ? { words: true } : {}),
});
+ const finished = Date.now();
const worker = used.worker;
if (outcome !== "transcribed" || !worker) {
throw new Error(
@@ -508,7 +542,8 @@ export async function runTranscribeFile(
worker: describeTranscribeWorker(worker),
transcriptFormat,
transcribedAt: new Date().toISOString(),
- durationMs: Date.now() - started,
+ durationMs: finished - (used.startedAt ?? asked),
+ waitedMs: (used.startedAt ?? asked) - asked,
cues,
text: cues.map((c) => c.text.trim()).filter(Boolean).join(" "),
...(req.words ? { words: wordsFromTranscript(raw, req.start ?? 0) } : {}),
@@ -558,7 +593,10 @@ export async function enqueueTranscribeFile(
onLog(
`Transcribed with ${result.worker.appId} [${result.worker.id}]` +
`${result.worker.model ? ` model ${result.worker.model}` : ""}: ` +
- `${result.cues.length} cue(s) in ${(result.durationMs / 1000).toFixed(1)}s`,
+ `${result.cues.length} cue(s) in ${(result.durationMs / 1000).toFixed(1)}s` +
+ (result.waitedMs >= 1000
+ ? ` (after ${(result.waitedMs / 1000).toFixed(1)}s waiting for the worker)`
+ : ""),
);
if (req.out) {
await writeJsonAtomic(req.out, result, { indent: 2 });
diff --git a/common/controller/transcribeOne.ts b/common/controller/transcribeOne.ts
@@ -11,6 +11,7 @@ import {
type WorkerFilter,
} from "../jobs/workerPool";
import type { TaskTracker } from "../jobs/taskHooks";
+import type { SchedulerTier } from "../jobs/jobKinds";
import { normalizeTranscript } from "./normalizeTranscript";
import { diarizeOneVideo } from "./diarizeOne";
import { getSettings } from "../lib/settings";
@@ -376,6 +377,9 @@ export type TranscribeWithWorkerOptions = {
// Auto-runner units pass true so they park BEHIND any manual (foreground)
// acquire in the worker pool — a manual transcribe preempts queued auto work.
background?: boolean;
+ // The pool tier outright, over `background`: a short one-off file
+ // transcription asks "urgent", ahead of every parked batch (transcribeFile).
+ tier?: SchedulerTier;
// Narrows WHICH workers may take this video (the pool's `only`): a one-off
// file transcription keeps to local workers, or to the one it was told to use.
only?: WorkerFilter;
@@ -411,6 +415,7 @@ export async function transcribeWithWorker(
try {
lease = await pool.acquire(acquireSignal, {
background: opts.background,
+ ...(opts.tier ? { tier: opts.tier } : {}),
...(opts.only ? { only: opts.only } : {}),
});
} catch (err) {
diff --git a/common/lib/envVars.test.ts b/common/lib/envVars.test.ts
@@ -153,7 +153,16 @@ test("the docker audience is the ARCHILYZER_ set", () => {
// (one-core Phase 4 slice 3). The two exceptions keep names others depend on:
// Playwright's own convention, and the machine-global queue's nesting marker
// (its protocol is shared with checkouts on older code).
-const UNPREFIXED_TEST_VARS = new Set(["PLAYWRIGHT_BASE_URL", "QUEUE_LOCK_HELD"]);
+// The heavy slot's seams are named for the gate, not for e2e: the gate runs
+// builds and renders too, and its tests are what set them.
+const UNPREFIXED_TEST_VARS = new Set([
+ "PLAYWRIGHT_BASE_URL",
+ "QUEUE_LOCK_HELD",
+ "HEAVY_HELD",
+ "HEAVY_LOCK_FILE",
+ "HEAVY_MEMINFO_FILE",
+ "HEAVY_POLL_MS",
+]);
test("every test-only variable carries the E2E_ prefix", () => {
const bad = ENV_VARS.filter(
diff --git a/common/lib/envVars.ts b/common/lib/envVars.ts
@@ -147,6 +147,9 @@ const DECLARED: EnvVarDecl[] = [
{ name: "PARAKEET_DECODER", audience: "runtime", default: "parakeet-cli's", readBy: "scripts/parakeet-stitch.mjs", doc: "`ctc` or `tdt`, passed through to parakeet-cli." },
{ name: "PARAKEET_LANG", audience: "runtime", default: "parakeet-cli's", readBy: "scripts/parakeet-stitch.mjs", doc: "A locale, passed through to parakeet-cli." },
{ name: "PARAKEET_DEVICE", audience: "runtime", default: "parakeet-cli's", readBy: "scripts/parakeet-stitch.mjs", doc: "Compute device (`cpu`, `CUDA0`, `Vulkan1`, …), exported to parakeet-cli." },
+ { name: "HEAVY", audience: "runtime", default: "on", readBy: "scripts/queue-lock.mjs", doc: "`0` skips the heavy slot AND the memory floor: the machine-wide one-at-a-time gate that `pnpm heavy -- <cmd>`, every e2e entry point and the publish stages' `next build` go through." },
+ { name: "HEAVY_MIN_FREE_MB", audience: "runtime", default: "`6000`", readBy: "scripts/queue-lock.mjs", doc: "The memory floor: a heavy job, once it holds the slot, waits until /proc/meminfo's MemAvailable is at least this many MB. `0` turns the floor off; a machine whose MemTotal is under it runs without waiting." },
+ { name: "HEAVY_TIMEOUT", audience: "runtime", default: "wait forever", readBy: "scripts/queue-lock.mjs", doc: "Seconds a `pnpm heavy` run waits for the slot, and then for the floor, before giving up (exit 3). An e2e run uses `E2E_QUEUE_TIMEOUT` for both." },
// ── internal: the pipeline sets these for a process it spawns ──────────
{ name: "SITE_ID", audience: "internal", default: "—", readBy: "common/bin/compose-site.ts, export/app/lib/site.ts", doc: "Which site a compose or an export build is for. The build stage (`archilyzer publish build <id>`) sets it for its children; `compose site`, `build site` and `deploy site` fall back to it when no id is given." },
@@ -185,6 +188,10 @@ const DECLARED: EnvVarDecl[] = [
{ name: "E2E_PORT_GRACE_MS", audience: "test", default: "`3000`", readBy: "scripts/queue-lock.mjs", doc: "How long the port check waits for a just-freed port." },
{ name: "E2E_QUEUE_LOCK_FILE", audience: "test", default: "one per machine", readBy: "scripts/queue-lock.mjs", doc: "The queue's lock file; the queue's own tests point it elsewhere." },
{ name: "QUEUE_LOCK_HELD", audience: "test", default: "—", readBy: "scripts/queue-lock.mjs", doc: "Set by the queue for the command it runs, so a nested wrapper passes through." },
+ { name: "HEAVY_HELD", audience: "test", default: "—", readBy: "scripts/queue-lock.mjs", doc: "Set by the heavy slot for the command it runs, so a heavy command inside it (a `pnpm heavy -- pnpm e2e`, a build stage under an e2e suite's editor) passes through." },
+ { name: "HEAVY_LOCK_FILE", audience: "test", default: "one per machine", readBy: "scripts/queue-lock.mjs", doc: "The heavy slot's lock file; the gate's own tests point it elsewhere." },
+ { name: "HEAVY_MEMINFO_FILE", audience: "test", default: "`/proc/meminfo`", readBy: "scripts/queue-lock.mjs", doc: "Where the memory floor reads MemAvailable; the gate's tests hand it a fake." },
+ { name: "HEAVY_POLL_MS", audience: "test", default: "`5000`", readBy: "scripts/queue-lock.mjs", doc: "How often a run waiting for the memory floor re-reads it." },
{ name: "PLAYWRIGHT_BASE_URL", audience: "test", default: "`http://localhost:<PORT>`", readBy: "editor/playwright.config.ts, editor/e2e/baseUrl.ts", doc: "The editor test server's URL; the worktree injector sets it." },
{ name: "E2E_AUDIO_CHECK_INTERVAL_MS", audience: "test", default: "the real cadence", readBy: "common/ytdlp/audioCheckedDownload.ts", doc: "Shrinks the mid-download audio check so the e2e suite sees it fire." },
{ name: "E2E_AUDIO_CHECK_SIZE_GATE", audience: "test", default: "the real gate", readBy: "common/ytdlp/audioCheckedDownload.ts", doc: "Likewise, the size gate." },
diff --git a/common/lib/safeStreamController.test.ts b/common/lib/safeStreamController.test.ts
@@ -0,0 +1,64 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { makeSafeController } from "./safeStreamController";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test lib/safeStreamController.test.ts
+//
+// The guard the media file route and the job log streams wrap their
+// controllers in. A consumer that goes away (a scrubbing <audio> cancelling its
+// range request) closes the stream under the producer; a raw controller then
+// throws ERR_INVALID_STATE from inside the encoder's pipeline, where it leaks as
+// an uncaughtException. The safe one goes quiet instead.
+
+function stream() {
+ const safe = makeSafeController<Uint8Array>();
+ let raw!: ReadableStreamDefaultController<Uint8Array>;
+ const rs = new ReadableStream<Uint8Array>({
+ start(c) {
+ raw = c;
+ safe.setController(c);
+ },
+ });
+ return { safe, raw: () => raw, rs };
+}
+
+test("a raw controller throws once the stream is closed — the hazard", () => {
+ const { raw } = stream();
+ raw().close();
+ assert.throws(() => raw().enqueue(new Uint8Array(1)), {
+ code: "ERR_INVALID_STATE",
+ });
+});
+
+test("after a cancel, enqueue/close/error are no-ops", async () => {
+ const { safe, rs } = stream();
+ await rs.cancel();
+ safe.markClosed();
+ safe.safeEnqueue(new Uint8Array(1));
+ safe.safeClose();
+ safe.safeError(new Error("late"));
+ assert.equal(safe.isClosed(), true);
+});
+
+test("a stream closed under it is noticed at the next enqueue, without a throw", () => {
+ const { safe, raw } = stream();
+ // Closed by someone else — the guard has not been told.
+ raw().close();
+ assert.equal(safe.isClosed(), false);
+ safe.safeEnqueue(new Uint8Array(1));
+ assert.equal(safe.isClosed(), true);
+ safe.safeClose();
+ safe.safeError(new Error("late"));
+});
+
+test("close and error each happen once", async () => {
+ const { safe, rs } = stream();
+ const reader = rs.getReader();
+ safe.safeEnqueue(new Uint8Array([1, 2]));
+ safe.safeClose();
+ safe.safeClose();
+ safe.safeError(new Error("after close"));
+ assert.deepEqual((await reader.read()).value, new Uint8Array([1, 2]));
+ assert.equal((await reader.read()).done, true);
+});
diff --git a/common/publish/build.test.ts b/common/publish/build.test.ts
@@ -20,6 +20,7 @@ import {
homepageOutDir,
dockerSiteOutDir,
dockerSiteStagingDir,
+ heavyGated,
resolveOutDir,
} from "./build";
@@ -130,6 +131,33 @@ test("EXPORT_NEXT_BIN replaces `pnpm exec next build` in a site's and the hub's
assert.deepEqual(plain[1].args, ["exec", "next", "build"]);
});
+// A real `next build` runs through the heavy slot (scripts/queue-lock.mjs
+// --heavy): one heavy job machine-wide, above the memory floor. The e2e fake is
+// never gated, and a root without the gate script (every pure test above, whose
+// /repo does not exist) runs the step unchanged.
+test("a real next build goes through the heavy slot; the e2e fake and a root without the gate do not", () => {
+ const root = mkdtempSync(path.join(os.tmpdir(), "build-heavy-"));
+ try {
+ mkdirSync(path.join(root, "scripts"));
+ const gate = path.join(root, "scripts", "queue-lock.mjs");
+ writeFileSync(gate, "");
+ const p = { ...paths, monorepoRoot: root, exportDir: path.join(root, "export") } as Paths;
+ const [, next] = buildSiteSteps({ siteId: "jer", paths: p, skipData: true, baseEnv: {} });
+ assert.equal(next.command, process.execPath);
+ assert.deepEqual(next.args, [gate, "--heavy", "--", "pnpm", "exec", "next", "build"]);
+ assert.equal(next.cwd, path.join(root, "export"));
+ const hub = buildHubSteps({ paths: p, baseEnv: {} });
+ assert.deepEqual(hub[1].args.slice(0, 3), [gate, "--heavy", "--"]);
+ assert.equal(hub[1].env.INSTANCE_MODE, "hub");
+ const fake = buildSiteSteps({ siteId: "jer", paths: p, skipData: true, baseEnv: { EXPORT_NEXT_BIN: "/bin/fake-next" } });
+ assert.deepEqual([fake[1].command, ...fake[1].args], ["/bin/fake-next", "build"]);
+ const step = { command: "pnpm", args: ["x"], cwd: "/", env: {} };
+ assert.equal(heavyGated({ monorepoRoot: path.join(root, "nope") }, step), step);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
test("buildHubSteps: compose:hub, then next build with INSTANCE_MODE=hub, in export/", () => {
const steps = buildHubSteps({ paths, baseEnv: { PATH: "/bin" } });
const env = {
diff --git a/common/publish/build.ts b/common/publish/build.ts
@@ -114,7 +114,36 @@ export function nextBuildStep(paths: Paths, env: NodeJS.ProcessEnv): BuildStep {
const bin = env.EXPORT_NEXT_BIN?.trim();
return bin
? { command: bin, args: ["build"], cwd: paths.exportDir, env }
- : { command: "pnpm", args: ["exec", "next", "build"], cwd: paths.exportDir, env };
+ : heavyGated(paths, {
+ command: "pnpm",
+ args: ["exec", "next", "build"],
+ cwd: paths.exportDir,
+ env,
+ });
+}
+
+/**
+ * A real `next build` run through the HEAVY SLOT (scripts/queue-lock.mjs
+ * --heavy, the same gate as `pnpm heavy -- <cmd>`): one heavy job — an e2e
+ * run, a build, a render — at a time machine-wide, started only once
+ * MemAvailable is at least HEAVY_MIN_FREE_MB (6000). Two concurrent builds, or a
+ * build beside an e2e suite, is how this machine OOMed. The wait is logged
+ * ("waiting for the heavy slot — held by …") into the stage's own log, and a
+ * Cancel still stops it: the gate forwards SIGTERM to the build.
+ *
+ * Inside a docker runner container the slot is the container's own (no shared
+ * lock); the floor still reads the host's /proc/meminfo, which throttles a
+ * fan-out when the host runs low. HEAVY=0 bypasses both; a checkout without
+ * the gate script (a test's temp root) runs the step as it was.
+ */
+export function heavyGated(paths: Pick<Paths, "monorepoRoot">, step: BuildStep): BuildStep {
+ const gate = path.join(paths.monorepoRoot ?? "", "scripts", "queue-lock.mjs");
+ if (!paths.monorepoRoot || !existsSync(gate)) return step;
+ return {
+ ...step,
+ command: process.execPath,
+ args: [gate, "--heavy", "--", step.command, ...step.args],
+ };
}
// Run a list of child steps in order, streaming into `onLog`, stopping at the
@@ -695,12 +724,12 @@ export async function buildHomepage(
}
}
return runSteps(onLog, signal, [
- {
+ heavyGated(paths, {
command: "pnpm",
args: ["exec", "next", "build"],
cwd: homepageDir(paths),
env: homepageEnv(paths),
- },
+ }),
]);
}
diff --git a/common/publish/composeReports.ts b/common/publish/composeReports.ts
@@ -229,7 +229,7 @@ export type ComposedReports = {
// ─── Reading the corpus ───
-type CitedRecord = {
+export type CitedRecord = {
summary: Pick<TranscriptSummary, "id" | "slug" | "title" | "uploadDate" | "platform" | "webpageUrl" | "channel">;
cues: Cue[];
// Every transcript of the record that has cues, by file name: the
@@ -304,6 +304,45 @@ export async function readCitedRecord(
return { summary, cues, tracks, archiveOrg, wayback };
}
+// THE SPAN QUOTE CHECK — the one compose runs, and `archilyzer reports
+// verify-quotes` and `reports check` report: the quote against the cue window
+// of EVERY transcript the record has (lib/citations/verify.ts), the best match
+// is the verification (its method names the track). `tracks` is each track's
+// own score, so a caller can say what the `en-orig` track — the words as
+// spoken — makes of a quote a served `en` rewrite matched.
+export type SpanQuoteCheck = {
+ verification: ReturnType<typeof quoteVerification>;
+ // The best track's cues (what a moment page shows), or null when the record
+ // had no track and its default cues were used.
+ cues: Cue[] | null;
+ tracks: { name: string; score: number }[];
+};
+
+export function checkSpanQuote(
+ record: Pick<CitedRecord, "cues" | "tracks">,
+ c: { quote: string; start: number; end: number },
+ now: string,
+): SpanQuoteCheck {
+ let best: { name: string; cues: Cue[]; v: ReturnType<typeof quoteVerification> } | null = null;
+ const tracks: SpanQuoteCheck["tracks"] = [];
+ for (const t of record.tracks) {
+ const v = quoteVerification(c.quote, cueWindowText(t.cues, c.start, c.end), now);
+ tracks.push({ name: t.name, score: v.quoteScore ?? 0 });
+ if (!best || (v.quoteScore ?? 0) > (best.v.quoteScore ?? 0)) best = { name: t.name, cues: t.cues, v };
+ }
+ return best
+ ? { verification: { ...best.v, method: `${best.v.method}; text: ${best.name}` }, cues: best.cues, tracks }
+ : { verification: quoteVerification(c.quote, cueWindowText(record.cues, c.start, c.end), now), cues: null, tracks };
+}
+
+// The sentence compose fails a drifted quote with.
+export function quoteDriftMessage(score: number | undefined): string {
+ return (
+ `the quote matches ${Math.round((score ?? 0) * 100)}% of what the record says there ` +
+ `(at least ${Math.round(QUOTE_DRIFT_THRESHOLD * 100)}% is required): quote it verbatim, or fix the span`
+ );
+}
+
const isoDay = (uploadDate: string | undefined): string | undefined =>
uploadDate && /^\d{8}$/.test(uploadDate)
? `${uploadDate.slice(0, 4)}-${uploadDate.slice(4, 6)}-${uploadDate.slice(6, 8)}`
@@ -541,25 +580,17 @@ export async function resolveSiteReports(opts: ResolveSiteReportsOptions): Promi
problems.push({ kind: "no-cues", citation: ref, report: report.id, message: `${c.channel}/${c.id} has no transcript cues to check the quote against` });
continue;
}
- let best: { name: string; cues: Cue[]; v: ReturnType<typeof quoteVerification> } | null = null;
- for (const t of record.tracks) {
- const v = quoteVerification(c.quote, cueWindowText(t.cues, c.start, c.end), now);
- if (!best || (v.quoteScore ?? 0) > (best.v.quoteScore ?? 0)) best = { name: t.name, cues: t.cues, v };
- }
- c.verification = best
- ? { ...best.v, method: `${best.v.method}; text: ${best.name}` }
- : quoteVerification(c.quote, cueWindowText(record.cues, c.start, c.end), now);
+ const checked = checkSpanQuote(record, c, now);
+ c.verification = checked.verification;
const mk = momentKeyOf(c);
- if (best && mk && !momentCues.has(mk)) momentCues.set(mk, best.cues);
+ if (checked.cues && mk && !momentCues.has(mk)) momentCues.set(mk, checked.cues);
}
if (quoteDrifted(c.verification)) {
problems.push({
kind: "quote-drift",
citation: ref,
report: report.id,
- message:
- `the quote matches ${Math.round((c.verification.quoteScore ?? 0) * 100)}% of what the record says there ` +
- `(at least ${Math.round(QUOTE_DRIFT_THRESHOLD * 100)}% is required): quote it verbatim, or fix the span`,
+ message: quoteDriftMessage(c.verification.quoteScore),
});
}
}
diff --git a/common/publish/reportVideo.ts b/common/publish/reportVideo.ts
@@ -0,0 +1,251 @@
+// A REPORT'S VIDEO — `archilyzer reports attach-video <report.json> <video>`:
+// the video encoded (or remuxed) to fit the publish limit, written beside the
+// report as `video.mp4` with a poster, and `video` set in report.json.
+//
+// The limit is the one compose enforces on the report's video
+// (lib/builtExport.ts PUBLISH_MAX_FILE_BYTES, 24 MiB — Pages allows 25 per
+// file); a video over it fails compose ("report-video"). So:
+//
+// - an mp4 already H.264 (+ AAC, or no audio) and under the limit is
+// REMUXED, not re-encoded: streams copied, `+faststart` so it plays
+// before it has loaded;
+// - anything else is ENCODED: H.264 + AAC at an average bitrate the
+// duration allows inside 96% of the limit (capped at 2.5 Mb/s of video,
+// no wider than 1280 px, narrower as the bitrate falls). A single pass
+// can land over its target, so a result over the limit is encoded again
+// at a bitrate scaled down by the overshoot, up to three times;
+// - a video so long that it would get under 100 kb/s of picture is refused
+// with its length — trim it, the encoder cannot make it watchable.
+//
+// The poster is the one given (png, jpg or webp), else the report's existing
+// one when it is on disk, else a frame from 10% into the video. Nothing is
+// written into report.json unless the video (and poster) are in place and
+// its `video` validates.
+
+import { copyFile, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { execa } from "execa";
+import { PUBLISH_MAX_FILE_BYTES, publishFileSizeProblem } from "../lib/builtExport";
+import { parseReport } from "../lib/report/validate";
+
+export const REPORT_VIDEO_NAME = "video.mp4";
+export const REPORT_POSTER_NAME = "poster.jpg";
+// The share of the limit an encode aims at: the rest is container overhead
+// and the encoder's own overshoot.
+export const ATTACH_TARGET_FRACTION = 0.96;
+export const MAX_VIDEO_KBPS = 2500;
+export const MIN_VIDEO_KBPS = 100;
+const MAX_ENCODE_ATTEMPTS = 3;
+const POSTER_EXTS = new Set([".png", ".jpg", ".jpeg", ".webp"]);
+
+export type VideoProbe = {
+ durationSec: number;
+ formatName: string;
+ videoCodec: string | null;
+ audioCodec: string | null;
+ width: number | null;
+};
+
+export type EncodePlan =
+ | { mode: "remux" }
+ | { mode: "encode"; videoKbps: number; audioKbps: number; maxWidth: number }
+ | { mode: "refuse"; reason: string };
+
+// What to do with a video of `bytes` described by `probe`, to fit `limitBytes`.
+export function encodePlan(probe: VideoProbe, bytes: number, limitBytes = PUBLISH_MAX_FILE_BYTES): EncodePlan {
+ if (!probe.videoCodec) return { mode: "refuse", reason: "it has no video stream" };
+ if (!(probe.durationSec > 0)) return { mode: "refuse", reason: "its duration cannot be read" };
+ const isMp4 = /(^|,)(mp4|mov)(,|$)/.test(probe.formatName);
+ if (
+ isMp4 &&
+ probe.videoCodec === "h264" &&
+ (probe.audioCodec === null || probe.audioCodec === "aac") &&
+ bytes <= limitBytes
+ ) {
+ return { mode: "remux" };
+ }
+ const totalKbps = Math.floor((limitBytes * ATTACH_TARGET_FRACTION * 8) / 1000 / probe.durationSec);
+ const audioKbps = probe.audioCodec === null ? 0 : totalKbps >= 1000 ? 128 : 64;
+ // ~2% for the container.
+ const videoKbps = Math.min(MAX_VIDEO_KBPS, Math.floor((totalKbps - audioKbps) * 0.98));
+ if (videoKbps < MIN_VIDEO_KBPS) {
+ const minutes = (probe.durationSec / 60).toFixed(1);
+ return {
+ mode: "refuse",
+ reason:
+ `at ${minutes} min it would get ${Math.max(0, videoKbps)} kb/s of picture inside ` +
+ `${(limitBytes / (1024 * 1024)).toFixed(0)} MiB (at least ${MIN_VIDEO_KBPS} is watchable) — trim it`,
+ };
+ }
+ const maxWidth = videoKbps >= 1200 ? 1280 : videoKbps >= 500 ? 854 : 640;
+ return { mode: "encode", videoKbps, audioKbps, maxWidth };
+}
+
+export function encodeArgs(src: string, dst: string, plan: Exclude<EncodePlan, { mode: "refuse" }>): string[] {
+ const head = ["-nostdin", "-hide_banner", "-v", "error", "-y", "-i", src];
+ if (plan.mode === "remux") return [...head, "-map", "0:v:0", "-map", "0:a:0?", "-c", "copy", "-movflags", "+faststart", dst];
+ const v = plan.videoKbps;
+ return [
+ ...head,
+ "-map", "0:v:0",
+ "-map", "0:a:0?",
+ "-vf", `scale='min(${plan.maxWidth},iw)':-2`,
+ "-c:v", "libx264",
+ "-preset", "medium",
+ "-b:v", `${v}k`,
+ "-maxrate", `${Math.round(v * 1.5)}k`,
+ "-bufsize", `${v * 2}k`,
+ "-pix_fmt", "yuv420p",
+ ...(plan.audioKbps > 0 ? ["-c:a", "aac", "-b:a", `${plan.audioKbps}k`, "-ac", "2"] : ["-an"]),
+ "-movflags", "+faststart",
+ dst,
+ ];
+}
+
+export function posterArgs(src: string, dst: string, atSec: number): string[] {
+ return [
+ "-nostdin", "-hide_banner", "-v", "error", "-y",
+ "-ss", String(Math.max(0, Math.round(atSec * 100) / 100)),
+ "-i", src,
+ "-frames:v", "1",
+ "-vf", "scale='min(1280,iw)':-2",
+ "-q:v", "3",
+ dst,
+ ];
+}
+
+export async function probeVideo(ffprobeBin: string, file: string): Promise<VideoProbe> {
+ const { stdout } = await execa(ffprobeBin, [
+ "-v", "error", "-print_format", "json", "-show_format", "-show_streams", file,
+ ]);
+ const doc = JSON.parse(stdout) as {
+ format?: { duration?: string; format_name?: string };
+ streams?: { codec_type?: string; codec_name?: string; width?: number; duration?: string }[];
+ };
+ const streams = doc.streams ?? [];
+ const video = streams.find((s) => s.codec_type === "video");
+ const audio = streams.find((s) => s.codec_type === "audio");
+ const duration = Number(doc.format?.duration ?? video?.duration ?? NaN);
+ return {
+ durationSec: Number.isFinite(duration) ? duration : 0,
+ formatName: doc.format?.format_name ?? "",
+ videoCodec: video?.codec_name ?? null,
+ audioCodec: audio?.codec_name ?? null,
+ width: video?.width ?? null,
+ };
+}
+
+export type AttachVideoOptions = {
+ reportFile: string;
+ video: string;
+ poster?: string;
+ caption?: string;
+ ffmpegBin: string;
+ ffprobeBin: string;
+ limitBytes?: number;
+ onLog?: (line: string) => void;
+};
+
+export type AttachedVideo = {
+ video: { src: string; poster?: string; caption?: string };
+ bytes: number;
+ mode: "remux" | "encode";
+ attempts: number;
+};
+
+export class AttachVideoError extends Error {}
+
+const sizeOf = async (p: string) => (await stat(p).catch(() => null))?.size ?? null;
+
+export async function attachReportVideo(opts: AttachVideoOptions): Promise<AttachedVideo> {
+ const log = opts.onLog ?? (() => {});
+ const limit = opts.limitBytes ?? PUBLISH_MAX_FILE_BYTES;
+ const dir = path.dirname(path.resolve(opts.reportFile));
+
+ // The report first: nothing is encoded for a file that is not one.
+ let raw: Record<string, unknown>;
+ try {
+ raw = JSON.parse(await readFile(opts.reportFile, "utf8")) as Record<string, unknown>;
+ } catch (err) {
+ throw new AttachVideoError(`${opts.reportFile} is not readable JSON (${(err as Error).message})`);
+ }
+ if (!parseReport(raw).ok) throw new AttachVideoError(`${opts.reportFile} is not a report (archilyzer reports check)`);
+
+ const srcBytes = await sizeOf(opts.video);
+ if (srcBytes === null) throw new AttachVideoError(`${opts.video} does not exist`);
+ if (opts.poster !== undefined) {
+ if (!POSTER_EXTS.has(path.extname(opts.poster).toLowerCase())) {
+ throw new AttachVideoError(`the poster must be a png, jpg or webp: ${opts.poster}`);
+ }
+ if ((await sizeOf(opts.poster)) === null) throw new AttachVideoError(`${opts.poster} does not exist`);
+ }
+
+ const probe = await probeVideo(opts.ffprobeBin, opts.video);
+ let plan = encodePlan(probe, srcBytes, limit);
+ if (plan.mode === "refuse") throw new AttachVideoError(`${opts.video} cannot be attached: ${plan.reason}`);
+ log(
+ plan.mode === "remux"
+ ? `${path.basename(opts.video)}: H.264 and under the limit — remuxing (+faststart)`
+ : `${path.basename(opts.video)}: ${probe.durationSec.toFixed(1)} s ${probe.videoCodec}/${probe.audioCodec ?? "no audio"} — encoding at ${plan.videoKbps} kb/s video, ${plan.audioKbps} kb/s audio, ≤${plan.maxWidth} px wide`,
+ );
+
+ const dst = path.join(dir, REPORT_VIDEO_NAME);
+ const tmp = path.join(dir, `.${REPORT_VIDEO_NAME}.${process.pid}.tmp.mp4`);
+ let bytes = 0;
+ let attempts = 0;
+ try {
+ for (;;) {
+ attempts++;
+ await execa(opts.ffmpegBin, encodeArgs(opts.video, tmp, plan));
+ bytes = (await sizeOf(tmp)) ?? 0;
+ if (bytes > 0 && bytes <= limit) break;
+ if (plan.mode !== "encode" || attempts >= MAX_ENCODE_ATTEMPTS) {
+ throw new AttachVideoError(
+ publishFileSizeProblem(REPORT_VIDEO_NAME, bytes) ?? `ffmpeg wrote nothing for ${opts.video}`,
+ );
+ }
+ const scaled = Math.floor(plan.videoKbps * ((limit * ATTACH_TARGET_FRACTION) / bytes) * 0.95);
+ if (scaled < MIN_VIDEO_KBPS) {
+ throw new AttachVideoError(`${opts.video} does not fit the limit above ${MIN_VIDEO_KBPS} kb/s of picture — trim it`);
+ }
+ log(` ${(bytes / (1024 * 1024)).toFixed(1)} MiB is over the limit — again at ${scaled} kb/s`);
+ plan = { ...plan, videoKbps: scaled };
+ }
+ await rename(tmp, dst);
+ } finally {
+ await rm(tmp, { force: true });
+ }
+ log(` ${REPORT_VIDEO_NAME}: ${(bytes / (1024 * 1024)).toFixed(2)} MiB (limit ${(limit / (1024 * 1024)).toFixed(0)} MiB)`);
+
+ // The poster: given, kept, or drawn from the video.
+ const existing = (raw.video as { poster?: unknown; caption?: unknown } | undefined) ?? undefined;
+ let poster: string | undefined;
+ if (opts.poster !== undefined) {
+ poster = `poster${path.extname(opts.poster).toLowerCase()}`;
+ if (path.resolve(opts.poster) !== path.join(dir, poster)) await copyFile(opts.poster, path.join(dir, poster));
+ } else if (typeof existing?.poster === "string" && (await sizeOf(path.join(dir, existing.poster))) !== null) {
+ poster = existing.poster;
+ } else {
+ poster = REPORT_POSTER_NAME;
+ await execa(opts.ffmpegBin, posterArgs(dst, path.join(dir, poster), probe.durationSec * 0.1));
+ }
+ const posterProblem = publishFileSizeProblem(poster, (await sizeOf(path.join(dir, poster))) ?? 0);
+ if (posterProblem) throw new AttachVideoError(posterProblem);
+
+ const caption = opts.caption ?? (typeof existing?.caption === "string" ? existing.caption : undefined);
+ const video = { src: REPORT_VIDEO_NAME, poster, ...(caption ? { caption } : {}) };
+ const next = { ...raw, video };
+ // Only the video's own problems refuse the write: a report with others (a
+ // draft's dangling link) gets its video all the same, and `reports check`
+ // says the rest.
+ const parsed = parseReport(next);
+ const videoProblems = parsed.problems.filter((p) => p.path === "video" || p.path.startsWith("video."));
+ if (!parsed.ok || videoProblems.length > 0) {
+ const first = videoProblems[0] ?? parsed.problems[0];
+ throw new AttachVideoError(`report.json would not validate with the video: ${first ? `${first.path}: ${first.message}` : "?"}`);
+ }
+ const tmpJson = `${opts.reportFile}.${process.pid}.tmp`;
+ await writeFile(tmpJson, `${JSON.stringify(next, null, 2)}\n`);
+ await rename(tmpJson, opts.reportFile);
+ return { video, bytes, mode: plan.mode, attempts };
+}
diff --git a/common/publish/stageRun.test.ts b/common/publish/stageRun.test.ts
@@ -223,9 +223,17 @@ test("a live holder of the publish lock is waited for; a cancel during the wait
const logs: string[] = [];
const running = runStage(
{ kind: "build-site", target: "jer", runId: "t" },
- { paths, signal: ac.signal, onLog: (l) => logs.push(l), lockEnv: { pollMs: 5 } },
+ {
+ paths,
+ signal: ac.signal,
+ // Cancel once the wait is announced — a fixed delay raced the announcement under load.
+ onLog: (l) => {
+ logs.push(l);
+ if (/waiting for the publish lock/.test(l)) ac.abort();
+ },
+ lockEnv: { pollMs: 5 },
+ },
);
- setTimeout(() => ac.abort(), 50);
const r = await running;
assert.equal(r.code, 130);
assert.match(logs.join(""), /waiting for the publish lock — held by build-site x/);
diff --git a/common/ytdlp/ffmpegStreamClassify.test.ts b/common/ytdlp/ffmpegStreamClassify.test.ts
@@ -0,0 +1,70 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { classifyFfmpegProbe } from "./ffmpegStreamClassify";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test ytdlp/ffmpegStreamClassify.test.ts
+//
+// The ffmpeg probe-result classifier. Pure: an exit code and a stderr string
+// in, a verdict out. (These lived in the editor's e2e suite as
+// audio-check-classifier.spec.ts, booting a dev server they never used.)
+
+const DECODER_ERROR =
+ "[aist#0:1/aac @ 0x2] [dec:aac @ 0x3] Error submitting packet to decoder: Invalid data found when processing input\n";
+const PARTIAL_FILE =
+ "[in#0/mov,mp4,m4a,3gp,3g2,mj2 @ 0x4] stream 1, offset 0x1626f8a: partial file\n";
+
+test("exit 0 with empty stderr → clean", () => {
+ assert.equal(classifyFfmpegProbe(0, ""), "clean");
+ assert.equal(classifyFfmpegProbe(0, "\n \t\n"), "clean");
+});
+
+test("exit 0 with 'partial file' stderr → partial", () => {
+ assert.equal(
+ classifyFfmpegProbe(
+ 0,
+ "[in#0/mov,mp4,m4a,3gp,3g2,mj2 @ 0x1234] stream 1, offset 0x10483924: partial file\n",
+ ),
+ "partial",
+ );
+});
+
+test("exit 0 with many decoder errors and no 'partial file' → malformed", () => {
+ // ffmpeg can exit 0 even when the av_codec layer rejects hundreds of
+ // packets — the encoder keeps producing output from whatever decoded. A wall
+ // of "Error submitting packet to decoder" lines without a "partial file"
+ // demuxer warning is mid-stream corruption, not a clean truncation.
+ const aacStorm = Array.from(
+ { length: 50 },
+ (_, i) =>
+ `[aac @ 0x1] channel element ${i % 3}.${i % 16} is not allocated\n` +
+ DECODER_ERROR,
+ ).join("");
+ assert.equal(classifyFfmpegProbe(0, aacStorm), "malformed");
+});
+
+test("exit 0 with many decoder errors AND 'partial file' → malformed (corruption wins over truncation)", () => {
+ const stormPlusPartial =
+ Array.from({ length: 50 }, () => DECODER_ERROR).join("") + PARTIAL_FILE;
+ assert.equal(classifyFfmpegProbe(0, stormPlusPartial), "malformed");
+});
+
+test("exit 0 with a small tail of decoder errors AND 'partial file' → partial", () => {
+ // Truncated containers often emit a couple of trailing decoder errors as the
+ // encoder eats the last partial packets. Below the threshold the file is
+ // still classifiable as partial.
+ const tail = DECODER_ERROR + DECODER_ERROR + PARTIAL_FILE;
+ assert.equal(classifyFfmpegProbe(0, tail), "partial");
+});
+
+test("non-zero exit → malformed (regardless of stderr)", () => {
+ assert.equal(
+ classifyFfmpegProbe(
+ 1,
+ "[aac @ 0x1] Sample rate index in program config element does not match the sample rate index configured by the container.\n",
+ ),
+ "malformed",
+ );
+ assert.equal(classifyFfmpegProbe(2, ""), "malformed");
+ assert.equal(classifyFfmpegProbe(null, "killed by signal"), "malformed");
+});
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -4,6 +4,9 @@
- **A home seeder of last resort, behind a VPN.** `archilyzer seed` seeds the playable torrents of the sites named in the new `settings.seeder` (`sites`, `trackers`, `maxUploadKiBps`, `maxConnections`, `pollSeconds`, `standbyAfterSeconds`, `bindInterface`; SETTINGS.md) to desktop clients over TCP and to browsers over WebRTC — but each torrent only while no other seeder has it: other seeders seen on every poll for `standbyAfterSeconds` puts that torrent on standby (it stops announcing and closes its peers, keeping the data), and it comes back at once when a leecher is waiting with no other source, or after the same window with no other seeder. Every change is logged with its reason. No DHT, no local discovery, no UPnP. `archilyzer tracker` is a self-hosted HTTP + WebSocket tracker that tracks only those torrents. `docker-compose.seeder.yml` (profile `seeder`) runs both inside a WireGuard container's network namespace (gluetun, its firewall always on), so a tunnel that is down means no network, never the home connection; the WireGuard config is yours (`SEEDER_WG_CONF`, required, mounted read-only). `archilyzer doctor` compares the seeder's egress address with the host's and fails when they are the same; it says "seeder not configured" until `seeder.sites` names a site.
- **Saved videos can be made browser-playable, with a torrent each.** `pnpm ops prepare-playable` (`POST /api/ops/prepare-playable`) and `archilyzer media playable <slug>` remux each of a channel's saved containers — without re-encoding (`-c copy`) — into an mp4 with its index in front, or a webm when it already is one (VP9/AV1 with Opus), drop subtitles, metadata and chapters, and make one single-file torrent of the copy: named `<id>.<ext>`, no web seed, no comment, no "created by", 256 KiB–1 MiB pieces. They go to `playable/<slug>/<id>/` beside the saved-video store, listed in `playable/<slug>/playable.json` with each infohash. `"trackers"` is the announce list written into each torrent (none by default); it is not part of the infohash, so the same torrent can be announced elsewhere later. A video already prepared from the same source (by sha256) is skipped, so a re-run is a no-op; a codec a browser cannot play without re-encoding (HEVC, MPEG-4 Part 2) is listed and left alone.
- **A channel's videos can get their media from a local archive.** `pnpm ops attach-media` (`POST /api/ops/attach-media`) and `archilyzer media attach <slug> <source>` take `{"slug", "source"}` — an absolute path to a directory, a `.zip` (read in place: a stored entry is copied straight out of it, with no temp dir) or a `.7z` — and put each held video's file into the saved-video store as its source container, so clip windows and report clips can be cut from it with nothing fetched. The id is the folder's trailing `(<id>)`, else the file's yt-dlp suffix; `"items": [{"id", "path"}]` names exact files and `"match"` narrows the folders. The pointer records where the file came from: `origin: {kind: "local-archive", archive, entry, sha256, attachedAt}`. A video that already has a saved container is left alone unless `"replace": true`; `"createRecords": true` writes a record for a video the channel does not hold (from the folder's yt-dlp `.info.json`, else its `description.txt` and name); `[LOST]` folders are listed and never attached. `"dryRun": true` lists what each folder is, and the held videos the archive has no media for, and writes nothing. The archive is never written to.
+- **Reports can be checked, their quotes verified and their video attached from the command line.** `archilyzer reports check <site> [--reports a,b] [--allow-missing-media]` says what the site's compose would say about its reports — each report validated, every cited quote checked against its record, every cited moment's prepared media present and current, the report's video under the publish limit — without a build, and exits 1 with the list; `--reports` checks those reports, drafts not yet in site.json included. `archilyzer reports verify-quotes <report.json> [--json]` runs compose's own quote check over every video, audio and post quote of one report and prints each one's score and the transcript it matched best, and, where the record has an `en-orig` track, that track's score: a quote that matches a served `en` track but not `en-orig` — the words as spoken — is reported, though compose would pass it. `archilyzer reports attach-video <report.json> <video> [--poster <image>] [--caption <line>]` makes the video the report's: an H.264 mp4 under the 24 MiB limit is remuxed, anything else encoded to fit (refused, with its length, when it would be unwatchable), written beside report.json as `video.mp4` with a poster — the one given, the one it had, or a frame of the video — and `video` set in report.json.
+- **A file transcription is timed from the engine, and a short one does not queue behind batches.** `pnpm ops transcribe`'s `durationMs` is now the engine's own time — from the moment a worker took the job — and the new `waitedMs` is how long it waited for a free worker; the job's log says both. A file of up to 15 minutes of audio (a window, usually) waits in the worker pool ahead of every parked transcription, a manual batch's next video included, not only ahead of the transcription lane; a longer file keeps its place behind batches queued before it. Nothing running is interrupted. Needs a restart of the editor.
+- **Heavy work takes turns, above a memory floor.** A publish stage's `next build` — a site's, the hub's, the homepage's — now waits for the machine's one heavy slot, which every e2e run and any `pnpm heavy -- <cmd>` (a video render) take too, and then until at least 6000 MB is available; the stage's log says whom it waits behind ("waiting for the heavy slot — held by …") or how much memory there is ("waiting for memory — 4210 MB available, the floor is 6000 MB"). Cancel still stops it. `HEAVY_MIN_FREE_MB` moves the floor (`0` turns it off) and `HEAVY=0` skips the gate. Needs a restart of the editor.
- **A curated tag can exist on some sites only.** A tag's new **Sites** field on /tags (`sites` in `transcripts/tags.json`; `pnpm ops tags` takes it in a define) names the sites it exists on. Its rules then fire, and its pins apply, only to videos on those sites' channels, and every other site drops it from its records, its counts and its `/tags.json` — where **Hidden** only hid the chip. Empty is every site, as before. Setting it, or changing the channels of those sites, re-derives the corpus's tags once at the next index update. The Eva tags are what this is for: they belong on Anilyzer alone.
- **The publish lane.** Publishing can run itself: turn it on at **/operations/publish** (the runner's Start, Drain and Stop, the hold, and the lane's settings; or `publish.enabled` in settings) and the lane checks every `checkEveryMinutes` (10) whether the index is stale; when it is — and its last update is at least `refreshEveryMinutes` (360) old — it updates it, then builds every site whose channels changed or whose data the new index moved, one stage at a time on the `publish` queue. What it may do with a site is the site's own — the **Publish policy** on the site's settings form, `site.json` `publish.auto` —: `off` (the default: left alone), `build`, `preview` (built and deployed to the preview branch `publish.previewBranch`) or `production`; the hub and the homepage have `publish.hub` and `publish.homepage`. A private site is only ever built, and a site needs its Cloudflare Pages project before it may deploy. Hold the lane and the stage running finishes and no next one starts; quiet hours (`publish.quietHours`) do the same; Drain finishes the stage and ends the runner. The lane never forces a stage: a stage that finds its target current does nothing. On /jobs every stage of one run reads `run <id> · <target>`, and a stage still queued when the editor restarts is cancelled, never re-queued — the lane works out again what is stale from what is on disk. `archilyzer publish now` runs the same plan from the command line, one stage after another in its own process.
- **One index for every site.** The index is updated once and every site, the hub and the homepage are built from it; `archilyzer publish status` says, per site, whether its build is current — "stale: 3 channels changed (a, b, c)" as soon as a download, transcription or digest on one of its channels finishes, before any index runs; "stale: data changed" once the index has run and the site's data moved; "stale: config changed" after its site.json, tags or aliases changed — and whether what is deployed is that build, with a build made by older code marked "code newer" but not stale.
diff --git a/editor/app/api/channels/[slug]/videos/[id]/files/[name]/route.test.ts b/editor/app/api/channels/[slug]/videos/[id]/files/[name]/route.test.ts
@@ -0,0 +1,68 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+// Run with:
+// pnpm -C editor exec tsx --test "app/api/channels/*/videos/*/files/*/route.test.ts"
+//
+// The media file route's byte ranges, in-process. The ABORT regression
+// (`Controller is already closed` when a scrubbing browser cancels range
+// requests) stays e2e in media-file-abort.spec.ts: a burst of cancelled
+// bodies run against this handler in-process passes even with the naive
+// wrapper that caused it — the race is in the server's response pipeline, not
+// in the handler — so only the HTTP round trip is a real test of it. The guard
+// itself is unit-tested in common/lib/safeStreamController.test.ts.
+
+const ROOT = await mkdtemp(path.join(os.tmpdir(), "media-file-route-"));
+const CORPUS = path.join(ROOT, "transcripts");
+const VIDEO_DIR = path.join(CORPUS, "channels", "chan", "data", "vidA");
+await mkdir(VIDEO_DIR, { recursive: true });
+await writeFile(path.join(VIDEO_DIR, "audio.m4a"), Buffer.alloc(1024 * 1024, 7));
+// Set before anything that caches getPaths() is first imported.
+process.env.TRANSCRIPTS_DIR = CORPUS;
+process.env.SETTINGS_FILE = path.join(ROOT, "settings.json");
+const { GET } = await import("./route");
+test.after(() => rm(ROOT, { recursive: true, force: true }));
+
+function get(range?: string) {
+ return GET(
+ new Request("http://localhost/api/channels/chan/videos/vidA/files/audio.m4a", {
+ headers: range ? { range } : {},
+ }),
+ { params: Promise.resolve({ slug: "chan", id: "vidA", name: "audio.m4a" }) },
+ );
+}
+
+test("a range request is a 206 with the bytes asked for", async () => {
+ const res = await get("bytes=100-4195");
+ assert.equal(res.status, 206);
+ assert.equal(res.headers.get("content-range"), "bytes 100-4195/1048576");
+ const body = Buffer.from(await res.arrayBuffer());
+ assert.equal(body.length, 4096);
+});
+
+test("a suffix range is the last N bytes; no or a bad range is the whole file", async () => {
+ const suffix = await get("bytes=-100");
+ assert.equal(suffix.status, 206);
+ assert.equal(suffix.headers.get("content-range"), "bytes 1048476-1048575/1048576");
+ assert.equal((await suffix.arrayBuffer()).byteLength, 100);
+ for (const range of [undefined, "bytes=5-2", "bytes=0-9999999", "lines=1-2"]) {
+ const res = await get(range);
+ assert.equal(res.status, 200, String(range));
+ assert.equal(res.headers.get("content-length"), "1048576");
+ await res.body!.cancel();
+ }
+});
+
+test("a path that leaves the video dir is refused; a missing file is a 404", async () => {
+ const bad = await GET(new Request("http://localhost/x"), {
+ params: Promise.resolve({ slug: "chan", id: "..%2F..", name: "audio.m4a" }),
+ });
+ assert.equal(bad.status, 400);
+ const missing = await GET(new Request("http://localhost/x"), {
+ params: Promise.resolve({ slug: "chan", id: "vidA", name: "nope.m4a" }),
+ });
+ assert.equal(missing.status, 404);
+});
diff --git a/editor/app/api/ops/transcribe/route.ts b/editor/app/api/ops/transcribe/route.ts
@@ -17,7 +17,8 @@ export const dynamic = "force-dynamic";
//
// The job's log ends with the result as ONE line, `@@transcribe-result
// {json}`: `{version, path, window, worker: {id, name, appId, model, device},
-// transcriptFormat, transcribedAt, durationMs, cues: [{start, end, text}],
+// transcriptFormat, transcribedAt, durationMs (the engine's time), waitedMs (the
+// wait for a free worker), cues: [{start, end, text}],
// text}`, cue times on the SOURCE file's clock. `pnpm ops transcribe --wait`
// prints that JSON on stdout. `out` writes it to that file as well.
//
diff --git a/editor/app/api/view/[name]/route.test.ts b/editor/app/api/view/[name]/route.test.ts
@@ -0,0 +1,126 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, readdir, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { VIEW_NAMES } from "yt-dlp-transcript-common/views/names";
+
+// Run with:
+// pnpm -C editor exec tsx --test "app/api/view/[name]/route.test.ts"
+//
+// ONE POLLING ROUTE, AND THE OLD PATHS THAT STILL ANSWER — the parts of that
+// contract a handler call can check. They were e2e (view-route.spec.ts) and
+// booted a server to ask a dispatcher for a 404.
+//
+// - the eight old paths are REWRITES to their view, and nothing else is: the
+// table in next.config.ts, read as data. A rewrite to the right view is
+// the same endpoint by construction, which is what the e2e compared bodies
+// to prove.
+// - an unknown name, a near-miss and every /api/test/* harness name are 404s
+// from the dispatcher, before any input is built.
+// - /api/widget/presets keeps its own route and answers.
+//
+// What stays e2e is the one thing only a running Next can show: that a
+// rewrite carries the QUERY STRING (`/api/pulse?rev=` — e2e/view-route.spec.ts).
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const EDITOR = path.resolve(HERE, "..", "..", "..", "..");
+
+const ROOT = await mkdtemp(path.join(os.tmpdir(), "view-route-"));
+const CORPUS = path.join(ROOT, "transcripts");
+await mkdir(path.join(CORPUS, "channels"), { recursive: true });
+// Set before anything that caches getPaths() is first imported.
+process.env.TRANSCRIPTS_DIR = CORPUS;
+process.env.SETTINGS_FILE = path.join(ROOT, "settings.json");
+const { GET } = await import("./route");
+const { GET: presetsGET } = await import("../../widget/presets/route");
+// next.config.ts says `__dirname`, which Next's config loader provides and an
+// ES module does not; a global of that name is what the free identifier finds.
+(globalThis as { __dirname?: string }).__dirname = EDITOR;
+const { default: nextConfig } = await import("../../../../next.config");
+delete (globalThis as { __dirname?: string }).__dirname;
+test.after(() => rm(ROOT, { recursive: true, force: true }));
+
+const PAIRS: Array<[string, string]> = [
+ ["/api/pulse", "/api/view/pulse"],
+ ["/api/jobs/active", "/api/view/activeJobs"],
+ ["/api/workers", "/api/view/workers"],
+ ["/api/auto-queue/status", "/api/view/autoQueueStatus"],
+ ["/api/scheduler/status", "/api/view/schedulerStatus"],
+ ["/api/widget/sync", "/api/view/widgetSync"],
+ ["/api/widget/actionable", "/api/view/widgetActionable"],
+ ["/api/widget/cleanable", "/api/view/cleanable"],
+];
+
+async function view(name: string): Promise<number> {
+ const res = await GET(new Request(`http://localhost/api/view/${name}`), {
+ params: Promise.resolve({ name }),
+ });
+ return res.status;
+}
+
+type Rewrite = { source: string; destination: string };
+
+async function rewrites(): Promise<Rewrite[]> {
+ const r = await nextConfig.rewrites!();
+ // The array form is `afterFiles`; the object form would split it.
+ assert.ok(Array.isArray(r), "next.config rewrites() is the array form");
+ return r as Rewrite[];
+}
+
+test("each old path is a rewrite to its view, and every view has one", async () => {
+ const table = await rewrites();
+ const apiRewrites = table.filter((r) => r.source.startsWith("/api/"));
+ assert.deepEqual(
+ apiRewrites.map((r) => [r.source, r.destination]),
+ PAIRS,
+ );
+ // Every destination is a name the dispatcher serves — a typo here would be a
+ // rewrite to a 404.
+ for (const [, viewPath] of PAIRS) {
+ const name = viewPath.slice("/api/view/".length);
+ assert.ok(
+ (VIEW_NAMES as readonly string[]).includes(name),
+ `${viewPath} is not a view`,
+ );
+ }
+ assert.deepEqual(
+ [...VIEW_NAMES].sort(),
+ PAIRS.map(([, v]) => v.slice("/api/view/".length)).sort(),
+ );
+});
+
+test("an unknown view name is 404, not 500", async () => {
+ for (const name of ["nope", "Pulse", "activejobs", "presets"]) {
+ assert.equal(await view(name), 404, `/api/view/${name}`);
+ }
+});
+
+// The dispatcher has no guard by design (these are read-only polls), but the
+// harness routes DO — and none may be reachable through it.
+test("no test-harness name is a view", async () => {
+ const harness = (
+ await readdir(path.join(EDITOR, "app", "api", "test"), {
+ withFileTypes: true,
+ })
+ )
+ .filter((d) => d.isDirectory())
+ .map((d) => d.name);
+ assert.ok(harness.includes("invalidate-cache"), "the harness dir was read");
+ for (const name of harness) {
+ assert.equal(await view(name), 404, `/api/view/${name}`);
+ }
+});
+
+// /api/widget/presets is a menu fetch on open, not a poll: it is not a view,
+// it keeps its own route, and no rewrite shadows it.
+test("/api/widget/presets is untouched", async () => {
+ const table = await rewrites();
+ assert.ok(!table.some((r) => r.source === "/api/widget/presets"));
+ const res = await presetsGET();
+ assert.equal(res.status, 200);
+ const body = (await res.json()) as { builtIn: unknown; saved: unknown };
+ assert.ok(Array.isArray(body.builtIn));
+ assert.ok(Array.isArray(body.saved));
+});
diff --git a/editor/app/api/worker/unit/route.test.ts b/editor/app/api/worker/unit/route.test.ts
@@ -0,0 +1,197 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import http from "node:http";
+import type { AddressInfo } from "node:net";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+// Run with:
+// pnpm -C editor exec tsx --test "app/api/worker/unit/route.test.ts"
+//
+// The unit-executor protocol (/api/worker/unit) — the generalisation of the
+// remote-transcription protocol to backfill kinds — driven through its four
+// route handlers in-process, in a temp corpus. (It was e2e, worker-unit.spec.ts,
+// and needed nothing of the server but these handlers.) Three angles:
+// 1. Auth + the door guard (only backfill KINDS are accepted — download and
+// transcription are refused, which is what keeps download politeness
+// single-machine).
+// 2. A full round trip: an attribution-text unit whose model calls land on an
+// ollama STUB started here — proving the scratch-corpus materialization
+// (cues written last passes the mtime freshness gate), the config
+// injection, and the result pull, with no real model anywhere.
+// 3. Cleanup: DELETE removes the scratch and the result 404s.
+
+const ROOT = await mkdtemp(path.join(os.tmpdir(), "worker-unit-route-"));
+const CORPUS = path.join(ROOT, "transcripts");
+await mkdir(path.join(CORPUS, "channels"), { recursive: true });
+const SETTINGS_FILE = path.join(ROOT, "settings.json");
+await writeFile(SETTINGS_FILE, JSON.stringify({ workers: [] }));
+const TOKEN = "test-worker-token";
+// Set before anything that caches getPaths() or the token is first imported.
+process.env.WORKER_TOKEN = TOKEN;
+process.env.TRANSCRIPTS_DIR = CORPUS;
+process.env.SETTINGS_FILE = SETTINGS_FILE;
+const { POST } = await import("./route");
+const { DELETE } = await import("./[id]/route");
+const { GET: eventsGET } = await import("./[id]/events/route");
+const { GET: resultGET } = await import("./[id]/result/route");
+test.after(() => rm(ROOT, { recursive: true, force: true }));
+
+const AUTH = { authorization: `Bearer ${TOKEN}` };
+const BASE = "http://localhost/api/worker/unit";
+
+function post(body: unknown, headers: Record<string, string> = AUTH) {
+ return POST(
+ new Request(BASE, {
+ method: "POST",
+ headers: { ...headers, "content-type": "application/json" },
+ body: JSON.stringify(body),
+ }),
+ );
+}
+
+function byId(id: string) {
+ return { params: Promise.resolve({ id }) };
+}
+
+test("the unit endpoint enforces the bearer token and refuses non-kinds", async () => {
+ const noAuth = await post(
+ { op: "attribution-text", channelSlug: "c", videoId: "v", files: {} },
+ {},
+ );
+ assert.equal(noAuth.status, 401);
+
+ // download/transcription are ExternalOperations, not backfill kinds — the
+ // executor refuses them at the door.
+ for (const op of ["download", "transcription", "nonsense"]) {
+ const refused = await post({
+ op,
+ channelSlug: "c",
+ videoId: "v",
+ files: {},
+ target: {},
+ });
+ assert.equal(refused.status, 400, op);
+ }
+});
+
+test("an attribution unit round-trips against a scratch corpus and a stub ollama", async () => {
+ // A fake ollama the EXECUTOR's injected appConfig.baseUrl points at. The
+ // /api/chat reply names one speaker, in the schema the turn prompt pins.
+ const stub = http.createServer((req, res) => {
+ res.setHeader("content-type", "application/json");
+ if (req.url?.startsWith("/api/tags")) {
+ res.end(JSON.stringify({ models: [{ name: "stub-model" }] }));
+ return;
+ }
+ // Drain the request, then answer as ollama would.
+ req.resume();
+ req.on("end", () => {
+ res.end(
+ JSON.stringify({
+ model: "stub-model",
+ message: {
+ content: JSON.stringify({
+ turns: [{ start: "00:00:01", speaker: "Host" }],
+ }),
+ },
+ }),
+ );
+ });
+ });
+ await new Promise<void>((resolve) => stub.listen(0, "127.0.0.1", resolve));
+ const stubUrl = `http://127.0.0.1:${(stub.address() as AddressInfo).port}`;
+
+ try {
+ const cues = {
+ version: 1,
+ id: "unitvid1",
+ title: "Unit test video",
+ channel: "unit-chan",
+ duration: 9,
+ cues: [
+ { start: 0, end: 4, text: "hello there" },
+ { start: 4, end: 9, text: "general kenobi" },
+ ],
+ };
+ const b64 = (s: string) => Buffer.from(s).toString("base64");
+ const res = await post({
+ op: "attribution-text",
+ channelSlug: "unit-chan",
+ videoId: "unitvid1",
+ files: {
+ "metadata.info.json": b64(
+ JSON.stringify({ id: "unitvid1", title: "Unit test video", duration: 9 }),
+ ),
+ "transcript.json": b64(JSON.stringify({ transcription: [] })),
+ // Materialized LAST by the executor whatever this map's order is —
+ // the mtime freshness gate depends on it.
+ "transcript.cues.json": b64(JSON.stringify(cues)),
+ },
+ target: {},
+ config: {
+ // The primary's identity, injected. Without this the executor's
+ // default settings (attribution disabled) would fail the job loudly.
+ attribution: {
+ enabled: true,
+ appId: "ollama-direct",
+ model: "stub-model",
+ diarizedEnabled: false,
+ textOnlyEnabled: true,
+ promptVersion: 2,
+ },
+ appConfig: { model: "stub-model", baseUrl: stubUrl, numCtx: 8192 },
+ context: { hash: "none" },
+ },
+ });
+ assert.equal(res.status, 202);
+ const { remoteJobId } = (await res.json()) as { remoteJobId: string };
+ assert.ok(remoteJobId);
+
+ const deadline = Date.now() + 30_000;
+ let status = "";
+ while (Date.now() < deadline) {
+ const ev = await eventsGET(
+ new Request(`${BASE}/${remoteJobId}/events`, { headers: AUTH }),
+ byId(remoteJobId),
+ );
+ status = ((await ev.json()) as { status: string }).status;
+ if (status === "done" || status === "error") break;
+ await new Promise((r) => setTimeout(r, 100));
+ }
+ assert.equal(status, "done");
+
+ const result = await resultGET(
+ new Request(`${BASE}/${remoteJobId}/result`, { headers: AUTH }),
+ byId(remoteJobId),
+ );
+ assert.equal(result.status, 200);
+ const body = (await result.json()) as {
+ outcome: string;
+ files: Record<string, string>;
+ };
+ assert.equal(body.outcome, "done");
+ const record = JSON.parse(body.files["attribution.json"]) as {
+ speakers: Array<{ label: string }>;
+ provenance: { method: string; model: string };
+ };
+ assert.equal(record.speakers[0]?.label, "Host");
+ assert.equal(record.provenance.method, "text-only");
+ assert.equal(record.provenance.model, "stub-model");
+
+ // Cleanup removes the scratch corpus; the result then 404s.
+ const del = await DELETE(
+ new Request(`${BASE}/${remoteJobId}`, { method: "DELETE", headers: AUTH }),
+ byId(remoteJobId),
+ );
+ assert.equal(del.status, 200);
+ const gone = await resultGET(
+ new Request(`${BASE}/${remoteJobId}/result`, { headers: AUTH }),
+ byId(remoteJobId),
+ );
+ assert.equal(gone.status, 404);
+ } finally {
+ await new Promise<void>((resolve) => stub.close(() => resolve()));
+ }
+});
diff --git a/editor/e2e/audio-check-classifier.spec.ts b/editor/e2e/audio-check-classifier.spec.ts
@@ -1,70 +0,0 @@
-// Pure-function tests for the ffmpeg probe-result classifier. These don't
-// need the dev server, fixtures, or a browser — but the project uses
-// Playwright for everything, so they live here too.
-
-import { test, expect } from "@playwright/test";
-import { classifyFfmpegProbe } from "../../common/ytdlp/ffmpegStreamClassify";
-
-test.describe("classifyFfmpegProbe", () => {
- test("exit 0 with empty stderr → clean", () => {
- expect(classifyFfmpegProbe(0, "")).toBe("clean");
- expect(classifyFfmpegProbe(0, "\n \t\n")).toBe("clean");
- });
-
- test("exit 0 with 'partial file' stderr → partial", () => {
- expect(
- classifyFfmpegProbe(
- 0,
- "[in#0/mov,mp4,m4a,3gp,3g2,mj2 @ 0x1234] stream 1, offset 0x10483924: partial file\n",
- ),
- ).toBe("partial");
- });
-
- test("exit 0 with many decoder errors and no 'partial file' → malformed", () => {
- // ffmpeg can exit 0 even when the av_codec layer rejects hundreds of
- // packets — the encoder keeps producing output from whatever decoded.
- // A wall of "Error submitting packet to decoder" lines without a
- // "partial file" demuxer warning is mid-stream corruption, not a
- // clean truncation.
- const aacStorm = Array.from(
- { length: 50 },
- (_, i) =>
- `[aac @ 0x1] channel element ${i % 3}.${i % 16} is not allocated\n` +
- `[aist#0:1/aac @ 0x2] [dec:aac @ 0x3] Error submitting packet to decoder: Invalid data found when processing input\n`,
- ).join("");
- expect(classifyFfmpegProbe(0, aacStorm)).toBe("malformed");
- });
-
- test("exit 0 with many decoder errors AND 'partial file' → malformed (corruption wins over truncation)", () => {
- const aacStormPlusPartial =
- Array.from(
- { length: 50 },
- () =>
- `[aist#0:1/aac @ 0x2] [dec:aac @ 0x3] Error submitting packet to decoder: Invalid data found when processing input\n`,
- ).join("") +
- "[in#0/mov,mp4,m4a,3gp,3g2,mj2 @ 0x4] stream 1, offset 0x1626f8a: partial file\n";
- expect(classifyFfmpegProbe(0, aacStormPlusPartial)).toBe("malformed");
- });
-
- test("exit 0 with a small tail of decoder errors AND 'partial file' → partial", () => {
- // Truncated containers often emit a couple of trailing decoder errors
- // as the encoder eats the last partial packets. Below threshold, the
- // file is still classifiable as partial.
- const tail =
- "[aist#0:1/aac @ 0x2] [dec:aac @ 0x3] Error submitting packet to decoder: Invalid data found when processing input\n" +
- "[aist#0:1/aac @ 0x2] [dec:aac @ 0x3] Error submitting packet to decoder: Invalid data found when processing input\n" +
- "[in#0/mov,mp4,m4a,3gp,3g2,mj2 @ 0x4] stream 1, offset 0x1626f8a: partial file\n";
- expect(classifyFfmpegProbe(0, tail)).toBe("partial");
- });
-
- test("non-zero exit → malformed (regardless of stderr)", () => {
- expect(
- classifyFfmpegProbe(
- 1,
- "[aac @ 0x1] Sample rate index in program config element does not match the sample rate index configured by the container.\n",
- ),
- ).toBe("malformed");
- expect(classifyFfmpegProbe(2, "")).toBe("malformed");
- expect(classifyFfmpegProbe(null, "killed by signal")).toBe("malformed");
- });
-});
diff --git a/editor/e2e/view-route.spec.ts b/editor/e2e/view-route.spec.ts
@@ -8,96 +8,17 @@ import { resetData } from "./helpers";
// REWRITES in next.config.ts — server-internal, so a pinned widget and the
// dashboard keep polling exactly what they always polled.
//
-// The ~30 assertions the rest of the suite makes at the old paths are that
-// remap's real regression test; nothing there was edited. What this spec adds
-// is the part those cannot see: that each old path and its new twin return the
-// SAME BODY, that an unknown view name 404s instead of 500ing, that none of
-// this wants a credential, and that the query string survives the rewrite —
-// which is the whole of `/api/pulse?rev=`.
-
-// The fields that move between two back-to-back calls. Everything else in these
-// payloads is read from disk or from in-memory state that does not move in an
-// idle fixture, so it is compared verbatim. Dotted keys reach one level down.
-const VOLATILE: Record<string, string[]> = {
- // `builtAt` is Date.now() at build time. `disk.freeBytes` is a live statfs:
- // it is null only while the fixture's disk gate is off (minFreeDiskGB: 0).
- "/api/jobs/active": ["builtAt", "disk.freeBytes"],
- // `now` is stamped so the console can age its rows client-side.
- "/api/scheduler/status": ["now"],
-};
-
-const PAIRS: Array<[string, string]> = [
- ["/api/pulse", "/api/view/pulse"],
- ["/api/jobs/active", "/api/view/activeJobs"],
- ["/api/workers", "/api/view/workers"],
- ["/api/auto-queue/status", "/api/view/autoQueueStatus"],
- ["/api/scheduler/status", "/api/view/schedulerStatus"],
- ["/api/widget/sync", "/api/view/widgetSync"],
- ["/api/widget/actionable", "/api/view/widgetActionable"],
- ["/api/widget/cleanable", "/api/view/cleanable"],
-];
-
-function strip(body: unknown, keys: string[]): unknown {
- if (!body || typeof body !== "object") return body;
- const copy = { ...(body as Record<string, unknown>) };
- for (const key of keys) {
- const [head, tail] = key.split(".");
- if (tail === undefined) {
- delete copy[head];
- } else if (copy[head] && typeof copy[head] === "object") {
- const inner = { ...(copy[head] as Record<string, unknown>) };
- delete inner[tail];
- copy[head] = inner;
- }
- }
- return copy;
-}
+// The rewrite table, the dispatcher's 404s and the presets route are unit
+// tests now (app/api/view/[name]/route.test.ts), and the ~30 assertions the
+// rest of the suite makes at the old paths are the remap's runtime regression
+// test. What stays here is the one thing only a running Next can show: that a
+// rewrite carries the QUERY STRING, which is the whole of `/api/pulse?rev=`.
test.describe("/api/view/[name]", () => {
test.beforeEach(async () => {
await resetData("channel-with-counts");
});
- for (const [oldPath, viewPath] of PAIRS) {
- test(`${oldPath} and ${viewPath} are the same endpoint`, async ({
- request,
- }) => {
- const before = await request.get(oldPath);
- const after = await request.get(viewPath);
- expect(before.status(), `${oldPath} status`).toBe(200);
- expect(after.status(), `${viewPath} status`).toBe(200);
-
- const keys = VOLATILE[oldPath] ?? [];
- expect(strip(await after.json(), keys)).toEqual(
- strip(await before.json(), keys),
- );
- });
- }
-
- test("an unknown view name is 404, not 500", async ({ request }) => {
- for (const name of ["nope", "Pulse", "activejobs", "presets"]) {
- const res = await request.get(`/api/view/${name}`);
- expect(res.status(), `/api/view/${name}`).toBe(404);
- }
- });
-
- // The dispatcher has no guard by design (these are read-only polls), but the
- // harness routes DO — and they must not be reachable through it.
- test("a test-harness name is not a view", async ({ request }) => {
- const res = await request.get("/api/view/invalidate-cache");
- expect(res.status()).toBe(404);
- });
-
- // /api/widget/presets is a menu fetch on open, not a poll: it is not a view,
- // it keeps its own route, and nothing here shadows it.
- test("/api/widget/presets is untouched", async ({ request }) => {
- const res = await request.get("/api/widget/presets");
- expect(res.status()).toBe(200);
- const body = await res.json();
- expect(Array.isArray(body.builtIn)).toBe(true);
- expect(Array.isArray(body.saved)).toBe(true);
- });
-
test("the rev query survives the rewrite", async ({ request }) => {
const seed = await (await request.get("/api/view/pulse")).json();
expect(seed.changed).toBe(true);
diff --git a/editor/e2e/worker-unit.spec.ts b/editor/e2e/worker-unit.spec.ts
@@ -1,170 +0,0 @@
-// Unit-executor protocol (/api/worker/unit) — the generalisation of the
-// remote-transcription protocol to backfill kinds. The test server runs with
-// WORKER_TOKEN set (see package.json dev:test), so the endpoints are live.
-// Three angles:
-// 1. Auth + the door guard (only backfill KINDS are accepted — download and
-// transcription are refused, which is what keeps download politeness
-// single-machine).
-// 2. A full round trip: an attribution-text unit whose model calls land on an
-// ollama STUB started inside this test — proving the scratch-corpus
-// materialization (cues written last passes the mtime freshness gate), the
-// config injection, and the result pull, with no real model anywhere.
-// 3. Cleanup: DELETE removes the scratch and the result 404s.
-
-import http from "node:http";
-import type { AddressInfo } from "node:net";
-import { test, expect } from "@playwright/test";
-import { resetData } from "./helpers";
-import { baseUrl } from "./baseUrl";
-
-const TOKEN = "test-worker-token";
-const AUTH = { authorization: `Bearer ${TOKEN}` };
-
-test.beforeEach(async () => {
- await resetData("empty");
-});
-
-test("unit endpoint enforces the bearer token and refuses non-kinds", async ({
- request,
-}) => {
- const noAuth = await request.post(`${baseUrl}/api/worker/unit`, {
- data: { op: "attribution-text", channelSlug: "c", videoId: "v", files: {} },
- });
- expect(noAuth.status()).toBe(401);
-
- // download/transcription are ExternalOperations, not backfill kinds — the
- // executor refuses them at the door.
- for (const op of ["download", "transcription", "nonsense"]) {
- const refused = await request.post(`${baseUrl}/api/worker/unit`, {
- headers: AUTH,
- data: { op, channelSlug: "c", videoId: "v", files: {}, target: {} },
- });
- expect(refused.status(), op).toBe(400);
- }
-});
-
-test("an attribution unit round-trips against a scratch corpus and a stub ollama", async ({
- request,
-}) => {
- test.setTimeout(60_000);
- // A fake ollama the EXECUTOR's injected appConfig.baseUrl points at. The
- // /api/chat reply names one speaker, in the schema the turn prompt pins.
- const stub = http.createServer((req, res) => {
- res.setHeader("content-type", "application/json");
- if (req.url?.startsWith("/api/tags")) {
- res.end(JSON.stringify({ models: [{ name: "stub-model" }] }));
- return;
- }
- // Drain the request, then answer as ollama would.
- req.resume();
- req.on("end", () => {
- res.end(
- JSON.stringify({
- model: "stub-model",
- message: {
- content: JSON.stringify({
- turns: [{ start: "00:00:01", speaker: "Host" }],
- }),
- },
- }),
- );
- });
- });
- await new Promise<void>((resolve) => stub.listen(0, "127.0.0.1", resolve));
- const stubUrl = `http://127.0.0.1:${(stub.address() as AddressInfo).port}`;
-
- try {
- const cues = {
- version: 1,
- id: "unitvid1",
- title: "Unit test video",
- channel: "unit-chan",
- duration: 9,
- cues: [
- { start: 0, end: 4, text: "hello there" },
- { start: 4, end: 9, text: "general kenobi" },
- ],
- };
- const b64 = (s: string) => Buffer.from(s).toString("base64");
- const post = await request.post(`${baseUrl}/api/worker/unit`, {
- headers: AUTH,
- data: {
- op: "attribution-text",
- channelSlug: "unit-chan",
- videoId: "unitvid1",
- files: {
- "metadata.info.json": b64(
- JSON.stringify({ id: "unitvid1", title: "Unit test video", duration: 9 }),
- ),
- "transcript.json": b64(JSON.stringify({ transcription: [] })),
- // Materialized LAST by the executor whatever this map's order is —
- // the mtime freshness gate depends on it.
- "transcript.cues.json": b64(JSON.stringify(cues)),
- },
- target: {},
- config: {
- // The primary's identity, injected. Without this the executor's
- // default settings (attribution disabled) would fail the job loudly.
- attribution: {
- enabled: true,
- appId: "ollama-direct",
- model: "stub-model",
- diarizedEnabled: false,
- textOnlyEnabled: true,
- promptVersion: 2,
- },
- appConfig: { model: "stub-model", baseUrl: stubUrl, numCtx: 8192 },
- context: { hash: "none" },
- },
- },
- });
- expect(post.status()).toBe(202);
- const { remoteJobId } = await post.json();
- expect(remoteJobId).toBeTruthy();
-
- await expect
- .poll(
- async () => {
- const r = await request.get(
- `${baseUrl}/api/worker/unit/${remoteJobId}/events`,
- { headers: AUTH },
- );
- return ((await r.json()) as { status: string }).status;
- },
- { timeout: 30_000 },
- )
- .toBe("done");
-
- const result = await request.get(
- `${baseUrl}/api/worker/unit/${remoteJobId}/result`,
- { headers: AUTH },
- );
- expect(result.status()).toBe(200);
- const body = (await result.json()) as {
- outcome: string;
- files: Record<string, string>;
- };
- expect(body.outcome).toBe("done");
- const record = JSON.parse(body.files["attribution.json"]) as {
- speakers: Array<{ label: string }>;
- provenance: { method: string; model: string };
- };
- expect(record.speakers[0]?.label).toBe("Host");
- expect(record.provenance.method).toBe("text-only");
- expect(record.provenance.model).toBe("stub-model");
-
- // Cleanup removes the scratch corpus; the result then 404s.
- const del = await request.delete(
- `${baseUrl}/api/worker/unit/${remoteJobId}`,
- { headers: AUTH },
- );
- expect(del.status()).toBe(200);
- const gone = await request.get(
- `${baseUrl}/api/worker/unit/${remoteJobId}/result`,
- { headers: AUTH },
- );
- expect(gone.status()).toBe(404);
- } finally {
- await new Promise<void>((resolve) => stub.close(() => resolve()));
- }
-});
diff --git a/editor/package.json b/editor/package.json
@@ -9,7 +9,6 @@
"start:test": "WORKER_TOKEN=test-worker-token TRANSCRIPTS_DIR=$(pwd)/test-transcripts EXPORT_PUBLIC_DIR=$(pwd)/test-transcripts/.export-public SETTINGS_FILE=$(pwd)/test-settings.json EDITOR_CHANGELOG_FILE=$(pwd)/test-changelog.md EXPORT_CHANGELOG_FILE=$(pwd)/test-export-changelog.md YTDLP_BIN=$(pwd)/e2e/fixtures/bin/fake-ytdlp.mjs GALLERY_DL_BIN=$(pwd)/e2e/fixtures/bin/fake-gallery-dl.mjs WHISPER_BIN=$(pwd)/e2e/fixtures/bin/fake-whisper.mjs WHISPER_MODEL=/dev/null CHOUGH_BIN=$(pwd)/e2e/fixtures/bin/fake-chough.mjs CHOUGH_MODEL=/dev/null PARAKEET_STITCH_BIN=$(pwd)/e2e/fixtures/bin/fake-parakeet-stitch.mjs PARAKEET_CLI=/dev/null PARAKEET_MODEL=/dev/null DIARIZE_BIN=$(pwd)/e2e/fixtures/bin/fake-diarize.mjs FFMPEG_BIN=$(pwd)/e2e/fixtures/bin/fake-ffmpeg.mjs FFPROBE_BIN=$(pwd)/e2e/fixtures/bin/fake-ffprobe.mjs OLLAMA_URL=http://127.0.0.1:${OLLAMA_STUB_PORT:-11435} CLAUDE_BIN=$(pwd)/e2e/fixtures/bin/fake-claude.mjs FINDMNT_BIN=$(pwd)/e2e/fixtures/bin/fake-findmnt.mjs UDISKSCTL_BIN=$(pwd)/e2e/fixtures/bin/fake-udisksctl.mjs next start --port ${PORT:-3011}",
"build": "next build",
"start": "UV_THREADPOOL_SIZE=${UV_THREADPOOL_SIZE:-16} next start --port ${EDITOR_PORT:-3001}",
- "lint": "eslint",
"test": "tsx --test \"app/**/*.test.ts\"",
"e2e": "node ../scripts/queue-lock.mjs --ports PORT:3011,EXPORT_PORT:3010,OLLAMA_STUB_PORT:11435 -- playwright test",
"e2e:ui": "playwright test --ui"
diff --git a/homepage/CHANGELOG.md b/homepage/CHANGELOG.md
@@ -1,6 +1,7 @@
# Homepage Changelog
## [Unreleased]
+- **The *Running an archive* doc says how to run one from a shell or an agent.** A new section, *From a shell or an agent*, names the three surfaces — `pnpm ops` (the running editor's actions over HTTP, with `ARCHILYZER_EDITOR_URL` and `WORKER_TOKEN`), `pnpm archilyzer` (the core's command line) and the MCP server — with three example commands, and links the source's OPERATING.md (recipes) and COMMANDS.md (every command and action).
- **The homepage builds and deploys from the Docker image too.** It is a publish stage like a site's: `archilyzer publish homepage [--deploy]` (or its row on /sites → Publish) builds it — the `/source` mirror included, from the repository `docker-compose.source.yml` mounts read-only — into `homepage/out`, stamps it, and deploys that build to its Pages project, or with `--to local` into the volume the container's `homepage` service serves. The image has the pinned git-filter-repo; it has no gitleaks or stagit, so a container build skips the secret scan with a warning and ships no history pages.
- **A site that publishes only its reports is not on the homepage.** A site with `publish: "cited"` has no Official Instances card, chart series, `/stats` entry or recent item, is not in `channel-sites.json`, and the channels only it carries count in no total — as an unlisted site, whatever its listing setting says. A site with reports that publishes its full corpus is listed as before.
- **The AI and MCP doc has a Ten-minute setup.** Right after the MCP server's introduction, one block runs Claude Code against a published archive, the Jeralyzer as the example: clone the source (or unpack the tarball on Downloads), `pnpm install`, `claude mcp add archilyzer`, start `claude` and try `/ask`; then what it needs, why the server must be registered as `archilyzer` (the shipped `/ask` and `/sweep` call `mcp__archilyzer__…`), the two optional editor lines for `fetch_clip`, `TRANSCRIPT_HUB_URL`, where the `mcp.json` form for other clients is, and WSL2 on Windows. "What it can do" is a heading of its own after it. Every archive's **Use with AI** link now lands on this page.
diff --git a/homepage/content/docs/operate.md b/homepage/content/docs/operate.md
@@ -107,4 +107,32 @@ Audio for recordings that have disappeared upstream is protected from cleanup, o
the reasoning that a local copy of something no longer available anywhere is the
one thing you cannot re-fetch.
+## From a shell or an agent
+
+Everything above can be run without a browser, three ways:
+
+- **`pnpm ops <action>`** sends the editor the same action a click does — add a
+ channel, sync, download, import, transcribe, tag, fetch posts, prepare reports,
+ publish — over HTTP. It needs the running editor's address
+ (`ARCHILYZER_EDITOR_URL`) and its `WORKER_TOKEN`. A job-starting action returns
+ once the job is queued; `--wait` follows it to the end.
+- **`pnpm archilyzer <command>`** is the core's own command line: the index,
+ publish stages (`publish now`, `publish build <id>`, `publish deploy <id>`),
+ reports, offline refreshes, and `doctor`, which says what the machine can do.
+- **The MCP server** lets an AI assistant read a published archive — search,
+ transcripts, reports — and ask the editor for the media behind a cited moment.
+ See [Use with AI](/docs/ai-and-mcp/).
+
+```sh
+pnpm ops create-channel --json '{"fields":{"name":"Example","handling":"youtube","url":"https://www.youtube.com/@example"}}'
+pnpm ops sync --json '{"slug":"example"}' --wait
+pnpm ops publish --json '{"verb":"now"}' --wait
+```
+
+Every fetch still goes through the editor's paced, per-platform queues; nothing
+here downloads around them. Recipes for each task are in
+[OPERATING.md](https://archilyzer.pages.dev/source/tree/OPERATING.md), and every
+command and action in
+[COMMANDS.md](https://archilyzer.pages.dev/source/tree/COMMANDS.md).
+
Next: [Deploy to Cloudflare](/docs/deploy-cloudflare/).
diff --git a/package.json b/package.json
@@ -20,9 +20,12 @@
"dev:umtool": "node scripts/worktree.mjs run -- pnpm --filter umtool run dev",
"deploy:homepage": "pnpm --filter homepage run deploy",
"e2e": "node scripts/worktree.mjs run -- pnpm --filter editor run e2e",
+ "heavy": "node scripts/queue-lock.mjs --heavy --",
"wt": "node scripts/worktree.mjs",
"e2e:sharded": "node scripts/run-sharded-e2e.mjs",
- "test:scripts": "node --test scripts/*.test.mjs umtool/report-to-video/*.test.mjs umtool/lib/report/*.test.mjs umtool/lib/annotations/*.test.mjs umtool/lib/articles/*.test.mjs",
+ "test:scripts": "node --test scripts/*.test.mjs umtool/report-to-video/*.test.mjs umtool/lib/*.test.mjs umtool/lib/report/*.test.mjs umtool/lib/annotations/*.test.mjs umtool/lib/articles/*.test.mjs",
+ "test": "pnpm -r --no-bail --no-sort --workspace-concurrency=1 run test; a=$?; pnpm run test:scripts; b=$?; [ $a -eq 0 ] && [ $b -eq 0 ]",
+ "typecheck": "pnpm -r --no-bail --no-sort --workspace-concurrency=1 exec tsc --noEmit",
"lint": "pnpm --filter export run lint",
"ops": "node scripts/archilyzer-ops.mjs"
},
diff --git a/plans/FACTS.md b/plans/FACTS.md
@@ -8974,3 +8974,113 @@ Build stats jobs carry a "Superseded by release 18" line pointing here.
`Authentication error [code: 10000]` and exits 1 with no argv sidecar. `--version` answers
`4.147.0`. `publish.spec.ts` reads and writes both under `test-transcripts/.export-builds`
(`:32`, `:65`, `:174`).
+
+## Main's October work, outside release 18 (verified 2026-10-09, `4cffda3f`)
+
+The record is [`landed-2026-10.md`](landed-2026-10.md). Every anchor below was read at `4cffda3f`.
+
+### `pnpm ops transcribe` — one local file through the editor's workers
+
+- **The body is `path`, `start`, `end`, `workerId`, `out`, `words`** (`TRANSCRIBE_FILE_BODY_KEYS`,
+ `common/controller/transcribeFile.ts:71-78`); any other key is a 400. `path` is absolute and may be anywhere,
+ the corpus included (`:15-17`); `out` is refused inside any corpus root (`:307`); `"words"` must be a boolean
+ (`:171`).
+- **Job kind `transcribe-file`** (`:64`), `queueKey: ""` (`:547`): parallel at the queue; the worker pool
+ serialises it (`common/jobs/jobKinds.ts:884-896`, not drainable, not replayable). Only `kind: "local"` workers;
+ default = the one auto-transcribe would get.
+- **The audio is always a 16 kHz mono WAV cut by ffmpeg** into scratch (`:21-25`, `:374`), so parakeet's wrapper
+ never writes beside the source.
+- **The result** (`TranscribeFileResult`, `:107`) carries `cues` (seconds, shifted back onto the SOURCE file's
+ clock), `text`, `worker`, `durationMs`, and with `"words": true` a `words` array of
+ `TranscribedWord = {w, start, end, conf?}` (`:97`). Only parakeet emits word timings (`--words`,
+ `common/lib/transcriptionApps.ts:252-256`); any other engine gives `words: []`.
+- **`durationMs` includes the wait for a worker**: `started` is taken at `:444`, before the ffmpeg cut and the
+ "Waiting for a free local worker…" lease (`:472`); it is read at `:511`. (Release 19 B4 changes this.)
+- **The job's last log line is `@@transcribe-result <json>`** (`TRANSCRIBE_RESULT_MARKER`, `:68`, the same
+ string at `scripts/archilyzer-ops.mjs:205`); `pnpm ops transcribe --wait` prints that JSON alone on stdout
+ (`resultMarker`, `:331`).
+
+### Refresh one video's metadata
+
+- **Job kind `refresh-metadata`** (`REFRESH_METADATA_JOB_KIND`, `common/controller/refreshVideoMetadataJob.ts:47`;
+ `common/jobs/jobKinds.ts:648-656`: platform-queued, replayable, `needsText`). It runs on the channel's
+ download queue (`resolveQueueKey(downloadQueueKey(channelConfig), …)`, `refreshVideoMetadataJob.ts:95`); a
+ rate limit records the platform's backoff, a clean pass settles it (`:115-116`).
+- **One yt-dlp spawn**: `--skip-download --write-info-json`, no subtitles, `--sleep-requests 1`, the channel's
+ extra args before the negations (`buildRefreshArgs`, `common/controller/refreshVideoMetadata.ts:181`).
+- **The rewrite goes through `withMetadataHistory`** (`:424`) as writer `"refresh"` (`:418`;
+ `common/lib/metadataHistory.ts:80`), so what moved lands in `metadata.history.json`.
+- **Refused:** a video with no `data/<id>/` (`notFetchedRefusal`, `:92` — the directory is never created); a
+ record a non-yt-dlp writer owns — archive.org, Wayback, feed backfill (`NON_YTDLP_WRITERS`, `:154`); a held
+ or cooling platform. The same server action backs the video page's button and `POST /api/ops/refresh-metadata`
+ `{slug, id, queueKey?}`.
+
+### fetch-windows — a paced batch of clip windows
+
+- **One job per queue key, and the WINDOW'S URL picks it**, not the channel's (`planFetchWindows`,
+ `editor/app/channels/[slug]/videos/fetchWindowsAction.ts:165-169`, `queueKeyForUrl`). Job kind
+ `fetch-windows`: platform-queued, drainable, replayable, `needsText` (`common/jobs/jobKinds.ts:318-326`).
+- **Pacing:** at least `CLIP_WINDOW_MIN_GAP_SECONDS` = 30 between two network fetches
+ (`common/controller/fetchWindows.ts:82`), Rumble 120 (`CLIP_WINDOW_PLATFORM_MIN_GAP_SECONDS`, `:90-91`). The
+ gap ACROSS runs is keyed `clip-window:<platform>` (`:387`) in `common/jobs/platformGap.ts`, which is
+ process-wide and IN MEMORY (`globalThis.__yttPlatformGap__`, `platformGap.ts:13-20`): an editor restart
+ forgets it.
+- **Cache:** a window is `data/<id>/clips/<from>-<to>.mp4`; `findContainingClipWindow`
+ (`common/lib/clipWindow-server.ts:79`) answers with the tightest held window that covers the ask, checked
+ before any pacing (`fetchWindows.ts:345`) and costing no pause.
+- **Dedupe is within ONE request only** (`dedupeFetchWindowsItems`, `fetchWindows.ts:213-218`): nothing looks
+ for an existing job, so two requests for one window make two jobs (release 19 A5).
+- **Rumble gets `-extension_picky 0` on the first try** (`common/ytdlp/fetchWindowManaged.ts:261`): every
+ attempt reloads the Rumble page, and Cloudflare refuses a share of loads. Elsewhere it is a retry on
+ `HLS_EXTENSION_REFUSED`, and a picky run that finds no HLS retries without it (`:65-67`, `:288-296`).
+- **The /jobs row** says who asked, for what, how many: the first log line
+ ``Fetch windows: N window(s) for <requestedBy>[ · <manifest>]`` (`fetchWindows.ts:269`); progress metric
+ `clips`.
+
+### Channel create, rename and delete over ops
+
+- `create-channel`, `rename-channel`, `delete-channel` and `channel-config`'s `sites` are the New channel form
+ and the channel page's Danger zone over HTTP; there is no separate sites route.
+- **A social URL makes a social channel**: `parseChannelForm` infers `sourceKind: "social"` from the platform
+ (X, Bluesky, XenForo; `isSocialPlatform`, `common/lib/platform.ts:42`) and refuses one with no derivable
+ handle (`editor/app/channels/components/parseChannelForm.ts:85-107`).
+- **Confirmation:** delete needs `confirm` = the slug, rename the current slug
+ (`editor/app/channels/actions.ts:635`, `:695`). Both are refused while `channelMediaBusyReason` names a job or a
+ lane unit on the channel (`:644`, `:723`; `editor/app/channels/lib/mediaBusy.ts:48`), and delete while
+ `.relocating.json` exists (`common/controller/channels.ts:581-592`).
+- A social channel's page now carries the same Danger zone (`data-social-danger`,
+ `editor/app/channels/[slug]/page.tsx:214`).
+
+### umtool's operator notes
+
+- **Where:** an article's notes are `SITES_DIR/<site>/reports/<report>/notes.json`, beside `report.json`; a
+ report-video project's are `<project>/notes.json` beside its manifest (`umtool/lib/annotations/targets.mjs:1-12`).
+ The article file is the ONE corpus file umtool writes, only through `isCorpusNotesFile`
+ (`umtool/lib/paths.mjs:252`: exactly `<site>/reports/<report>/notes.json`, realpath-checked).
+ `SITES_DIR` = `$SITES_DIR`, else `$TRANSCRIPTS_DIR/sites`, else `<repo>/transcripts/sites` (`paths.mjs:217-223`).
+- **Shape:** `{format: "umtool-notes", version: 1, subject, source?, notes}` (`umtool/lib/annotations/shape.mjs:13-14`);
+ statuses `open|resolved|wontfix`, authors `operator|agent` (`:16-17`); anchors
+ `text|cite|section|whole|moment|entry|take|edit` (`:18`) — a `text` anchor is a quote with prefix/suffix,
+ re-located on read, never an offset (`umtool/lib/annotations/anchor.mjs`).
+- **One writer, `writeOp`** (`umtool/lib/annotations/store.mjs:139`): a `<file>.lock` taken exclusively (stale
+ after `LOCK_STALE_MS` 30 s, `:28`), an mtime+size token (`notesToken`, `:50`) — a stale token is `StaleNotes`
+ (409, `:32`), an unparseable file is never overwritten (`NotesUnreadable`, `:41`) — then tmp + rename (`:152`);
+ the last note deleted removes the file.
+- **Who:** `/api/notes` stamps every write `operator` (`umtool/app/api/notes/route.ts:46`); `umtool notes` stamps
+ `agent` (`umtool/lib/annotations/cli.mjs:67`, `:80`). The agent digest (`digest`,
+ `umtool/lib/annotations/digest.mjs:160`) is what both `umtool notes <target>` and
+ `GET /api/notes/context` print.
+- **Never published:** compose and the report history read named files only; the guard is the test
+ `common/publish/composeReports.test.ts:794` (a sentinel in `notes.json` appears in no built file and no
+ history revision).
+
+### umtool kinds declare capabilities; nothing branches on a kind id
+
+- `PROJECT_KINDS` (`umtool/lib/projects/kinds.mjs`): `report-video` declares `notes: true` and
+ `linksArticles: true` (`:109`, `:112`); `song` and `sweep-report` neither. Callers ask
+ `kindTakesNotes` / `kindLinksArticles` (`:192`, `:195`) — `umtool/lib/annotations/targets.mjs:80`, `:164`;
+ `umtool/lib/articles/links.mjs:47` — and an e2e spec fails on a kind id written as a literal outside
+ `lib/projects/` and `components/projects/`.
+- **/sites** lists every site's articles (`umtool/lib/articles/sites.ts`): published ids unioned with draft
+ report dirs, each with its open notes, its source and its linked video. An article row's `kind` is the REPORT's
+ kind (`factcheck|sweep`), not a project kind.
diff --git a/plans/release-18.md b/plans/release-18.md
@@ -1556,3 +1556,19 @@ export unit ok, editor build ok, e2e (ops-api, jobs, publish) **38 passed, 0 fai
`main` in the primary checkout. Owed, with the rollout, to the operator (`~/reports/release-18/RUNBOOK.html`; scripts
`~/reports/release-18/scripts/r18-{build,restart,smoke,publish-now,jeralyzer-preview,hub}.sh`, guarded on
`bf796701`).
+
+**Wave 0 (2026-10-09) — `main` `e921f82f` merged in again**, `4cffda3f`: six conflicts, both sides kept
+(`jobDetail` reads a publish stage's run and a fetch-windows batch; `GET_ARG_OPTIONAL` keeps `tags`, `channels` and
+`publish`; PUBLISH.md keeps the "three ways to drive it" table with main's fetch-windows row and main's two-step
+reports paragraph; the changelogs release 18 first). `ac5832de`: the jobKinds invariant names fetch-windows as
+drainable but not ingest (it writes only the clip cache). Gates: tsc clean; common **3465**; editor unit **159**;
+`test:scripts` **677 + 3 skipped**; mcp **292**; export unit **116**; homepage unit **23**; `docs env|files --check`,
+`settings example --check` clean; builds ok — editor 119 s, export 60 s (over the committed fixture compose: the
+primary's `export/public` holds a cited-only compose), homepage `build:nodata` 37 s, umtool capped 67 s. **The whole
+editor suite on `ac5832de`: 711 passed, 10 failed, 12 skipped, 71 min**, run while the machine sat at load 43 (three
+implementers' unit suites and a peer render): six `audio-check-scenarios`, two `auto-queue`, `dashboard-answers`'
+5 s budget, `undownloaded` "one-click whisper". The four specs alone: **43 passed, 2 failed** (audio-check "happy
+path" and "mid-stream corruption" — the fixture's corrupt checkpoint landed before any `.good` existed); the
+audio-check spec alone again: **14 passed, 1 failed** — "happy path", its first test against a cold test server
+both times, green in the full run. No code on the audio-check path changed on either side of the merge. The rollout
+scripts are re-guarded on `ac5832de`. Step 5 (fast-forward `main` to `r18/integration`) is still the operator's.
diff --git a/plans/release-19.md b/plans/release-19.md
@@ -43,6 +43,7 @@ second editor against the real corpus; the homepage build gate is `build:nodata`
|---|---|---|---|
| **A1** `pnpm ops` finds its token | `[unit]` | read `WORKER_TOKEN` / `ARCHILYZER_EDITOR_URL` from `editor/.env` when unset; `--wait` polls a real authed route instead of the `/api/jobs/active` rewrite | Wave 0 |
| **A2** jobs over ops | `[unit]` + `[spec ops-api]` | `get jobs [--active\|--failed\|--kind\|--slug]`, `get job <id> [--tail]`, `job cancel\|retry\|retry-failed\|drain\|promote\|force-release <id…>`, `--wait` on many ids with queue position; wraps `editor/app/jobs/actions.ts` | A1 |
+| **A2b** ops routes to unit tests | `[unit]` | the `ops-api.spec` route cases move to `route.test.ts` beside each route (moved from B6 on 2026-10-09: Track A owns the ops routes) | A2 |
| **A3** read side | `[unit]` | `get settings\|storage\|sites\|workers\|auto-queue\|scheduler\|cleanup <slug>` over the existing view builders/readers; `/api/auto-queue/control` gated behind `opsAuth` (unauthenticated today) | A2 |
| **A4** archival writes | `[unit]` | `settings` patch (through `saveSettings` + the schema); `lane` start/stop/drain for every lane incl. `publish`; `clear-platform-hold`; `workers enable\|disable`; per-video `transcribe-one\|delete-file\|do-not-clean`; cleanup buckets; `relocate {dryRun}`; `archilyzer storage report` (tierable bytes per channel) | A3 |
| **A5** fetch queue hygiene | `[unit]` | the editor dedupes identical fetch-window requests (same slug/id/window → the existing job); `fetch-via-editor.mjs` waits without a timeout, printing queue position; small fetch-window jobs get their own queue key / priority instead of waiting behind a multi-hour `persist-videos` | A4 |
@@ -57,11 +58,11 @@ second editor against the real corpus; the homepage build gate is `build:nodata`
| slice | class | what | after |
|---|---|---|---|
| **B1** heavy-work gate | `[unit]` | `scripts/queue-lock.mjs` generalized into `pnpm heavy -- <cmd>`: one heavy slot machine-wide + a ≥ 6 GB free-memory floor; used by e2e, `next build` (publish stages, `build-site.sh`), `build-video.mjs` renders; a render may hold the transcription lane for its duration (ops `lane`); documented in WORKTREES.md + AGENTS.md | — |
-| **B2** report-to-video robustness | `[unit]` | prune `*-frames` after the final mux (`--keep-frames` keeps them); `--chrome-only` builds missing segments; a manifest lint before render (teaser > 34 chars, image src relative to the manifest, a QR legible at 720p). `umtool/report-to-video/` is the Candace session's: coordinated with it | B1 |
+| **B2** report-to-video robustness | `[unit]` | prune `*-frames` after the final mux (`--keep-frames` keeps them); `--chrome-only` builds missing segments; a manifest lint before render (teaser > 34 chars, image src relative to the manifest, a QR legible at 720p). Track B's, after B5; the Candace session reviews it before merge (ruled 2026-10-09) | B5 |
| **B3** report CLI | `[unit]` | `archilyzer reports check <site> [--reports …]` (compose without a build); `reports verify-quotes <report.json>` (quote vs cue span, the en-orig check); `reports attach-video` encoding to fit the 24 MiB compose limit | — |
| **B4** ops transcribe | `[unit]` | `durationMs` excludes queue wait; a priority for short one-file jobs over long auto jobs | Wave 0 |
| **B5** umtool small debts | `[unit]` → `[spec umtool]` | per-worktree umtool e2e ports (`portFor`); `mix.spec` order dependence; `umtool window` writes edit notes; a selection spanning two blocks gets a Note button; the CLI finds `SITES_DIR` from the repo, not the cwd; umtool's /sites heading becomes "Articles" (naming hazard vs the editor's /sites). ONE umtool e2e run | B1 |
-| **B6** test economy | `[unit]` | pure/API e2e moves to unit: `audio-check-classifier.spec` → common; `ops-api.spec` routes → `route.test.ts`; `view-route`, `worker-unit`, `media-file-abort`. Root `pnpm test` and `pnpm typecheck`; the editor gets an eslint config or loses its broken `lint` script | — |
+| **B6** test economy | `[unit]` | pure/API e2e moves to unit: `audio-check-classifier.spec` → common; `view-route`, `worker-unit`, `media-file-abort`. Root `pnpm test` and `pnpm typecheck`; the editor gets an eslint config or loses its broken `lint` script | — |
### Track C — docs and plans (owner: the Track C implementer; no e2e)
@@ -77,17 +78,17 @@ second editor against the real corpus; the homepage build gate is `build:nodata`
| track | owns |
|---|---|
-| A | `editor/app/api/ops/**` (except B4's transcribe route), `scripts/archilyzer-ops.mjs` + its test, `mcp/src/**`, `editor/app/jobs/actions.ts`, `common/controller/fetchWindows.ts`, `fetch-via-editor.mjs` |
-| B | `scripts/queue-lock.mjs` and the heavy gate, `umtool/**` (except `umtool/report-to-video/*`, the Candace session's), the e2e specs B6 moves, the report CLI, `common/controller/transcribeFile.ts`, `editor/app/api/ops/transcribe/**` |
+| A | `editor/app/api/ops/**` (except B4's transcribe route), `scripts/archilyzer-ops.mjs` + its test, `mcp/src/**`, `editor/app/jobs/actions.ts`, `common/controller/fetchWindows.ts`, `fetch-via-editor.mjs`, the `ops-api.spec` cases A2b moves |
+| B | `scripts/queue-lock.mjs` and the heavy gate, `umtool/**` (`umtool/report-to-video/*` for B2 only, the Candace session reviewing before merge), the e2e specs B6 moves, the report CLI, `common/controller/transcribeFile.ts`, `editor/app/api/ops/transcribe/**` |
| C | `plans/**` (except the records others append), root `*.md` docs, `homepage/content/docs/**`, `mcp/README.md`, the `archilyzer docs cli` generator |
| shared, append-only | `editor/CHANGELOG.md`, `AGENTS.md`, `common/bin/_cli.ts` (rows only) |
### Graph and merge order
```
-Wave 0 (release 18 lands) ──► A1 ──► A2 ──► … ──► A9 ──► release-end suite (Track A is sequential: A1–A5, then A6–A9)
+Wave 0 (release 18 lands) ──► A1 ──► A2 ──► A2b ──► … ──► A9 ──► release-end suite (Track A is sequential: A1–A5, then A6–A9)
└─► B4
-now: B1 ──► {B2, B5}; B3; B6 (B6 touches no release-18 file)
+now: B1 ──► B5 ──► B2 (Candace session review); B3; B6 (B6 touches no release-18 file)
now: C1, C3 ──► C2 ──► C4, C5; C2 regenerated after A's actions land
```
@@ -95,7 +96,7 @@ now: C1, C3 ──► C2 ──► C4, C5; C2 regenerated after A's actions la
18). C1, C3 and B6 touch no release-18 file and may merge first. Track A merges slice by slice in its own order;
B and C merge whenever a slice is green. The parent regenerates the command reference (C2) after each Track A merge
that adds an ops action or CLI row.
-- **Do not touch:** `umtool/report-to-video/*` without the Candace session; `transcripts/**` except through the
+- **Do not touch:** `umtool/report-to-video/*` except B2, which the Candace session reviews before merge; `transcripts/**` except through the
writers; the working sessions' `~/reports/*` workspaces.
## Verification
@@ -118,6 +119,165 @@ now: C1, C3 ──► C2 ──► C4, C5; C2 regenerated after A's actions la
### Track B
+Branch `worktree-agent-a8b654c51bf472562` off `4cffda3f` (r18/integration with main merged), one Opus implementer,
+its own worktree under `.claude/worktrees/`. Scratch files `b-*` in the job's `tmp`. Slices in the order shipped:
+B6, B1, B4, B3, B5. **B2 did not ship** (below). No editor e2e was run, as planned.
+
+#### Slice B6, as shipped — test economy
+
+- **Pure and route-handler e2e moved to unit tests; the editor suite shrinks by 19** (`playwright test --list`, which
+ boots no server: 733 tests in 132 files → 714 in 130). `audio-check-classifier.spec` (6) →
+ `common/ytdlp/ffmpegStreamClassify.test.ts`. `worker-unit.spec` (2) → `editor/app/api/worker/unit/route.test.ts`:
+ the four handlers in-process, a temp corpus, an ollama stub. `view-route.spec` 12 → 1; the rest →
+ `editor/app/api/view/[name]/route.test.ts`: the rewrite table in `next.config.ts` read as data (each old path a
+ rewrite to its view, every view one; the test sets a global `__dirname` for the config, which uses it), the
+ dispatcher's 404 for unknown names and for every `/api/test/*` directory, and `/api/widget/presets` untouched. The
+ test that stays e2e is the query string surviving a rewrite (`/api/pulse?rev=`): only a running Next shows that.
+- **`media-file-abort.spec` stays e2e.** Cancelled bodies run against the handler in-process pass even with
+ `Readable.toWeb` or a naive enqueue-after-cancel wrapper (both tried), so the race happens in the server's
+ response pipeline, and only the HTTP round trip tests it. Added beside it: `files/[name]/route.test.ts` (a range, a
+ suffix range, a bad range returns the whole file, traversal is a 400, a missing file a 404) and
+ `common/lib/safeStreamController.test.ts` (a raw controller throws `ERR_INVALID_STATE` once it is closed; the guard
+ goes quiet).
+- **Root `pnpm test`** runs every package's unit suite and then `test:scripts`, and is non-zero if either fails.
+ **Root `pnpm typecheck`** runs the tsc sweep. Both pass `--no-sort`. Under pnpm 11, `--no-bail` alone still SKIPS
+ every dependent of a failed package: a red `common` ran no editor, export, homepage or mcp test, and no `tsc` in
+ them. **The documented gate `pnpm -r --no-bail --workspace-concurrency=1 exec tsc --noEmit` has the same hole.** Not
+ changed here: it is gate text in the plans and the rules (Track C's). `pnpm typecheck` is the corrected spelling.
+- **The editor's `lint` script is removed.** The editor had no eslint config. With export's config, eslint finds 35
+ errors in 25 editor files (17 `react-hooks/set-state-in-effect`, 7 `react/no-unescaped-entities`, 4
+ `react-hooks/purity`, 4 `react-hooks/refs`, 3 `@next/next/no-html-link-for-pages`). Each fix would change how a UI
+ file behaves.
+
+#### Slice B1, as shipped — the heavy slot (`pnpm heavy`)
+
+- `scripts/queue-lock.mjs` adds a second machine-global lock, `<git-common-dir>/heavy-queue.lock`, and a memory floor.
+ Once the slot is held, the run waits until `/proc/meminfo` MemAvailable ≥ `HEAVY_MIN_FREE_MB` (6000). The slot is
+ taken first and the floor checked second, so no one slips in while the holder waits for memory. `pnpm heavy --
+ <cmd>` is `queue-lock.mjs --heavy`, and pnpm's own `--` is accepted.
+- **Decision: e2e takes the heavy slot too, FIRST, then the e2e queue.** With one order everywhere, nesting cannot
+ deadlock. `HEAVY_HELD` lets a heavy command inside one pass through, as `QUEUE_LOCK_HELD` does. `E2E_QUEUE=0` skips
+ both locks and keeps the floor. The e2e queue's own banner, semantics and bypasses are unchanged. Covered entry
+ points: every package's `e2e` scripts (the CLI) and `run-sharded-e2e.mjs` (`withQueue`).
+- **The publish stages' real `next build`** (a site, the hub, the homepage) runs through the slot (`build.ts`
+ `heavyGated`). The e2e fake `EXPORT_NEXT_BIN` does not. A heavy run forwards SIGTERM/SIGHUP to its command, so a
+ stage's Cancel, which signals the wrapper, still stops the build. **`docker/build-site.sh` is not edited.** Inside
+ a docker-runner container the gate runs through the same code (`nextBuildStep`). The slot there is the container's
+ own, and the floor reads the host's meminfo, so a fan-out slows down when the host runs low but is not serialised.
+- **Renders:** documented as `pnpm heavy -- node umtool/report-to-video/build-video.mjs …`. The in-file wiring was B2
+ (not shipped). **"A render holds the transcription lane"** is a documented recipe (WORKTREES.md) with `pnpm ops
+ lane` hold/release, which exists. It is not automated.
+- A waiter is told who it waits behind and what they are running. A machine whose MemTotal is under the floor runs
+ with a note. With no usable `flock`, the gate warns and enforces the floor only. Bypasses: `HEAVY=0`,
+ `HEAVY_MIN_FREE_MB`, `HEAVY_TIMEOUT`. Test seams: `HEAVY_LOCK_FILE`, `HEAVY_MEMINFO_FILE`, `HEAVY_POLL_MS`.
+ ENVIRONMENT.md is regenerated (`docs env`), and WORKTREES.md and AGENTS.md each get a section.
+- **Manual collision on the real lock:** a second `pnpm heavy` printed `waiting for the heavy slot — held by
+ agent-a8b… (…, pid …) for 4s: sleep 6` and ran when the first finished. The floor seen live: while the parent's
+ suite was up, `pnpm heavy -- echo` waited at 2.8 GB available.
+
+#### Slice B4, as shipped — `ops transcribe`: the engine's time, and a short file first
+
+- `durationMs` now runs from the moment a worker takes the job (`onWorker`, the last attempt's) to the transcript. A
+ new `waitedMs` is the queue time. The job's log gives both.
+- A cut of ≤ 15 min of audio (`URGENT_MAX_AUDIO_SEC`, measured from the cut WAV's size) acquires a worker in the
+ pool's existing `urgent` tier. It goes ahead of parked manual batches as well as the lane (which was already
+ `background`). A longer file keeps `foreground`. The tier orders waiters only, so nothing running is interrupted.
+ `transcribeWithWorker` takes `tier`.
+
+#### Slice B3, as shipped — `reports check`, `verify-quotes`, `attach-video`
+
+- `archilyzer reports check <site> [--reports a,b] [--allow-missing-media]` is `resolveSiteReports` with nothing
+ written. It exits 1 and prints compose's own problem lines. `--reports` covers drafts that are not in site.json.
+- `archilyzer reports verify-quotes <report.json> [--json]` runs compose's quote check on each citation and prints
+ the best track plus the `en-orig` track's score. It reports `en-orig-drift` when a served `en` track matches the
+ quote and en-orig does not. The span check is now ONE function, `checkSpanQuote`, which compose also calls.
+- `archilyzer reports attach-video <report.json> <video> [--poster] [--caption]` (`publish/reportVideo.ts`) remuxes
+ an H.264 mp4 that is under 24 MiB. Anything else is encoded to fit, re-encoded smaller if it overshoots (≤ 3
+ passes). A video too long to stay watchable is refused with its length. `video` is written to report.json only if
+ its own fields validate.
+
+#### Slice B5, as shipped — umtool debts
+
+- **Per-worktree umtool e2e ports.** umtool's `e2e` script now runs through the worktree injector. The injector's
+ index was also wrong for nested worktrees: it took the FIRST root that contains the cwd, and every
+ `.claude/worktrees/<agent>` sits inside the main checkout. So every agent worktree got offset 0, the main
+ checkout's ports, for every suite. It now takes the most specific root (`indexForPath`, tested). This worktree's
+ umtool suite ran on 4251/4252 instead of 3051/3052.
+- **mix.spec's order dependence is fixed.** The file moves the fixture corpus's clip windows aside for its own tests
+ and puts them back afterwards. Its render-scratch test now matches an exact option label. In the full run all 12
+ mix tests passed; `:166`/`:201` had failed in every full run since FACTS recorded it.
+- **`umtool window` runs through the routes' edit guard.** On a generated manifest, each change is also an `edit`
+ note. The guard is now `lib/report/edit-guard.mjs`, and `guard.ts` only types it.
+- **A selection that spans two sections gets a Note button.** The note is anchored in the section where the
+ selection starts, up to that section's end. A spec covers it.
+- **The CLI finds SITES_DIR and CHANNELS_DIR from its own checkout:** `REPO_ROOT` comes from the entry script when
+ that script is `<repo>/umtool/bin/*.mjs`. The app keeps the cwd walk. `umtool/lib/*.test.mjs` joins
+ `test:scripts`.
+- **/sites appears as "articles"** in the nav and the crumbs, lowercase like every other umtool nav entry. The URL is
+ unchanged. sites.spec asserts the new name and that no "sites" link is left.
+
+#### Track B gates
+
+| gate | result |
+|---|---|
+| tsc (`pnpm typecheck`) | clean at every commit |
+| common unit | 3489 tests, 3486 pass, 3 fail. All 3 fail at the base `4cffda3f` (autoRunner ×2, jobKinds ×1; jobKinds is fixed by r19's `ac5832de`, now merged) |
+| editor unit / export / homepage / mcp | 168 / 116 / 23 / 292, all pass |
+| `test:scripts` | 698 tests, 696 pass, 2 skipped (queue-lock 11 → 23 tests) |
+| umtool build (capped, corpus linked) | ok, 71 s |
+| umtool e2e, full | 284 tests: 250 passed, 20 failed, 14 skipped, 26.5 min (after a queue wait). Ours: article-notes `:77` (a race in the new spec) and projects `:208` (a kind id in a new test), both fixed in `dd149861`. Environment: triage ×9 (the song data is present this time, so these specs ran; they expect a visible `sort` link that the `song ▸` nav group has folded away since 2026-08-25) and faces ×4 (`facedet: false`, so the venv is missing (503); these four do not check that capability). Load: browse `:15` (page load timeout), deliver `:250`, usage `:78`/`:112`, video-notes `:69` |
+| umtool e2e, focused rerun | 74 tests (article-notes, projects, browse, deliver, usage, video-notes), 69 passed, 5 failed, 12.1 min: article-notes (with the new two-section test), projects and video-notes all pass. Still failing: browse `:15`/`:34` (`/browse` page-load timeouts), deliver `:250` (this time the progress never showed within 5 s of the click), usage `:65` (ECONNRESET from the dev server) and `:112`. No Track B change touches those pages or jobs. Track B's only shared-code change on their path is `REPO_ROOT` from the entry script, which resolves to the same checkout for `cut-from-cache.mjs`. A baseline run was NOT made, so "environment/load" is a judgement, not a measurement |
+| editor e2e | none (Track B rule) |
+
+**Not done:**
+- **B2 (report-to-video robustness and the build-video heavy wiring).** The permission system refused writes to
+ `umtool/report-to-video/` ("modify shared resources"), even though the Candace session had handed the slice over.
+ The partial work is kept OUTSIDE the branch in the job's `tmp/b2-partial/`: `lint.mjs` (the manifest lint: teaser
+ house style as a warning; image src not relative or missing as an error; QR px/module at 720p, below 1.5 an error
+ and below 2.0 a warning; the threads and flips validators), `prune-frames.mjs` (prune `chrome/*-frames` and
+ `chrome/work-*` after the mux, recorded in `chrome/pruned.json`), a `verify-build.mjs` patch (pruned sequences
+ read from the record), and the build-video edit script (`--keep-frames`, `--lint`, `--chrome-only` builds missing
+ segments from the cache and never writes an existing one, the heavy slot around a render). None of it has been run.
+- `docker/build-site.sh` gets no gate of its own (see B1).
+- triage.spec and faces.spec, found failing above, are not ours to fix here.
+
### Track C
+#### Slices C1–C5, as shipped (2026-10-09)
+
+Branch `worktree-agent-ad0ba0d0eda014c20` from `4cffda3f`; C1 and C3 merged into `r19/integration` at `e5e7c55e`.
+
+| commit | what |
+|---|---|
+| `11cb1b09` | C1: STATE "Now" (2026-10-09; older "Now" entries read "Previously"); this file and `release-20.md`; PLAN.md "Beyond the AI track: releases"; [`landed-2026-10.md`](landed-2026-10.md) (main's first-parent `39abec18..e921f82f`, 70 entries); one-core Phase 4 DONE 2026-09-28; `editor-operations-ia.md` COMPLETE (slice 9 = one-core Phase 1 slice 1.3); `stats-cache-key.md` merged `10cefd15`, rolled out with release 15; `plans/README.md` names the release and landed files |
+| `2010600d` | C3: `mcp/README.md` lists `list_tags` (now every tool in `TOOLS`); README's docs table adds SETTINGS, SITE, CHANNEL, REPORT, CITATIONS, AGENTS |
+| `86e4e436` | C2: `archilyzer docs cli [--check]` (`common/bin/cli-docs.ts` + test) → `COMMANDS.md`; `OPERATING.md`; links from AGENTS.md, README, CONTRIBUTING, RUNNING_IN_DOCKER |
+| `d9564923` | C5: FACTS "Main's October work, outside release 18"; this file's A2b / B2 scope change |
+| `e35523a4` | C4: `homepage/content/docs/operate.md` "From a shell or an agent"; homepage changelog |
+| `b064c835` | merge `r19/integration` (release 21's plan, the jobKinds test fix) |
+
+**How the reference is made.** `COMMANDS.md` is a whole generated file, as ENVIRONMENT.md and SETTINGS.md are;
+OPERATING.md links it. The `archilyzer` rows come from `COMMANDS` (`common/bin/archilyzer.ts`, where the table
+lives — the `docs cli` row is there, not in `_cli.ts`). The `pnpm ops` rows come from the exported `usage()` of
+`scripts/archilyzer-ops.mjs`, read by dynamic import, and from its header comment's `// pnpm ops <action> …`
+example lines; the script is not changed. A help paragraph that opens with action names is that row's text; the
+other paragraphs print as they stand. `--check` (and a plain run) also fails when OPERATING.md names a
+`pnpm ops …` or `pnpm archilyzer …` that does not exist.
+
+**Gates** (logs `$T/c-gates-*.log`, `$T/c-homepage-build.log`): tsc clean (before C2's commit and after the
+merge); `docs env|files|cli --check` 0; `bin/cli-docs.test.ts` 9/9; after the merge `bin/cli-docs.test.ts`,
+`bin/_cli.test.ts`, `architecture.test.ts` and `jobs/jobKinds.test.ts` 49/49; homepage unit 23/23; homepage `build:nodata` ok (45 s, 5 GB scope, after the e2e run
+ended). The whole common suite was started once and stopped on the parent's throttle (the editor suite was
+running); only the changed files' tests ran. No e2e (class `[none]`/`[unit]`).
+
+**Found and left:**
+- 15 ops actions have no help paragraph in `usage()`: `channel-priority`, `metadata-scan`, `import-video`,
+ `import-archive-org`, `feed-metadata`, `refresh-report`, `sync`, `download-missing`, `relocate`,
+ `relocate-back`, `evict-clips`, `lane`, `tags`, `tag-videos`, `keep-videos`. COMMANDS.md lists them with their
+ header examples (none for `download-missing`, `relocate-back`, `evict-clips`, `keep-videos`); OPERATING.md's
+ recipes cover the common ones. A paragraph each in `usage()` is Track A's file.
+- `pnpm ops transcribe`'s `durationMs` includes the wait for a worker (FACTS; B4).
+- fetch-windows: the cross-run platform gap is in memory, and dedupe is within one request (FACTS; A5).
+- C2 is regenerated after each Track A merge that adds an action or a CLI row (`pnpm archilyzer docs cli`).
+
## Rollout
diff --git a/plans/release-22.md b/plans/release-22.md
@@ -0,0 +1,118 @@
+# Release 22 — one article, two shapes: slides, and the isometric overview
+
+Written 2026-10-09. An article (a `report.json`) can be read as the article it is, as a slide deck, or in an
+**isometric overview** that lays the two side by side in one tilted 3-D plane — each section beside its slide —
+so a reader sees the same article both ways and enters whichever they prefer at the same place.
+
+## Rulings (operator, 2026-10-09)
+
+- Articles may be **built a bit differently** to make this work: `report.json` gains authoring fields for slides.
+- **"Isometric render" is the overview described here** (operator confirmed 2026-10-09): a tilted plane pairing
+ each article section with its slide; click either to enter that view.
+
+## What is there (verified)
+
+- An article is `report.json` (`common/lib/report/schema.ts`, `validate.ts`, REPORT.md): `kind` `factcheck | sweep`,
+ title/subtitle/summary/method, `sections[]` → `claims[]` (`text`, `verdict`, `gist`, `findings`, `citations`),
+ markdown bodies citing `[label](cite:<id>)`, `video`, `sources`, `citations`.
+- Two renderers of one view model (`ReportPageView`, `common/lib/report/views.ts`): the export site's
+ `export/app/components/reports/ReportArticle.tsx` (route `export/app/reports/[reportId]/page.tsx`, static) and
+ umtool's `ArticleReader.tsx`/`ArticleBody.tsx` (`umtool/app/sites/[site]/[report]/page.tsx`, with its own lenient
+ resolver `umtool/lib/articles/article.ts` `articleView` for drafts).
+- Shared citation components: `common/components/citations/*` (`CitedMarkdown`, `InlineCite`, `CitationCard`,
+ `ReferenceList`), `common/components/report/VerdictChip.tsx`; markdown via `markdown-to-jsx`.
+- Exports: `REPORT_EXPORT_FORMATS` (`views.ts`), `common/publish/reportExports.ts` (`report.html` self-contained via
+ `common/lib/report/exportHtml.ts`, `report.pdf` by Chromium), `reportExportFiles.ts`, the downloads table in
+ `ReportArticle.tsx`, the revision-history commit (`reportHistory.ts`).
+- View toggles: the transcript `vm`/`vt` URL params (`common/components/urlState.ts`; absent = the default view).
+- **Naming hazard:** "deck" means umtool's video chrome overlay (`umtool/report-to-video/deck.mjs`, `chrome-deck.mjs`)
+ and several editor card stacks. This feature is **slides** everywhere: `slides`, `SlideView`, `?rv=slides`.
+
+## Design
+
+### The article gains slide fields (schema, backward-compatible, `version` stays 1)
+
+- `report.slides?`: `{ hide?: boolean, title?: string, closing?: string }` — the deck's own title-slide line and
+ closing line; `hide: true` publishes no slides view.
+- `section.slide?` and `claim.slide?`: `{ title?: string, points?: string[], cite?: <citation id>, layout?:
+ "points" | "evidence" | "quote" | "statement", hide?: boolean }`. `points` are the slide's own words (≤ 5, each
+ ≤ 140 chars, may cite inline); `cite` picks the evidence the slide shows (a still, a post shot, a clip's poster
+ and quote); `layout` overrides the default.
+- The validator checks them (counts, lengths, a `cite` that names a citation of that section/claim, inline cites
+ that resolve) and REPORT.md is regenerated (`docs files`). An article with no slide fields still gets slides,
+ derived as below; the fields make them good.
+
+### One builder: `buildReportSlides(view): SlideView[]` (`common/lib/report/slides.ts`, pure)
+
+Every slide carries an **anchor** (`{kind: "head" | "summary" | "found" | "section" | "claim" | "sources", id?}`)
+that is also an id in the article's DOM — the one mapping the toggle, the overview and notes all use.
+
+| slide | from | default layout |
+|---|---|---|
+| title | series, title, subtitle, dates, subject, `report.slides.title` | `title` |
+| the quick take | `summary`'s first paragraph (or `points`) | `statement` |
+| what the check found | the verdict tally (`foundGroups`) — factcheck only | `found` |
+| one per section | `section.slide.points`, else the body's first two sentences | `points` |
+| one per claim | claim `text`, `VerdictChip`, `gist`, the `cite`d (else first) citation's evidence | `evidence` |
+| sources | the reference count, the site link, `report.slides.closing` | `sources` |
+
+`hide` drops a slide; a claim with no citation renders `statement`. Long markdown never lands on a slide: a slide
+shows `points`/`gist`/one quote, and every slide links "Read this in the article" (its anchor).
+
+### The reader (export site and umtool, shared components in `common/components/report/slides/`)
+
+- **View switch** on the article header: `Article | Slides | Overview`, a segmented control in the transcript
+ `modeStrip` pattern. URL param `rv` (`slides`, `iso`; absent = article) plus the anchor in the hash, so a shared
+ link opens the same view at the same place. Switching keeps the place: the article's section in view ↔ that
+ section's slide. The article HTML stays the statically rendered page (search engines and no-JS readers get the
+ article); slides and the overview render client-side from the same `page.json`, loaded on demand.
+- **Slides** (`ReportSlides.tsx`): one 16:9 stage letterboxed in the viewport, scaled type; ←/→, PgUp/PgDn,
+ Home/End, Esc (back to the article at this slide's anchor), swipe on touch; a progress rail; slide number in
+ the hash (`#s-4`). Citations stay live (the `CitationsProvider` of the page): a cite opens its card. Per-site
+ accent and light/dark from the existing tokens. Phones: the stage fits the width; portrait shows the slide's
+ text at readable size with its evidence below.
+- **Overview** (`ReportOverview.tsx`): one plane tilted isometrically (CSS `rotateX(55deg) rotateZ(-45deg)`),
+ the article's sections as a column of page-blocks on the left, the slides as a column of 16:9 cards on the
+ right, a connector between each pair. Hover/focus a pair lights both; click the block to open the article at
+ it, the card to open the slides at it; arrow keys walk pairs. `prefers-reduced-motion`, narrow screens and
+ keyboard-first users get the same pairs flat (two columns, or stacked pairs on a phone) — the tilt is never the
+ only way to read it.
+- **umtool** `ArticleReader` gets the same switch over its lenient `articleView`, so a draft can be checked in all
+ three shapes before publishing; a slide shows the note count of its anchor, and the source tab's lint lists slide
+ problems (a section with no `points` whose body's first sentences run long, an over-long point).
+
+### Exports
+
+- `slides.html` (self-contained like `report.html`: inline style + a few lines of inline script for keys, images
+ inlined) and `slides.pdf` (Chromium, one 16:9 page per slide) join `REPORT_EXPORT_FORMATS`, the downloads table,
+ `evidence-pack.zip` and the revision commit's `exports.json`. Same 24 MiB rule.
+
+## Slices (Track E)
+
+| slice | class | what |
+|---|---|---|
+| **E1** schema + builder | `[unit]` | the slide fields in `schema.ts`/`validate.ts`, REPORT.md regenerated; `slides.ts` `buildReportSlides` with a table of tests (every default, every override, `hide`, a sweep with no tally, a claim with no citation, anchors unique and matching the article's ids); the demo fixture report gains slide fields |
+| **E2** slides view | `[unit]` + `[spec]` | `common/components/report/slides/` (`ReportSlides`, slide layouts, `ViewSwitch`), wired into `ReportArticle.tsx` and the export route; article anchors on sections/claims; `rv` + hash state |
+| **E3** overview | `[unit]` + `[spec]` | `ReportOverview` (isometric + flat fallbacks), pair navigation |
+| **E4** umtool | `[spec umtool]` | the switch in `ArticleReader`, note counts per slide, slide lint in the source tab |
+| **E5** exports | `[unit]` | `slides.html`, `slides.pdf`, downloads table, evidence pack, history |
+| **E6** records | `[none]` | this file's record, changelogs (export, umtool), REPORT.md, PUBLISH.md's exports paragraph |
+
+E1 → E2 → E3; E1 → E4; E1 → E5. e2e: the export report suite (`pnpm --filter export run e2e:report`) gains slides
+and overview cases over the fixture (switch keeps the place, keys, a cite opens its card, the reduced-motion
+overview, a phone viewport) — run once at E3 and once at E5; umtool's article specs once at E4.
+
+## Do not touch
+
+`umtool/report-to-video/**` (the Candace session's); the editor; `transcripts/**`. Published reports change only
+when a site is rebuilt — nothing here rewrites a report.json in the corpus.
+
+## Record
+
+### Track E
+
+## Rollout
+
+Rebuild and deploy each site with reports (the slides view is in the bundle, and `reports prepare` re-exports to
+add `slides.html`/`slides.pdf`). Authors add `slide` fields at their own pace; an article without them still has
+derived slides.
diff --git a/scripts/queue-lock.mjs b/scripts/queue-lock.mjs
@@ -1,5 +1,7 @@
#!/usr/bin/env node
-// Global e2e queue: exactly one e2e run at a time, machine-wide.
+// Global e2e queue: exactly one e2e run at a time, machine-wide — and the
+// HEAVY SLOT: one heavy job (an e2e run, a `next build`, a video render) at a
+// time, machine-wide, started only above a free-memory floor.
//
// Every checkout of this repo shares one lock file, so a suite started in a
// second worktree waits for the first to finish instead of racing it. That is
@@ -10,6 +12,27 @@
// then wipes that session's fixtures with no error at all.
//
// node scripts/queue-lock.mjs [--name e2e] [--ports PORT:3011,...] -- <cmd...>
+// node scripts/queue-lock.mjs --heavy -- <cmd...> (`pnpm heavy -- <cmd>`)
+//
+// THE HEAVY SLOT. Two OOMs on this machine (2026-10-08) took Xwayland and dbus
+// with them: two `next build`s, or a build beside an e2e suite, or a render
+// beside either. So every heavy entry point goes through `withHeavy`: one
+// machine-wide lock (`heavy-queue.lock`, beside the e2e one), and once it is
+// held, a wait until /proc/meminfo's MemAvailable is at least
+// HEAVY_MIN_FREE_MB (6000 by default). The slot is taken FIRST, then the floor
+// is waited for, so a later contender cannot slip in while the holder waits
+// for memory. Callers:
+// - every e2e entry point (this file's CLI and run-sharded-e2e.mjs, through
+// `withQueue`): the heavy slot first, then the e2e queue. One order
+// everywhere, so nesting cannot deadlock: an e2e run holds both, and a
+// `pnpm heavy -- pnpm e2e` passes through its own slot (HEAVY_HELD).
+// - the publish stages' `next build` (common/publish/build.ts, heavyGated).
+// - a render: `pnpm heavy -- node umtool/report-to-video/build-video.mjs …`.
+// Bypasses: HEAVY=0 (no slot, no floor), HEAVY_MIN_FREE_MB=0 (no floor),
+// HEAVY_TIMEOUT=<seconds>. E2E_QUEUE=0 skips the slot as it skips the queue;
+// the floor still applies. A machine whose MemTotal is under the floor is not
+// made to wait forever: it is told, and runs. The slot is a safety net, not a
+// correctness lock: with no usable `flock` it warns and runs on the floor alone.
//
// WHY THE LOCK IS HELD BY A SEPARATE CHILD.
// flock(1) deliberately keeps its lock fd open across exec — that is what the
@@ -48,6 +71,14 @@ const SELF = fileURLToPath(import.meta.url);
// wrapper) passes straight through instead of deadlocking against the lock its
// own parent is holding. Verified to survive nested `pnpm --filter` calls.
export const HELD_ENV = "QUEUE_LOCK_HELD";
+// The same, for the heavy slot: set while a command runs inside it, so a heavy
+// command that starts another (a `pnpm heavy -- pnpm e2e`, a build stage run
+// from inside an e2e suite's editor) passes through instead of waiting on
+// itself.
+export const HEAVY_HELD_ENV = "HEAVY_HELD";
+const HEAVY_NAME = "heavy";
+export const DEFAULT_MIN_FREE_MB = 6000;
+const DEFAULT_MEM_POLL_MS = 5_000;
const PROBE_HELD_EXIT = 91; // `flock -n -E 91`: distinguishes held from failed
const WAIT_TIMEOUT_EXIT = 92; // `flock -w N -E 92`
@@ -66,12 +97,16 @@ function parseArgv(argv) {
name: "e2e",
portSpec: null,
hold: false,
- timeoutMs: defaultTimeoutMs(),
+ heavy: false,
+ // Unset unless --timeout: the e2e queue defaults it from E2E_QUEUE_TIMEOUT,
+ // the heavy slot from HEAVY_TIMEOUT.
+ timeoutMs: undefined,
portGraceMs: Number(process.env.E2E_PORT_GRACE_MS ?? DEFAULT_PORT_GRACE_MS),
};
for (let i = 0; i < flags.length; i++) {
const f = flags[i];
if (f === "--hold") opts.hold = true;
+ else if (f === "--heavy") opts.heavy = true;
else if (f === "--name") opts.name = flags[++i];
else if (f === "--ports") opts.portSpec = flags[++i];
else if (f === "--timeout") opts.timeoutMs = Number(flags[++i]) * 1000;
@@ -86,8 +121,8 @@ function parseArgv(argv) {
// Waiting forever is the point of the feature, so that is the default. A
// bounded wait is available for anything that would rather fail than block.
-function defaultTimeoutMs() {
- const raw = process.env.E2E_QUEUE_TIMEOUT;
+function defaultTimeoutMs(envVar = "E2E_QUEUE_TIMEOUT") {
+ const raw = process.env[envVar];
if (raw == null || raw === "") return 0;
const n = Number(raw);
return Number.isFinite(n) && n > 0 ? n * 1000 : 0;
@@ -131,7 +166,13 @@ function gitOut(args) {
// .claude/worktrees/* alike — so one file is genuinely machine-global. Living
// inside .git/ it is also untracked by construction (no .gitignore entry).
function lockFileFor(name) {
- if (process.env.E2E_QUEUE_LOCK_FILE) return process.env.E2E_QUEUE_LOCK_FILE;
+ // Each lock has its own override: one file for both would make an e2e run
+ // (which takes the heavy slot, then the queue) wait on itself.
+ const override =
+ name === HEAVY_NAME
+ ? process.env.HEAVY_LOCK_FILE
+ : process.env.E2E_QUEUE_LOCK_FILE;
+ if (override) return override;
const dir =
gitOut(["rev-parse", "--path-format=absolute", "--git-common-dir"]) ??
os.tmpdir();
@@ -183,13 +224,15 @@ function readHolderJson(lock) {
}
}
-function describeHolder(lock) {
+function describeHolder(lock, showCmd = false) {
const h = readHolderJson(lock);
if (!h) return "another run (details unavailable)";
const where = h.worktree ? path.basename(h.worktree) : "?";
const age = h.startedAt ? ` for ${humanAge(Date.parse(h.startedAt))}` : "";
const dead = h.alive ? "" : " — pid gone, releasing";
- return `${where} (${h.branch}, pid ${h.pid})${age}${dead}`;
+ // The heavy slot is shared by kinds of work, so say which one is ahead.
+ const what = showCmd && h.cmd ? `: ${String(h.cmd).slice(0, 100)}` : "";
+ return `${where} (${h.branch}, pid ${h.pid})${age}${what}${dead}`;
}
function humanAge(startedMs) {
@@ -414,13 +457,15 @@ async function preflight(ports, graceMs) {
// ---------------------------------------------------------------- run + wait
-function runCommand(cmd, name) {
+function runCommand(cmd, name, { forwardTerm = false } = {}) {
return new Promise((resolve) => {
const child = spawn(cmd[0], cmd.slice(1), {
stdio: "inherit",
- env: { ...process.env, [HELD_ENV]: name },
+ // A heavy-only run (name null) leaves QUEUE_LOCK_HELD alone: it holds no
+ // e2e queue for a nested e2e run to pass through.
+ env: name ? { ...process.env, [HELD_ENV]: name } : { ...process.env },
});
- installSignalHandlers(() => child);
+ installSignalHandlers(() => child, forwardTerm);
child.on("error", (err) => {
process.stderr.write(`queue-lock: ${err.message}\n`);
resolve(1);
@@ -434,12 +479,24 @@ function runCommand(cmd, name) {
}
let signalHits = 0;
-function installSignalHandlers(getChild) {
+// `forwardTerm`: a heavy-slot run is usually started by a PROCESS, not a
+// terminal — a publish stage whose Cancel SIGTERMs this wrapper alone, then
+// SIGKILLs it, which would orphan the `next build` under it. So a heavy run
+// passes SIGTERM/SIGHUP on to its command (a terminal's SIGINT already reached
+// the whole group, and is still never forwarded).
+function installSignalHandlers(getChild, forwardTerm = false) {
for (const sig of ["SIGINT", "SIGTERM", "SIGHUP"]) {
process.on(sig, () => {
signalHits++;
const child = getChild();
if (signalHits === 1) {
+ if (forwardTerm && sig !== "SIGINT") {
+ try {
+ child?.kill(sig);
+ } catch {
+ /* already gone */
+ }
+ }
// The terminal already delivered this to the whole foreground process
// group, child included. We deliberately do not forward it: a second
// SIGINT is precisely how playwright skips globalTeardown, which is
@@ -460,11 +517,11 @@ function installSignalHandlers(getChild) {
}
}
-function startWaitBanner(lock) {
+function startWaitBanner(lock, what = E2E_LOCK) {
const t0 = Date.now();
process.stderr.write(
- `queue-lock: waiting for the e2e queue — held by ${describeHolder(lock)}\n` +
- " (one e2e run at a time, machine-wide; E2E_QUEUE=0 to bypass)\n",
+ `queue-lock: waiting for ${what.label} — held by ${describeHolder(lock, what.showCmd)}\n` +
+ ` ${what.hint}\n`,
);
const timer = setInterval(() => {
process.stderr.write(
@@ -480,28 +537,29 @@ function startWaitBanner(lock) {
};
}
-// ------------------------------------------------------------- the entry point
-
-/**
- * Run `fn` with the global queue lock held, after checking `ports` are free.
- * Used both by the CLI below and directly by scripts/run-sharded-e2e.mjs.
- */
-export async function withQueue(opts, fn) {
- const name = opts.name ?? "e2e";
- const ports = parsePorts(opts.portSpec ?? null);
- const graceMs = opts.portGraceMs ?? Number(process.env.E2E_PORT_GRACE_MS ?? DEFAULT_PORT_GRACE_MS);
- const timeoutMs = opts.timeoutMs ?? defaultTimeoutMs();
-
- // "Don't queue" never means "don't check the ports": the preflight is what
- // turns a silent cross-worktree data wipe into a loud abort.
- if (process.env.E2E_QUEUE === "0" || process.env[HELD_ENV] === name) {
- await preflight(ports, graceMs);
- return fn();
- }
-
- const lock = lockFileFor(name);
+// ------------------------------------------------------------ the two locks
+
+const E2E_LOCK = {
+ label: "the e2e queue",
+ hint: "(one e2e run at a time, machine-wide; E2E_QUEUE=0 to bypass)",
+ timeoutHint:
+ "Raise or unset E2E_QUEUE_TIMEOUT, or set E2E_QUEUE=0 to bypass the queue.",
+ showCmd: false,
+};
+
+const HEAVY_LOCK = {
+ label: "the heavy slot",
+ hint:
+ "(one heavy job — an e2e run, a next build, a render — at a time, machine-wide; HEAVY=0 to bypass)",
+ timeoutHint: "Raise or unset HEAVY_TIMEOUT, or set HEAVY=0 to bypass the heavy slot.",
+ showCmd: true,
+};
+
+// Take `lock`, announcing whom we wait behind; returns the release function.
+// A timeout exits EXIT_TIMEOUT, as it always has for the e2e queue.
+async function holdLock(lock, name, cmd, timeoutMs, what) {
let stopBanner = null;
- if (isHeld(lock)) stopBanner = startWaitBanner(lock);
+ if (isHeld(lock)) stopBanner = startWaitBanner(lock, what);
let holder;
try {
@@ -510,7 +568,7 @@ export async function withQueue(opts, fn) {
if (err.timeout) {
process.stderr.write(
`\nqueue-lock: gave up after ${humanDuration(timeoutMs)} waiting for ${lock}\n` +
- " Raise or unset E2E_QUEUE_TIMEOUT, or set E2E_QUEUE=0 to bypass the queue.\n",
+ ` ${what.timeoutHint}\n`,
);
process.exit(EXIT_TIMEOUT);
}
@@ -518,7 +576,7 @@ export async function withQueue(opts, fn) {
}
stopBanner?.();
- const holderFile = writeHolderJson(lock, name, opts.cmd ?? [name]);
+ const holderFile = writeHolderJson(lock, name, cmd);
const cleanup = () => {
try {
fs.rmSync(holderFile, { force: true });
@@ -532,30 +590,231 @@ export async function withQueue(opts, fn) {
}
};
process.on("exit", cleanup);
-
- try {
- await preflight(ports, graceMs);
- return await fn();
- } finally {
+ return async () => {
+ process.off("exit", cleanup);
try {
fs.rmSync(holderFile, { force: true });
} catch {
/* best effort */
}
await release(holder);
+ };
+}
+
+// --------------------------------------------------------- the memory floor
+
+// /proc/meminfo's MemAvailable and MemTotal, in MB, or null when the text has
+// neither (not Linux, or a reader handed something else).
+export function parseMeminfo(text) {
+ const kb = (key) => {
+ const m = new RegExp(`^${key}:\\s+(\\d+)\\s*kB`, "m").exec(String(text));
+ return m ? Number(m[1]) : null;
+ };
+ const available = kb("MemAvailable");
+ const total = kb("MemTotal");
+ if (available == null || total == null) return null;
+ return {
+ availableMb: Math.floor(available / 1024),
+ totalMb: Math.floor(total / 1024),
+ };
+}
+
+// HEAVY_MEMINFO_FILE is the tests' seam: a file they rewrite to move the
+// "available" figure under a waiting run.
+export function readMeminfo(file = process.env.HEAVY_MEMINFO_FILE || "/proc/meminfo") {
+ try {
+ return parseMeminfo(fs.readFileSync(file, "utf8"));
+ } catch {
+ return null;
}
}
+export function minFreeMb(env = process.env) {
+ const raw = env.HEAVY_MIN_FREE_MB;
+ if (raw == null || raw === "") return DEFAULT_MIN_FREE_MB;
+ const n = Number(raw);
+ return Number.isFinite(n) && n >= 0 ? n : DEFAULT_MIN_FREE_MB;
+}
+
+function memPollMs(env = process.env) {
+ const n = Number(env.HEAVY_POLL_MS);
+ return Number.isFinite(n) && n > 0 ? n : DEFAULT_MEM_POLL_MS;
+}
+
+/**
+ * Wait until MemAvailable >= `minMb`. Every input is injectable — `read`
+ * returns `{availableMb, totalMb}` or null — so the unit tests drive it with
+ * no real memory pressure. Resolves `{waitedMs}` or `{skipped}` (why the floor
+ * was not waited for); throws `{timeout: true}` past `timeoutMs` (0 = never).
+ */
+export async function waitForMemory({
+ minMb = minFreeMb(),
+ read = readMeminfo,
+ pollMs = memPollMs(),
+ tickMs = TICK_MS,
+ timeoutMs = 0,
+ log = (line) => process.stderr.write(line),
+ now = Date.now,
+ sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
+} = {}) {
+ if (!(minMb > 0)) return { skipped: "off" };
+ let m = read();
+ if (!m) {
+ log("heavy: /proc/meminfo is not readable — the memory floor is not checked\n");
+ return { skipped: "unreadable" };
+ }
+ // A floor the machine cannot reach would be a wait forever. Say so and run.
+ if (m.totalMb < minMb) {
+ log(
+ `heavy: this machine has ${m.totalMb} MB in all, under the ${minMb} MB floor — not waiting for it\n`,
+ );
+ return { skipped: "total" };
+ }
+ if (m.availableMb >= minMb) return { waitedMs: 0 };
+ const t0 = now();
+ let lastTick = t0;
+ log(
+ `heavy: waiting for memory — ${m.availableMb} MB available, the floor is ${minMb} MB\n` +
+ " (HEAVY_MIN_FREE_MB=<MB> to change it; 0, or HEAVY=0, to skip it)\n",
+ );
+ for (;;) {
+ await sleep(pollMs);
+ m = read() ?? m;
+ const waited = now() - t0;
+ if (m.availableMb >= minMb) {
+ log(`heavy: ${m.availableMb} MB available after ${humanDuration(waited)}\n`);
+ return { waitedMs: waited };
+ }
+ if (timeoutMs > 0 && waited >= timeoutMs) {
+ throw Object.assign(
+ new Error(
+ `heavy: gave up after ${humanDuration(waited)} waiting for ${minMb} MB available (${m.availableMb} MB)`,
+ ),
+ { timeout: true },
+ );
+ }
+ if (now() - lastTick >= tickMs) {
+ lastTick = now();
+ log(
+ `heavy: still waiting for memory (${m.availableMb} MB available, ${humanDuration(waited)})\n`,
+ );
+ }
+ }
+}
+
+async function memoryFloorOrExit(timeoutMs) {
+ try {
+ await waitForMemory({ timeoutMs });
+ } catch (err) {
+ if (!err.timeout) throw err;
+ process.stderr.write(
+ `\n${err.message}\n Raise or unset the timeout, lower HEAVY_MIN_FREE_MB, or set HEAVY=0.\n`,
+ );
+ process.exit(EXIT_TIMEOUT);
+ }
+}
+
+// ------------------------------------------------------------ the entry points
+
+/**
+ * Run `fn` in the heavy slot: one heavy job machine-wide, started only once
+ * MemAvailable is at or above the floor. `opts.cmd` names the work in the
+ * holder file (what a waiter is told it waits behind).
+ */
+export async function withHeavy(opts, fn) {
+ if (process.env.HEAVY === "0" || process.env[HEAVY_HELD_ENV]) return fn();
+ const timeoutMs = opts.timeoutMs ?? defaultTimeoutMs("HEAVY_TIMEOUT");
+ const cmd = opts.cmd ?? [HEAVY_NAME];
+
+ let releaseSlot = null;
+ try {
+ releaseSlot = await holdLock(
+ lockFileFor(HEAVY_NAME),
+ HEAVY_NAME,
+ cmd,
+ timeoutMs,
+ HEAVY_LOCK,
+ );
+ } catch (err) {
+ // No usable flock (a container image without util-linux): the slot is a
+ // safety net, so run on the floor alone rather than not at all.
+ process.stderr.write(
+ `heavy: the heavy slot is not held (${err?.message ?? err}) — the memory floor still applies\n`,
+ );
+ }
+ try {
+ await memoryFloorOrExit(timeoutMs);
+ process.env[HEAVY_HELD_ENV] = "1";
+ return await fn();
+ } finally {
+ delete process.env[HEAVY_HELD_ENV];
+ await releaseSlot?.();
+ }
+}
+
+/**
+ * Run `fn` with the global queue lock held, after checking `ports` are free.
+ * Used both by the CLI below and directly by scripts/run-sharded-e2e.mjs.
+ * Unless `opts.heavySlot === false`, the run takes the heavy slot first.
+ */
+export async function withQueue(opts, fn) {
+ const name = opts.name ?? "e2e";
+ const ports = parsePorts(opts.portSpec ?? null);
+ const graceMs = opts.portGraceMs ?? Number(process.env.E2E_PORT_GRACE_MS ?? DEFAULT_PORT_GRACE_MS);
+ const timeoutMs = opts.timeoutMs ?? defaultTimeoutMs();
+
+ // A nested invocation: the parent holds the queue (and the slot) already.
+ if (process.env[HELD_ENV] === name) {
+ await preflight(ports, graceMs);
+ return fn();
+ }
+ // "Don't queue" never means "don't check the ports": the preflight is what
+ // turns a silent cross-worktree data wipe into a loud abort. Nor does it
+ // mean "ignore the memory floor" — HEAVY=0 is that switch.
+ if (process.env.E2E_QUEUE === "0") {
+ await preflight(ports, graceMs);
+ if (process.env.HEAVY !== "0" && !process.env[HEAVY_HELD_ENV]) {
+ await memoryFloorOrExit(timeoutMs);
+ }
+ return fn();
+ }
+
+ const queued = async () => {
+ const releaseQueue = await holdLock(
+ lockFileFor(name),
+ name,
+ opts.cmd ?? [name],
+ timeoutMs,
+ E2E_LOCK,
+ );
+ try {
+ await preflight(ports, graceMs);
+ return await fn();
+ } finally {
+ await releaseQueue();
+ }
+ };
+ if (opts.heavySlot === false) return queued();
+ return withHeavy({ timeoutMs, cmd: opts.cmd ?? [name] }, queued);
+}
+
async function main() {
- const { opts, cmd } = parseArgv(process.argv.slice(2));
+ const { opts, cmd: rawCmd } = parseArgv(process.argv.slice(2));
if (opts.hold) return runHolder();
+ // `pnpm heavy -- <cmd>` may hand the separator through as the first word.
+ const cmd = opts.heavy && rawCmd[0] === "--" ? rawCmd.slice(1) : rawCmd;
if (cmd.length === 0) {
process.stderr.write(
- "usage: queue-lock.mjs [--name e2e] [--ports PORT:3011,...] -- <cmd...>\n",
+ "usage: queue-lock.mjs [--name e2e] [--ports PORT:3011,...] -- <cmd...>\n" +
+ " queue-lock.mjs --heavy [--timeout <s>] -- <cmd...>\n",
);
process.exit(2);
}
- const code = await withQueue({ ...opts, cmd }, () => runCommand(cmd, opts.name));
+ const code = opts.heavy
+ ? await withHeavy({ timeoutMs: opts.timeoutMs, cmd }, () =>
+ runCommand(cmd, null, { forwardTerm: true }),
+ )
+ : await withQueue({ ...opts, cmd }, () => runCommand(cmd, opts.name));
process.exit(code);
}
diff --git a/scripts/queue-lock.test.mjs b/scripts/queue-lock.test.mjs
@@ -1,8 +1,10 @@
-// Tests for the global e2e queue (scripts/queue-lock.mjs).
+// Tests for the global e2e queue and the heavy slot (scripts/queue-lock.mjs).
//
-// Every test drives the real CLI against a throwaway lock file via
-// E2E_QUEUE_LOCK_FILE, so none of them can touch the actual .git/e2e-queue.lock
-// or any real port. Run with: pnpm test:scripts
+// Every test drives the real CLI against throwaway lock files via
+// E2E_QUEUE_LOCK_FILE / HEAVY_LOCK_FILE, so none of them can touch the actual
+// .git/e2e-queue.lock or .git/heavy-queue.lock or any real port. The e2e-queue
+// tests run with HEAVY=0 (they are about the queue); the heavy tests turn it
+// on with their own lock and a fake /proc/meminfo. Run with: pnpm test:scripts
import assert from "node:assert/strict";
import { spawn } from "node:child_process";
import fs from "node:fs";
@@ -11,6 +13,7 @@ import os from "node:os";
import path from "node:path";
import test from "node:test";
import { fileURLToPath } from "node:url";
+import { parseMeminfo, waitForMemory } from "./queue-lock.mjs";
const SCRIPT = fileURLToPath(new URL("./queue-lock.mjs", import.meta.url));
@@ -18,21 +21,58 @@ function tmpDir() {
return fs.mkdtempSync(path.join(os.tmpdir(), "queue-lock-test-"));
}
-// Run the wrapper to completion, capturing output. `env` is merged over the
-// current environment; E2E_PORT_CHECK defaults off so tests that are not about
-// the preflight never probe a port.
-function runLock(args, env = {}) {
- return new Promise((resolve) => {
- const child = spawn(process.execPath, [SCRIPT, ...args], {
- env: { E2E_PORT_CHECK: "0", ...process.env, ...env },
- stdio: ["ignore", "pipe", "pipe"],
- });
- let stdout = "";
- let stderr = "";
- child.stdout.on("data", (d) => (stdout += d));
- child.stderr.on("data", (d) => (stderr += d));
- child.on("exit", (code) => resolve({ code, stdout, stderr }));
+// The environment a run sees: `env` over the current one. E2E_PORT_CHECK
+// defaults off so tests that are not about the preflight never probe a port;
+// HEAVY defaults off so the e2e-queue tests never take a heavy slot. A
+// pass-through marker inherited from whatever runs this suite (a
+// `pnpm heavy -- pnpm test:scripts`) is dropped unless the test sets it.
+function lockEnv(env) {
+ const out = { E2E_PORT_CHECK: "0", HEAVY: "0", ...process.env, ...env };
+ for (const k of ["HEAVY_HELD", "QUEUE_LOCK_HELD", "E2E_QUEUE"]) {
+ if (!(k in env)) delete out[k];
+ }
+ return out;
+}
+
+// Start the wrapper; `done` resolves at exit with the captured output, and
+// `waitFor(re)` resolves once stderr matches — how a test knows a contender is
+// queued (its banner is out) rather than guessing with a delay.
+function startLock(args, env = {}) {
+ const child = spawn(process.execPath, [SCRIPT, ...args], {
+ env: lockEnv(env),
+ stdio: ["ignore", "pipe", "pipe"],
});
+ let stdout = "";
+ let stderr = "";
+ const waiters = [];
+ child.stdout.on("data", (d) => (stdout += d));
+ child.stderr.on("data", (d) => {
+ stderr += d;
+ for (const w of waiters) if (w.re.test(stderr)) w.resolve();
+ });
+ const done = new Promise((resolve) =>
+ child.on("exit", (code) => resolve({ code, stdout, stderr })),
+ );
+ const waitFor = (re) =>
+ new Promise((resolve, reject) => {
+ if (re.test(stderr)) return resolve();
+ waiters.push({ re, resolve });
+ done.then(() => reject(new Error(`exited before stderr matched ${re}: ${stderr}`)));
+ });
+ return { child, done, waitFor };
+}
+
+function runLock(args, env = {}) {
+ return startLock(args, env).done;
+}
+
+// Resolves once `check()` is true (a holder.json written: a run has acquired).
+async function until(check, ms = 10_000) {
+ const t0 = Date.now();
+ while (!check()) {
+ if (Date.now() - t0 > ms) throw new Error("timed out waiting");
+ await delay(20);
+ }
}
// A command that records "S<id>" when it starts and "E<id>" when it ends, so
@@ -72,10 +112,17 @@ test("serves waiters in arrival order (FIFO)", async () => {
const log = path.join(dir, "fifo.log");
const env = { E2E_QUEUE_LOCK_FILE: lock };
- const runs = [];
- for (const id of ["1", "2", "3"]) {
- runs.push(runLock(["--", ...markerCmd(log, id, 300)], env));
- await delay(150); // stagger arrivals so the intended order is unambiguous
+ // Each arrival waits until the one before it is in line: the first holds
+ // (its holder.json is written), the next two have printed their banner and
+ // had a moment to block in flock. A fixed stagger raced a loaded machine.
+ const first = startLock(["--", ...markerCmd(log, "1", 600)], env);
+ await until(() => fs.existsSync(`${lock}.holder.json`));
+ const runs = [first.done];
+ for (const id of ["2", "3"]) {
+ const r = startLock(["--", ...markerCmd(log, id, 300)], env);
+ await r.waitFor(/waiting for the e2e queue/);
+ await delay(150);
+ runs.push(r.done);
}
await Promise.all(runs);
@@ -88,7 +135,7 @@ test("prints a banner naming the holder while waiting", async () => {
const env = { E2E_QUEUE_LOCK_FILE: lock };
const first = runLock(["--", process.execPath, "-e", "setTimeout(()=>{},600)"], env);
- await delay(200);
+ await until(() => fs.existsSync(`${lock}.holder.json`));
const second = await runLock(["--", process.execPath, "-e", "0"], env);
await first;
@@ -139,7 +186,7 @@ test("E2E_QUEUE=0 bypasses the queue entirely", async () => {
test("a SIGKILLed run releases the lock immediately", async () => {
const dir = tmpDir();
const lock = path.join(dir, "q.lock");
- const env = { E2E_PORT_CHECK: "0", ...process.env, E2E_QUEUE_LOCK_FILE: lock };
+ const env = lockEnv({ E2E_QUEUE_LOCK_FILE: lock });
const victim = spawn(
process.execPath,
@@ -242,3 +289,204 @@ test("removes its holder.json when the run finishes", async () => {
await runLock(["--", process.execPath, "-e", "0"], { E2E_QUEUE_LOCK_FILE: lock });
assert.equal(fs.existsSync(`${lock}.holder.json`), false);
});
+
+// ------------------------------------------------------------- the heavy slot
+
+// A fake /proc/meminfo: `availableMb` free of `totalMb`.
+function meminfo(file, availableMb, totalMb = 32_000) {
+ fs.writeFileSync(
+ file,
+ `MemTotal: ${totalMb * 1024} kB\nMemFree: 1024 kB\nMemAvailable: ${availableMb * 1024} kB\n`,
+ );
+}
+
+// The heavy slot on, against its own lock and a roomy fake meminfo.
+function heavyEnv(dir, extra = {}) {
+ const mem = path.join(dir, "meminfo");
+ if (!fs.existsSync(mem)) meminfo(mem, 20_000);
+ return {
+ HEAVY: "1",
+ HEAVY_LOCK_FILE: path.join(dir, "heavy.lock"),
+ HEAVY_MEMINFO_FILE: mem,
+ HEAVY_POLL_MS: "50",
+ ...extra,
+ };
+}
+
+test("parseMeminfo reads MemAvailable and MemTotal in MB", () => {
+ assert.deepEqual(
+ parseMeminfo("MemTotal: 32768000 kB\nMemFree: 1 kB\nMemAvailable: 6144000 kB\n"),
+ { availableMb: 6000, totalMb: 32000 },
+ );
+ assert.equal(parseMeminfo("nothing here"), null);
+});
+
+test("waitForMemory waits for the floor, polling the injected reader", async () => {
+ const readings = [2000, 4000, 5999, 6000];
+ const lines = [];
+ let clock = 0;
+ const res = await waitForMemory({
+ minMb: 6000,
+ read: () => ({ availableMb: readings.shift() ?? 6000, totalMb: 32_000 }),
+ pollMs: 1000,
+ tickMs: 2000,
+ log: (l) => lines.push(l),
+ now: () => clock,
+ sleep: async (ms) => {
+ clock += ms;
+ },
+ });
+ assert.equal(res.waitedMs, 3000);
+ assert.match(lines[0], /waiting for memory — 2000 MB available, the floor is 6000 MB/);
+ assert.ok(lines.some((l) => /still waiting for memory \(5999 MB/.test(l)), lines.join(""));
+ assert.match(lines.at(-1), /6000 MB available after 3s/);
+});
+
+test("waitForMemory: no wait above the floor, at 0, or under a MemTotal that can never reach it", async () => {
+ const never = () => {
+ throw new Error("must not sleep");
+ };
+ const read = (a, t = 32_000) => () => ({ availableMb: a, totalMb: t });
+ assert.deepEqual(await waitForMemory({ minMb: 6000, read: read(9000), sleep: never }), { waitedMs: 0 });
+ assert.deepEqual(await waitForMemory({ minMb: 0, read: read(10), sleep: never }), { skipped: "off" });
+ const lines = [];
+ assert.deepEqual(
+ await waitForMemory({ minMb: 6000, read: read(100, 4000), sleep: never, log: (l) => lines.push(l) }),
+ { skipped: "total" },
+ );
+ assert.match(lines.join(""), /4000 MB in all, under the 6000 MB floor/);
+ assert.deepEqual(
+ await waitForMemory({ minMb: 6000, read: () => null, sleep: never, log: () => {} }),
+ { skipped: "unreadable" },
+ );
+});
+
+test("waitForMemory gives up past its timeout", async () => {
+ let clock = 0;
+ await assert.rejects(
+ waitForMemory({
+ minMb: 6000,
+ read: () => ({ availableMb: 100, totalMb: 32_000 }),
+ pollMs: 1000,
+ timeoutMs: 3000,
+ log: () => {},
+ now: () => clock,
+ sleep: async (ms) => {
+ clock += ms;
+ },
+ }),
+ (err) => err.timeout === true,
+ );
+});
+
+test("two heavy contenders run one at a time; the second names what it waits behind", async () => {
+ const dir = tmpDir();
+ const log = path.join(dir, "order.log");
+ const env = heavyEnv(dir);
+ const first = startLock(["--heavy", "--", ...markerCmd(log, "1", 600)], env);
+ await until(() => fs.existsSync(`${env.HEAVY_LOCK_FILE}.holder.json`));
+ const second = await runLock(["--heavy", "--", ...markerCmd(log, "2", 50)], env);
+ assert.equal((await first.done).code, 0);
+ assert.equal(second.code, 0);
+ assert.equal(fs.readFileSync(log, "utf8"), "S1E1S2E2");
+ assert.match(second.stderr, /waiting for the heavy slot — held by .*pid \d+.*: .*appendFileSync/);
+ assert.equal(fs.existsSync(`${env.HEAVY_LOCK_FILE}.holder.json`), false);
+});
+
+test("an e2e run takes the heavy slot too: it waits for a heavy job, then runs", async () => {
+ const dir = tmpDir();
+ const log = path.join(dir, "order.log");
+ const env = heavyEnv(dir, { E2E_QUEUE_LOCK_FILE: path.join(dir, "q.lock") });
+ const build = startLock(["--heavy", "--", ...markerCmd(log, "b", 600)], env);
+ await until(() => fs.existsSync(`${env.HEAVY_LOCK_FILE}.holder.json`));
+ const e2e = await runLock(["--", ...markerCmd(log, "e", 50)], env);
+ await build.done;
+ assert.equal(e2e.code, 0);
+ assert.equal(fs.readFileSync(log, "utf8"), "SbEbSeEe");
+ assert.match(e2e.stderr, /waiting for the heavy slot/);
+});
+
+test("a heavy run inside a heavy run passes through (no self-deadlock), and so does an e2e run inside one", async () => {
+ const dir = tmpDir();
+ const env = heavyEnv(dir, { E2E_QUEUE_LOCK_FILE: path.join(dir, "q.lock") });
+ const inner = `${JSON.stringify(process.execPath)} ${JSON.stringify(SCRIPT)}`;
+ const res = await runLock(
+ ["--heavy", "--", "sh", "-c", `${inner} --heavy -- true && ${inner} -- true && echo nested-ok`],
+ env,
+ );
+ assert.equal(res.code, 0, res.stderr);
+ assert.match(res.stdout, /nested-ok/);
+ assert.doesNotMatch(res.stderr, /waiting for the heavy slot/);
+});
+
+test("a heavy run waits under the memory floor and starts once memory is back", async () => {
+ const dir = tmpDir();
+ const env = heavyEnv(dir);
+ meminfo(env.HEAVY_MEMINFO_FILE, 1500);
+ const run = startLock(["--heavy", "--", process.execPath, "-e", 'console.log("ran")'], env);
+ await run.waitFor(/waiting for memory — 1500 MB available, the floor is 6000 MB/);
+ meminfo(env.HEAVY_MEMINFO_FILE, 7000);
+ const res = await run.done;
+ assert.equal(res.code, 0);
+ assert.match(res.stdout, /ran/);
+ assert.match(res.stderr, /7000 MB available after/);
+});
+
+test("HEAVY_MIN_FREE_MB moves the floor; HEAVY=0 skips slot and floor", async () => {
+ const dir = tmpDir();
+ const env = heavyEnv(dir);
+ meminfo(env.HEAVY_MEMINFO_FILE, 1500);
+ const lowered = await runLock(["--heavy", "--", "true"], { ...env, HEAVY_MIN_FREE_MB: "1000" });
+ assert.equal(lowered.code, 0);
+ assert.doesNotMatch(lowered.stderr, /waiting for memory/);
+ const off = await runLock(["--heavy", "--", "true"], { ...env, HEAVY: "0" });
+ assert.equal(off.code, 0);
+ assert.equal(off.stderr, "");
+});
+
+test("a SIGKILLed heavy holder hands the slot on at once (the stale holder)", async () => {
+ const dir = tmpDir();
+ const env = heavyEnv(dir);
+ const victim = startLock(["--heavy", "--", process.execPath, "-e", "setTimeout(()=>{},30000)"], env);
+ await until(() => fs.existsSync(`${env.HEAVY_LOCK_FILE}.holder.json`));
+ victim.child.kill("SIGKILL");
+ await victim.done;
+ // Its holder.json is left behind (SIGKILL runs no cleanup); the lock is not.
+ const t0 = Date.now();
+ const next = await runLock(["--heavy", "--", "true"], env);
+ assert.equal(next.code, 0);
+ assert.ok(Date.now() - t0 < 5000, "the heavy slot survived a SIGKILLed holder");
+});
+
+test("pnpm's `--` separator is accepted before a heavy command", async () => {
+ const dir = tmpDir();
+ const res = await runLock(
+ ["--heavy", "--", "--", process.execPath, "-e", 'console.log("ran")'],
+ heavyEnv(dir),
+ );
+ assert.equal(res.code, 0, res.stderr);
+ assert.match(res.stdout, /ran/);
+});
+
+test("SIGTERM to a heavy wrapper alone stops its command (a stage's Cancel)", async () => {
+ const dir = tmpDir();
+ const pidFile = path.join(dir, "cmd.pid");
+ const run = startLock(
+ [
+ "--heavy",
+ "--",
+ process.execPath,
+ "-e",
+ `require("fs").writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)); setTimeout(()=>{},30000)`,
+ ],
+ heavyEnv(dir),
+ );
+ await until(() => fs.existsSync(pidFile) && fs.readFileSync(pidFile, "utf8") !== "");
+ const cmdPid = Number(fs.readFileSync(pidFile, "utf8"));
+ const t0 = Date.now();
+ run.child.kill("SIGTERM");
+ const res = await run.done;
+ assert.ok(Date.now() - t0 < 5000, "the wrapper outlived its SIGTERM");
+ assert.equal(res.code, 128 + os.constants.signals.SIGTERM);
+ assert.throws(() => process.kill(cmdPid, 0), "the command survived the wrapper's SIGTERM");
+});
diff --git a/scripts/worktree.mjs b/scripts/worktree.mjs
@@ -76,12 +76,28 @@ function offsetForIndex(index) {
// Index of the worktree containing `dir` (default: cwd) in the worktree list.
function indexForDir(dir = process.cwd()) {
const trees = listWorktrees();
- const target = realpath(dir);
- for (let i = 0; i < trees.length; i++) {
- const root = realpath(trees[i].path);
- if (target === root || target.startsWith(root + path.sep)) return i;
+ return indexForPath(
+ trees.map((t) => realpath(t.path)),
+ realpath(dir),
+ );
+}
+
+// THE MOST SPECIFIC root containing `target`, by its index in `roots` (0 when
+// none does). Not the first: a worktree NESTED in the main checkout -- every
+// `.claude/worktrees/<agent>` is -- is also "inside" the main root, which
+// comes first in the list, so a first-match gave every agent worktree the
+// main checkout's ports (offset 0) and its e2e servers collided on them.
+export function indexForPath(roots, target) {
+ let best = 0;
+ let bestLen = -1;
+ for (let i = 0; i < roots.length; i++) {
+ const root = roots[i];
+ if ((target === root || target.startsWith(root + path.sep)) && root.length > bestLen) {
+ best = i;
+ bestLen = root.length;
+ }
}
- return 0;
+ return best;
}
// Read a simple KEY=VALUE file (e.g. .worktree-env) into an object.
diff --git a/scripts/worktree.test.mjs b/scripts/worktree.test.mjs
@@ -5,7 +5,7 @@
import assert from "node:assert/strict";
import test from "node:test";
import path from "node:path";
-import { worktreeDirFor } from "./worktree.mjs";
+import { indexForPath, worktreeDirFor } from "./worktree.mjs";
const MAIN = "/home/u/Projects/yt-dlp-transcript-browser";
const SIBLING = path.dirname(MAIN);
@@ -44,3 +44,19 @@ test("nothing escapes the sibling directory", () => {
assert.equal(dir, path.join(SIBLING, "..-..-etc-passwd"));
assert.equal(path.dirname(dir), SIBLING);
});
+
+test("a worktree nested in the main checkout gets its OWN index, not the main one's", () => {
+ // `.claude/worktrees/<agent>` lives INSIDE the main checkout. A first-match
+ // walk found the main root first and gave every such worktree offset 0 --
+ // the main checkout's ports -- so two agents' umtool suites bound the same
+ // 3051/3052.
+ const roots = [MAIN, path.join(SIBLING, "feature-x"), path.join(MAIN, ".claude", "worktrees", "agent-1")];
+ assert.equal(indexForPath(roots, MAIN), 0);
+ assert.equal(indexForPath(roots, path.join(MAIN, "umtool")), 0);
+ assert.equal(indexForPath(roots, path.join(SIBLING, "feature-x", "editor")), 1);
+ assert.equal(indexForPath(roots, path.join(MAIN, ".claude", "worktrees", "agent-1")), 2);
+ assert.equal(indexForPath(roots, path.join(MAIN, ".claude", "worktrees", "agent-1", "umtool")), 2);
+ // A sibling whose name only STARTS like the main root is not inside it.
+ assert.equal(indexForPath(roots, `${MAIN}-other`), 0);
+ assert.equal(indexForPath(roots, "/elsewhere"), 0);
+});
diff --git a/umtool/app/sites/[site]/[report]/evidence/page.tsx b/umtool/app/sites/[site]/[report]/evidence/page.tsx
@@ -43,9 +43,9 @@ export default async function EvidenceWalkPage({
return (
<div className="flex h-full flex-col">
<BrowseHeader
- active="sites"
+ active="articles"
crumbs={[
- { href: "/sites", label: "sites" },
+ { href: "/sites", label: "articles" },
{ href: `/sites/${site.siteId}`, label: site.siteId },
{ href: `/sites/${site.siteId}/${reportId}`, label: reportId },
{ label: "evidence" },
diff --git a/umtool/app/sites/[site]/[report]/page.tsx b/umtool/app/sites/[site]/[report]/page.tsx
@@ -94,8 +94,8 @@ export default async function ArticlePage({
const header = (
<BrowseHeader
- active="sites"
- crumbs={[{ href: "/sites", label: "sites" }, { href: `/sites/${site.siteId}`, label: site.siteId }, { label: reportId }]}
+ active="articles"
+ crumbs={[{ href: "/sites", label: "articles" }, { href: `/sites/${site.siteId}`, label: site.siteId }, { label: reportId }]}
note={`${notes.doc?.notes.filter((n) => n.status === "open").length ?? 0} open notes`}
/>
);
diff --git a/umtool/app/sites/[site]/page.tsx b/umtool/app/sites/[site]/page.tsx
@@ -47,8 +47,8 @@ export default async function SitePage({
return (
<div className="flex h-full flex-col">
<BrowseHeader
- active="sites"
- crumbs={[{ href: "/sites", label: "sites" }, { label: row.title }]}
+ active="articles"
+ crumbs={[{ href: "/sites", label: "articles" }, { label: row.title }]}
note={`${row.published} published · ${row.drafts} drafts · ${row.openNotes} open notes`}
/>
<main className="deck-main flex-1 space-y-6 p-4">
diff --git a/umtool/app/sites/page.tsx b/umtool/app/sites/page.tsx
@@ -35,7 +35,7 @@ export default async function SitesPage({ searchParams }: { searchParams: Promis
return (
<div className="flex h-full flex-col">
- <BrowseHeader active="sites" crumbs={[{ label: "sites" }]} note={`${all.length} sites · ${articles.length} articles · ${open} open notes`} />
+ <BrowseHeader active="articles" crumbs={[{ label: "articles" }]} note={`${all.length} sites · ${articles.length} articles · ${open} open notes`} />
<main className="deck-main flex-1 p-4">
<div className="mb-3 flex flex-wrap items-center gap-1.5">
<span className="micro">site</span>
diff --git a/umtool/bin/umtool.mjs b/umtool/bin/umtool.mjs
@@ -63,6 +63,7 @@ import { diffManifests, formatChange } from "../lib/report/manifest-diff.mjs";
import { EXPORT_FORMATS, exportProject } from "../lib/report/export.mjs";
import path from "node:path";
import { updateClip, updateStorage } from "../lib/report/manifest.mjs";
+import { withEditNotes } from "../lib/report/edit-guard.mjs";
import { hms } from "umtool-report-to-video/attribution";
import { buildSteps, checkSourcesSteps, PRESETS } from "../lib/report/driver.mjs";
import { openIndex, signRecord } from "../lib/projects/index-db.mjs";
@@ -795,9 +796,11 @@ async function cmdWindow() {
try {
// Through the SAME writer the bench uses: 2 dp, the CLI's own formatting,
// tmp+rename, one .bak. A second implementation here is how the two would
- // start disagreeing about a window.
- const res = await updateClip(p.dir, clipId, patch);
- if (json) return out({ ok: true, ...res });
+ // start disagreeing about a window. And through the same GUARD the bench's
+ // routes use: on a generated manifest the change is also an `edit` note
+ // for the agent that generates it, or the next rebuild undoes it silently.
+ const { result: res, editNotes } = await withEditNotes(p, () => updateClip(p.dir, clipId, patch));
+ if (json) return out({ ok: true, ...res, editNotes });
console.log(
`${clipId}: ${res.before.start}–${res.before.end} -> ${res.entry.start}–${res.entry.end}`,
);
@@ -826,6 +829,16 @@ async function cmdWindow() {
if (patch.verdict !== undefined || patch.correction !== undefined) {
console.log(` review: ${clipVerdict(res.entry)}`);
}
+ if (editNotes) {
+ const n = editNotes.added + editNotes.updated;
+ console.log(
+ editNotes.errors.length
+ ? ` edit NOT noted (${editNotes.errors.join("; ")}) — ${editNotes.generatedBy} will overwrite it on the next rebuild`
+ : n || editNotes.deleted
+ ? ` edit noted for ${editNotes.generatedBy} (${editNotes.added} added, ${editNotes.updated} updated, ${editNotes.deleted} withdrawn) — port it into the generator's inputs`
+ : ` (generated by ${editNotes.generatedBy}; nothing changed)`,
+ );
+ }
console.log(`\nRun resolve-windows to see whether the widener agrees:`);
console.log(` node umtool/report-to-video/resolve-windows.mjs ${p.dir}/video.manifest.json`);
} catch (e) {
diff --git a/umtool/components/AppNav.tsx b/umtool/components/AppNav.tsx
@@ -6,7 +6,7 @@ import NavGroup from "./NavGroup";
// is waiting, the two benches that are not a project (mix, find), and the song
// piles folded under one entry.
//
-// SEVEN visible entries (home, browse, decisions, sites, mix, find, song ▸),
+// SEVEN visible entries (home, browse, decisions, articles, mix, find, song ▸),
// and the cap is still NINE. A tenth wraps the header on
// a laptop, and a nav that wraps stops reading as one row of places and starts
// reading as a list. The next tool goes UNDER one of these, not beside them --
@@ -32,8 +32,10 @@ export default function AppNav({ active }: { active: string }) {
// browse because that is where every decision it names gets settled.
{ href: "/browse/decisions", label: "decisions" },
// Every site's articles -- published and drafts -- with their notes, their
- // evidence and the workspace they were written in.
- { href: "/sites", label: "sites" },
+ // evidence and the workspace they were written in. Named for what it lists:
+ // the editor's /sites is the sites themselves, and two "sites" a tab apart
+ // were two places with one name. The URL stays /sites.
+ { href: "/sites", label: "articles" },
{ href: "/mix", label: "mix" },
// Every occurrence of a word across the corpus. It sits with browse because
// what it retrieves is raw material for a build, not a pile to judge.
diff --git a/umtool/components/articles/anchorDom.ts b/umtool/components/articles/anchorDom.ts
@@ -82,17 +82,39 @@ export function wrapRange(root: Element, start: number, end: number, attrs: Reco
return out;
}
-/** The block a selection lies in, and its offsets, or null (collapsed, or across blocks). */
+/**
+ * The block a selection lies in, and its offsets, or null (collapsed, or in no
+ * block of `container`).
+ *
+ * A selection that runs ACROSS blocks -- the end of one section into the next,
+ * which is what a drag past a paragraph does -- is noted in ONE of them: a text
+ * anchor names one section (lib/annotations/anchor.mjs), and that is where
+ * `umtool notes` finds it again. The block it starts in, from the start to the
+ * block's end, when that part has any text; else the block it ends in, from its
+ * beginning. It used to get no Note button at all.
+ */
export function selectionIn(container: Element): { block: Element; start: number; end: number; rect: DOMRect } | null {
const sel = window.getSelection();
if (!sel || sel.rangeCount === 0 || sel.isCollapsed) return null;
const range = sel.getRangeAt(0);
const el = (n: Node) => (n.nodeType === Node.ELEMENT_NODE ? (n as Element) : n.parentElement);
- const a = el(range.startContainer)?.closest("[data-block]");
- const b = el(range.endContainer)?.closest("[data-block]");
- if (!a || a !== b || !container.contains(a)) return null;
- const start = offsetOf(a, range.startContainer, range.startOffset);
- const end = offsetOf(a, range.endContainer, range.endOffset);
- if (end <= start) return null;
- return { block: a, start, end, rect: range.getBoundingClientRect() };
+ const within = (e: Element | null | undefined) => (e && container.contains(e) ? e : null);
+ const a = within(el(range.startContainer)?.closest("[data-block]"));
+ const b = within(el(range.endContainer)?.closest("[data-block]"));
+ const rect = range.getBoundingClientRect();
+ if (a && a === b) {
+ const start = offsetOf(a, range.startContainer, range.startOffset);
+ const end = offsetOf(a, range.endContainer, range.endOffset);
+ return end > start ? { block: a, start, end, rect } : null;
+ }
+ if (a) {
+ const start = offsetOf(a, range.startContainer, range.startOffset);
+ const text = blockText(a).text;
+ if (text.slice(start).trim()) return { block: a, start, end: text.length, rect };
+ }
+ if (b) {
+ const end = offsetOf(b, range.endContainer, range.endOffset);
+ if (blockText(b).text.slice(0, end).trim()) return { block: b, start: 0, end, rect };
+ }
+ return null;
}
diff --git a/umtool/docs/cli.md b/umtool/docs/cli.md
@@ -51,7 +51,13 @@ projects answering to one name is reported, never resolved by picking one.
| `REPORTS_DIR` | `~/reports` (the parent of `SONG_REPORTS_DIR` when that is set) |
| `UMTOOL_MEDIA_DIR` | unset = `REPORTS_DIR`: `out/` stays in each project. Set, each project's `out` is a link to the same path under it ([folders.md](folders.md)) |
| `UMTOOL_CACHE_DIR` | `$XDG_CACHE_HOME/archilyzer/umtool`, else `~/.cache/archilyzer/umtool` (it was `<SONG_DIR>/.cache/umtool`) |
-| `CHANNELS_DIR` | `$TRANSCRIPTS_DIR/channels`, else the checkout's `transcripts/channels` (found by walking up from the cwd to `pnpm-workspace.yaml`) |
+| `CHANNELS_DIR` | `$TRANSCRIPTS_DIR/channels`, else the checkout's `transcripts/channels` |
+| `SITES_DIR` | `$TRANSCRIPTS_DIR/sites`, else the checkout's `transcripts/sites` |
+
+"The checkout" is the one the CLI script itself lives in (`<repo>/umtool/bin/umtool.mjs`),
+whatever directory it is run from — `umtool notes --all` from a report workspace under
+`REPORTS_DIR` reads the corpus's sites. The app (`next dev`/`start`) finds it by walking up
+from its cwd to `pnpm-workspace.yaml`.
| `VIDEO_ROOT` (`song/spec.mjs`, `song/video-dir.mjs`) | `~/reports/quartering-uh-song/videos` |
## `check` is the one to run before every build
@@ -85,7 +91,16 @@ many projects it only checked the routing of, and points at `/browse/decisions`.
## `window` goes through the same writer the bench does
2 dp, the CLI's own formatting, tmp+rename, one `.bak`. A second implementation is
-how the two would start disagreeing about where a clip ends.
+how the two would start disagreeing about where a clip ends. And through the same guard
+(`lib/report/edit-guard.mjs`): on a GENERATED manifest (`generatedBy`) each change is
+also an `edit` note in the project's notes.json, for the agent to port into the
+generator's inputs — the next rebuild would otherwise undo it without a trace:
+
+```
+$ umtool window polemic-x e1 --start 11
+e1: 10–20 -> 11–20
+ edit noted for polemics/video/make-videos.py (1 added, 0 updated, 0 withdrawn) — port it into the generator's inputs
+```
```
$ umtool window ferret-rescue c01 --start 43.12 --end 61.48 --lock-end
diff --git a/umtool/e2e/article-notes.spec.ts b/umtool/e2e/article-notes.spec.ts
@@ -71,6 +71,45 @@ test("select text, Note, save: a mark on the quote, a note beside report.json",
await expect(page.locator("mark[data-note]")).toHaveAttribute("data-active", "true");
});
+// A drag that runs past the end of a section into the next one: the note is
+// anchored in the section it STARTED in, from there to that section's end (a
+// text anchor names one section). It used to get no Note button at all.
+test("a selection across two sections gets a Note, anchored in the first", async ({ page }) => {
+ await page.goto(PAGE);
+ await page.evaluate(() => {
+ const at = (block: string, text: string): [Text, number] => {
+ const root = document.querySelector(`[data-block="${block}"]`)!;
+ const w = document.createTreeWalker(root, NodeFilter.SHOW_TEXT);
+ for (let n = w.nextNode() as Text | null; n; n = w.nextNode() as Text | null) {
+ const i = n.data.indexOf(text);
+ if (i >= 0) return [n, i];
+ }
+ throw new Error(`no "${text}" in ${block}`);
+ };
+ const [a, i] = at("first", "Nobody checked the claim");
+ const [b, j] = at("later", "on a different show");
+ const r = document.createRange();
+ r.setStart(a, i);
+ r.setEnd(b, j + "on a different".length);
+ const s = getSelection()!;
+ s.removeAllRanges();
+ s.addRange(r);
+ });
+ await page.locator('[data-block="later"]').dispatchEvent("mouseup");
+ await page.getByRole("button", { name: "Note", exact: true }).click();
+ await page.getByLabel("note text").fill("This runs on.");
+ await page.getByRole("button", { name: "save note" }).click();
+
+ // The mark is drawn once the note is saved: then the file is there.
+ await expect(page.locator('[data-block="first"] mark[data-note]').first()).toBeVisible();
+ await expect(page.locator('[data-block="later"] mark[data-note]')).toHaveCount(0);
+ const anchor = JSON.parse(readFileSync(NOTES, "utf8")).notes[0].anchor;
+ expect(anchor).toMatchObject({ kind: "text", section: "first" });
+ expect(anchor.quote.startsWith("Nobody checked the claim")).toBe(true);
+ expect(anchor.quote).toContain("for the record.");
+ expect(anchor.quote).not.toContain("different show");
+});
+
test("section, whole-article and citation notes; resolve, reopen, reply, filters", async ({ page }) => {
await page.goto(PAGE);
await page.getByRole("button", { name: "note on Later" }).click();
diff --git a/umtool/e2e/mix.spec.ts b/umtool/e2e/mix.spec.ts
@@ -1,4 +1,7 @@
import { test, expect } from "@playwright/test";
+import { existsSync, mkdirSync, readdirSync, renameSync, rmSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
// The mix bench, against SYNTHESISED tracks whose true answers are known in
// advance (see make-fixture.mjs):
@@ -15,6 +18,65 @@ import { test, expect } from "@playwright/test";
const BG = "bg.mp4";
const SONG = "song.mp4";
+// THE CORPUS'S CLIP WINDOWS, SET ASIDE FOR THIS FILE.
+//
+// The bench folds the corpus's clip windows (channels/<slug>/data/<id>/clips/,
+// what the editor's fetch writes) in with a project's own clips-raw. The
+// fixture is built once per run and specs before this one fetch windows
+// through the editor stub (testchan/vid1 0-14, say), so which file a clip
+// links to, and whether c02 is fetched at all, depended on what ran first:
+// `:166` and `:201` failed in every full run and passed alone (FACTS, "mix.spec.ts
+// IS ORDER-DEPENDENT"). These tests are about the project's clips-raw, so they
+// start with the corpus windows moved aside, and put them back after, for the
+// specs that follow.
+const FIXTURE = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", ".e2e-song");
+const CHANNELS = path.join(FIXTURE, "channels");
+const ASIDE = path.join(FIXTURE, "mix-spec-clips-aside");
+
+function clipDirs(): string[] {
+ const out: string[] = [];
+ if (!existsSync(CHANNELS)) return out;
+ for (const slug of readdirSync(CHANNELS)) {
+ const data = path.join(CHANNELS, slug, "data");
+ if (!existsSync(data)) continue;
+ for (const id of readdirSync(data)) {
+ const clips = path.join(data, id, "clips");
+ if (existsSync(clips)) out.push(path.relative(CHANNELS, clips));
+ }
+ }
+ return out;
+}
+
+function restoreClips() {
+ if (!existsSync(ASIDE)) return;
+ for (const rel of clipDirsUnder(ASIDE)) {
+ const back = path.join(CHANNELS, rel);
+ if (existsSync(back)) rmSync(back, { recursive: true, force: true });
+ mkdirSync(path.dirname(back), { recursive: true });
+ renameSync(path.join(ASIDE, rel), back);
+ }
+ rmSync(ASIDE, { recursive: true, force: true });
+}
+
+function clipDirsUnder(root: string): string[] {
+ const out: string[] = [];
+ for (const slug of readdirSync(root)) {
+ const data = path.join(root, slug, "data");
+ if (!existsSync(data)) continue;
+ for (const id of readdirSync(data)) out.push(path.join(slug, "data", id, "clips"));
+ }
+ return out;
+}
+
+test.beforeAll(() => {
+ restoreClips(); // a run killed mid-file left some aside
+ for (const rel of clipDirs()) {
+ mkdirSync(path.dirname(path.join(ASIDE, rel)), { recursive: true });
+ renameSync(path.join(CHANNELS, rel), path.join(ASIDE, rel));
+ }
+});
+test.afterAll(restoreClips);
+
test("the analysis finds a cue an envelope cannot see", async ({ request }) => {
const r = await request.get(`/api/mix/track?path=${encodeURIComponent(BG)}`);
expect(r.ok()).toBe(true);
@@ -112,7 +174,9 @@ test("the bench loads, lists real tracks, and hides render scratch", async ({ pa
await expect(page.getByRole("link", { name: "mix", exact: true })).toBeVisible();
const body = page.locator("select").first();
- await expect(body.locator("option", { hasText: "song.mp4" })).toHaveCount(1);
+ // The label exactly: `hasText` is a substring match, and a render another
+ // spec left (`…song.mp4`) counted as a second song.
+ await expect(body.locator("option", { hasText: /^song\.mp4$/ })).toHaveCount(1);
// polytmp-*/ and poly-song-*.wav are working files, never offerable.
await expect(page.locator("option").filter({ hasText: /poly-song-|polytmp-/ })).toHaveCount(0);
});
diff --git a/umtool/e2e/sites.spec.ts b/umtool/e2e/sites.spec.ts
@@ -20,7 +20,9 @@ test.beforeAll(async ({ playwright }) => {
test("/sites lists every site, private first, with published and draft articles", async ({ page }) => {
const res = await page.goto("/sites");
expect(res?.status()).toBe(200);
- await expect(page.getByRole("link", { name: "sites", exact: true }).first()).toHaveAttribute("aria-current", "page");
+ // The nav calls it "articles" (the editor's /sites is the sites themselves).
+ await expect(page.getByRole("link", { name: "articles", exact: true }).first()).toHaveAttribute("aria-current", "page");
+ await expect(page.getByRole("link", { name: "sites", exact: true })).toHaveCount(0);
const sites = page.locator("[data-site]");
await expect(sites).toHaveCount(2);
diff --git a/umtool/lib/paths.mjs b/umtool/lib/paths.mjs
@@ -4,7 +4,7 @@
// directory is read and which is written must not be able to differ between
// `umtool ls` and the page it is supposed to describe. lib/paths.ts re-exports
// everything here with types; nothing computes a root twice.
-import { existsSync } from "node:fs";
+import { existsSync, realpathSync } from "node:fs";
import { lstat, realpath } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
@@ -199,7 +199,35 @@ export function findRepoRoot(start) {
}
}
-export const REPO_ROOT = findRepoRoot(process.cwd());
+/**
+ * The checkout a umtool CLI belongs to -- `<repo>/umtool/bin/<cli>.mjs` as the
+ * entry script (`process.argv[1]`) -- or null for anything else (the Next
+ * server, a test runner, report-to-video's own CLIs).
+ *
+ * A CLI is run from wherever its user stands: `node ~/…/umtool/bin/umtool.mjs
+ * notes --all` from a report workspace under REPORTS_DIR has no
+ * pnpm-workspace.yaml above its cwd, so the cwd walk fell back to the cwd's
+ * parent and SITES_DIR (and CHANNELS_DIR) pointed at nothing. The script's own
+ * path names the checkout it is from. Only the bin/ entries: the server keeps
+ * the cwd walk, for the reason findRepoRoot gives.
+ *
+ * @param {string | undefined} entry
+ * @returns {string | null}
+ */
+export function cliRepoRoot(entry) {
+ if (!entry || !/[\\/]umtool[\\/]bin[\\/][^\\/]+\.mjs$/.test(entry)) return null;
+ let real;
+ try {
+ real = realpathSync(/* turbopackIgnore: true */ entry);
+ } catch {
+ return null;
+ }
+ // <repo>/umtool/bin/x.mjs -> <repo>, when <repo> is a checkout.
+ const repo = path.resolve(/* turbopackIgnore: true */ path.dirname(/* turbopackIgnore: true */ real), "..", "..");
+ return existsSync(path.join(/* turbopackIgnore: true */ repo, "pnpm-workspace.yaml")) ? repo : null;
+}
+
+export const REPO_ROOT = cliRepoRoot(process.argv[1]) ?? findRepoRoot(process.cwd());
export const CHANNELS_DIR = path.resolve(
/* turbopackIgnore: true */
diff --git a/umtool/lib/paths.test.mjs b/umtool/lib/paths.test.mjs
@@ -0,0 +1,69 @@
+// Where a umtool CLI finds the corpus: from its OWN checkout (the entry
+// script's path), whatever directory it is run from.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { cliRepoRoot } from "./paths.mjs";
+
+const PATHS = new URL("./paths.mjs", import.meta.url);
+
+function checkout() {
+ const root = mkdtempSync(path.join(tmpdir(), "umtool-paths-"));
+ writeFileSync(path.join(root, "pnpm-workspace.yaml"), "packages: []\n");
+ mkdirSync(path.join(root, "umtool", "bin"), { recursive: true });
+ return root;
+}
+
+test("cliRepoRoot: a umtool/bin entry names its checkout; anything else is null", (t) => {
+ const root = checkout();
+ t.after(() => rmSync(root, { recursive: true, force: true }));
+ const cli = path.join(root, "umtool", "bin", "umtool.mjs");
+ writeFileSync(cli, "");
+ assert.equal(cliRepoRoot(cli), root);
+ // The server, a test file, report-to-video's CLIs: the cwd walk decides.
+ assert.equal(cliRepoRoot(path.join(root, "node_modules", "next", "dist", "bin", "next")), null);
+ assert.equal(cliRepoRoot(path.join(root, "umtool", "report-to-video", "build-video.mjs")), null);
+ assert.equal(cliRepoRoot(undefined), null);
+ // A bin/ whose grandparent is not a checkout, and one that does not exist.
+ const loose = mkdtempSync(path.join(tmpdir(), "umtool-loose-"));
+ t.after(() => rmSync(loose, { recursive: true, force: true }));
+ mkdirSync(path.join(loose, "umtool", "bin"), { recursive: true });
+ writeFileSync(path.join(loose, "umtool", "bin", "x.mjs"), "");
+ assert.equal(cliRepoRoot(path.join(loose, "umtool", "bin", "x.mjs")), null);
+ assert.equal(cliRepoRoot(path.join(root, "umtool", "bin", "missing.mjs")), null);
+ // Reached through a link: the TARGET's checkout, not the link's.
+ const link = path.join(loose, "umtool", "bin", "linked.mjs");
+ symlinkSync(cli, link);
+ assert.equal(cliRepoRoot(link), root);
+});
+
+test("a CLI run from outside its checkout reads that checkout's sites and channels", (t) => {
+ const root = checkout();
+ t.after(() => rmSync(root, { recursive: true, force: true }));
+ const probe = path.join(root, "umtool", "bin", "probe.mjs");
+ writeFileSync(
+ probe,
+ `const p = await import(${JSON.stringify(PATHS.href)});\n` +
+ "console.log(JSON.stringify({ repo: p.REPO_ROOT, sites: p.SITES_DIR, channels: p.CHANNELS_DIR }));\n",
+ );
+ const away = mkdtempSync(path.join(tmpdir(), "umtool-away-"));
+ t.after(() => rmSync(away, { recursive: true, force: true }));
+ const env = { ...process.env };
+ for (const k of ["SITES_DIR", "CHANNELS_DIR", "TRANSCRIPTS_DIR"]) delete env[k];
+ const got = JSON.parse(execFileSync(process.execPath, [probe], { cwd: away, env, encoding: "utf8" }));
+ assert.deepEqual(got, {
+ repo: root,
+ sites: path.join(root, "transcripts", "sites"),
+ channels: path.join(root, "transcripts", "channels"),
+ });
+ // SITES_DIR still wins when it is set.
+ const set = JSON.parse(
+ execFileSync(process.execPath, [probe], { cwd: away, env: { ...env, SITES_DIR: away }, encoding: "utf8" }),
+ );
+ assert.equal(set.sites, away);
+});
diff --git a/umtool/lib/report/edit-guard.mjs b/umtool/lib/report/edit-guard.mjs
@@ -0,0 +1,46 @@
+// THE one wrapper every manifest writer goes through -- the routes (through
+// lib/report/guard.ts, which types it) and the `umtool window` CLI alike.
+//
+// A manifest with `generatedBy` is rebuilt by its generator, and the rebuild
+// overwrites edits made here (the banner on the project page, the bench and
+// the On-screen section says so). The edit is still made; what this adds is a
+// record of it: the manifest is read before and after the write, and every
+// change becomes an `edit` note in the project's notes.json for the agent to
+// port into the generator's inputs (./edit-notes.mjs). A hand-edited manifest
+// (no `generatedBy`) is written exactly as before.
+//
+// The notes are written AFTER the manifest and never fail the write: the edit
+// is saved either way, and `editNotes.errors` says when its note is not.
+//
+// Plain .mjs so the CLI can run it with bare node; the routes read it through
+// guard.ts.
+import { projectTargetFor } from "../annotations/targets.mjs";
+import { readManifest } from "../projects/report.mjs";
+import { editsBetween, recordEdits } from "./edit-notes.mjs";
+
+/**
+ * @template T
+ * @param {{ id: string, dir: string, kind: string }} project
+ * @param {() => Promise<T>} write
+ * @param {{ reportsRoot?: string }} [opts] the reports root notes may live under (tests)
+ * @returns {Promise<{ result: T, editNotes: { generatedBy: string, added: number, updated: number, deleted: number, errors: string[] } | null }>}
+ */
+export async function withEditNotes(project, write, opts = {}) {
+ const before = await readManifest(project.dir);
+ const result = await write();
+ const generatedBy = typeof before?.generatedBy === "string" ? before.generatedBy.trim() : "";
+ if (!generatedBy) return { result, editNotes: null };
+ const after = await readManifest(project.dir);
+ const edits = editsBetween(before, after);
+ if (!edits.length) return { result, editNotes: { generatedBy, added: 0, updated: 0, deleted: 0, errors: [] } };
+ try {
+ const target = await projectTargetFor(project, opts.reportsRoot ? { reportsRoot: opts.reportsRoot } : undefined);
+ const counts = await recordEdits(target, edits, generatedBy);
+ return { result, editNotes: { generatedBy, ...counts } };
+ } catch (e) {
+ return {
+ result,
+ editNotes: { generatedBy, added: 0, updated: 0, deleted: 0, errors: [e instanceof Error ? e.message : String(e)] },
+ };
+ }
+}
diff --git a/umtool/lib/report/edit-guard.test.mjs b/umtool/lib/report/edit-guard.test.mjs
@@ -0,0 +1,78 @@
+// The manifest writers' guard (edit-guard.mjs) and `umtool window` through it:
+// an edit to a GENERATED manifest is an `edit` note for its generator; a
+// hand-edited manifest gets none.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+import { readNotes } from "../annotations/store.mjs";
+import { PROJECT_KINDS, kindTakesNotes } from "../projects/kinds.mjs";
+import { updateClip } from "./manifest.mjs";
+import { withEditNotes } from "./edit-guard.mjs";
+
+const CLI = fileURLToPath(new URL("../../bin/umtool.mjs", import.meta.url));
+// A kind that takes notes, from the registry -- never named here (projects.spec
+// refuses a kind id outside lib/projects/).
+const NOTES_KIND = PROJECT_KINDS.find((k) => kindTakesNotes(k.id)).id;
+
+async function project(manifest) {
+ const root = await mkdtemp(path.join(tmpdir(), "umtool-editguard-"));
+ const dir = path.join(root, "ws", "clipcut");
+ await mkdir(dir, { recursive: true });
+ await writeFile(path.join(dir, "video.manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);
+ return { root, dir, project: { id: "ws/clipcut", dir, kind: NOTES_KIND } };
+}
+
+const MANIFEST = (generated) => ({
+ schemaVersion: 1,
+ slug: "clipcut",
+ ...(generated ? { generatedBy: "polemics/video/make-videos.py" } : {}),
+ timeline: [{ type: "clip", id: "e1", channel: "ch", video: "v1", start: 10, end: 20 }],
+});
+
+test("an edit to a generated manifest is written, and noted for its generator", async () => {
+ const { root, dir, project: p } = await project(MANIFEST(true));
+ try {
+ const { result, editNotes } = await withEditNotes(p, () => updateClip(dir, "e1", { start: 12 }), { reportsRoot: root });
+ assert.equal(result.entry.start, 12);
+ assert.deepEqual(editNotes, { generatedBy: "polemics/video/make-videos.py", added: 1, updated: 0, deleted: 0, errors: [] });
+ const { doc } = await readNotes(path.join(dir, "notes.json"));
+ assert.equal(doc.notes.length, 1);
+ assert.deepEqual(doc.notes[0].anchor, { kind: "edit", entry: "e1", field: "start", from: 10, to: 12 });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("a hand-edited manifest is written with no note", async () => {
+ const { root, dir, project: p } = await project(MANIFEST(false));
+ try {
+ const { editNotes } = await withEditNotes(p, () => updateClip(dir, "e1", { start: 12 }), { reportsRoot: root });
+ assert.equal(editNotes, null);
+ await assert.rejects(readFile(path.join(dir, "notes.json")));
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("`umtool window` on a generated manifest says it noted the edit, and the note is there", async () => {
+ const { root, dir } = await project(MANIFEST(true));
+ try {
+ const out = execFileSync(process.execPath, [CLI, "window", dir, "e1", "--start", "11", "--end", "21"], {
+ cwd: root,
+ env: { ...process.env, REPORTS_DIR: root },
+ encoding: "utf8",
+ });
+ assert.match(out, /e1: 10–20 -> 11–21/);
+ assert.match(out, /edit noted for polemics\/video\/make-videos\.py \(2 added/);
+ const { doc } = await readNotes(path.join(dir, "notes.json"));
+ assert.deepEqual(doc.notes.map((n) => n.anchor.field).sort(), ["end", "start"]);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
diff --git a/umtool/lib/report/guard.ts b/umtool/lib/report/guard.ts
@@ -1,19 +1,10 @@
-import { projectTargetFor } from "@/lib/annotations/targets.mjs";
-import { readManifest } from "@/lib/projects/report.mjs";
-import { editsBetween, recordEdits } from "./edit-notes.mjs";
+import { withEditNotes as withEditNotesMjs } from "./edit-guard.mjs";
-// THE one wrapper every manifest writer's route goes through.
-//
-// A manifest with `generatedBy` is rebuilt by its generator, and the rebuild
-// overwrites edits made here (the banner on the project page, the bench and
-// the On-screen section says so). The edit is still made; what this adds is a
-// record of it: the manifest is read before and after the write, and every
-// change becomes an `edit` note in the project's notes.json for the agent to
-// port into the generator's inputs (lib/report/edit-notes.mjs). A hand-edited
-// manifest (no `generatedBy`) is written exactly as before.
-//
-// The notes are written AFTER the manifest and never fail the request: the
-// edit is saved either way, and `editNotes.errors` says when its note is not.
+// THE one wrapper every manifest writer's route goes through: the manifest is
+// read before and after the write, and on a GENERATED manifest every change
+// becomes an `edit` note for the agent that generates it. The implementation
+// is ./edit-guard.mjs, plain JS so the `umtool window` CLI runs the same one;
+// this file only types it for the routes.
export type EditNotes = { generatedBy: string; added: number; updated: number; deleted: number; errors: string[] };
@@ -21,18 +12,5 @@ export async function withEditNotes<T>(
project: { id: string; dir: string; kind: string },
write: () => Promise<T>,
): Promise<{ result: T; editNotes: EditNotes | null }> {
- const before = await readManifest(project.dir);
- const result = await write();
- const generatedBy = typeof before?.generatedBy === "string" ? before.generatedBy.trim() : "";
- if (!generatedBy) return { result, editNotes: null };
- const after = await readManifest(project.dir);
- const edits = editsBetween(before, after);
- if (!edits.length) return { result, editNotes: { generatedBy, added: 0, updated: 0, deleted: 0, errors: [] } };
- try {
- const target = await projectTargetFor(project);
- const counts = await recordEdits(target, edits, generatedBy);
- return { result, editNotes: { generatedBy, ...counts } };
- } catch (e) {
- return { result, editNotes: { generatedBy, added: 0, updated: 0, deleted: 0, errors: [e instanceof Error ? e.message : String(e)] } };
- }
+ return withEditNotesMjs(project, write) as Promise<{ result: T; editNotes: EditNotes | null }>;
}
diff --git a/umtool/package.json b/umtool/package.json
@@ -8,7 +8,7 @@
"build": "next build",
"start": "next start --port ${UMTOOL_PORT:-3050}",
"typecheck": "tsc --noEmit",
- "e2e": "node ../scripts/queue-lock.mjs --ports UMTOOL_E2E_PORT:3051,EDITOR_STUB_PORT:3052 -- playwright test"
+ "e2e": "node ../scripts/worktree.mjs run -- node ../scripts/queue-lock.mjs --ports UMTOOL_E2E_PORT:3051,EDITOR_STUB_PORT:3052 -- playwright test"
},
"dependencies": {
"class-variance-authority": "^0.7.1",
