commit 7a7a8e9bd138a780aa801eb157636fb18483a443
parent 82b375cd86ab9d3ec55a43771c34d7b62625e0c9
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 6 Oct 2026 08:24:22 -0400
docker: the runtime image publishes — python, pipx and a pinned git-filter-repo; a yt-dlp substitution hook; build stamps
- runtime-base and runtime-cuda: python3 + yt-dlp's optional modules, pipx,
`pipx install git-filter-repo==2.47.0` into /opt/pipx + /usr/local/bin (a drift
test holds it to FILTER_REPO_PIPX_SPEC); ARCHILYZER_IMAGE_YTDLP beside
YTDLP_BIN; /usr/local/bin/yt-dlp-from-source (docker/yt-dlp-from-source.sh);
ARCHILYZER_COMMIT/ARCHILYZER_BRANCH build args -> ENV, last in each target.
- entrypoint: a `yt-dlp: <path> <version> (image|override)` line on every editor
boot (MISSING is a warning); YTDLP_AUTO_UPDATE skips an override with a
warning; the source mount is a git safe.directory (added once); the homepage
serves ARCHILYZER_HOMEPAGE_OUT when it is non-empty.
- compose: ARCHILYZER_CONFIG_DIR=/data/config/archilyzer and
ARCHILYZER_HOMEPAGE_OUT in x-app-env; build args; the homepage mounts builds;
overlays docker-compose.source.yml and docker-compose.ytdlp.yml.
- docker/publish-site.sh is a wrapper over publish index/build/deploy --to local.
- envVars: the publish credentials, the image's facts (IMAGE_COMMIT_ENV,
IMAGE_BRANCH_ENV, imageBuildFacts), the overlays' variables; ENVIRONMENT.md
regenerated.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
10 files changed, 355 insertions(+), 61 deletions(-)
diff --git a/Dockerfile b/Dockerfile
@@ -303,15 +303,34 @@ FROM ${RUNTIME_IMAGE} AS runtime-base
# from a `docker compose exec` shell. debian:slim ships without it.
# aria2: archive.org files over BitTorrent (archive.org as the web seed), then
# seeded for a while; without it they are downloaded directly.
+# python3 + yt-dlp's optional modules (certifi, brotli, websockets, mutagen,
+# pycryptodome, requests): NOT for the yt-dlp below, which is a standalone
+# binary with its own python — for /usr/local/bin/yt-dlp-from-source, which
+# runs a yt-dlp SOURCE tree mounted at run time (docker-compose.ytdlp.yml).
+# pipx (+ python3-venv, which it needs to make a venv): installs git-filter-repo
+# below, which the homepage's source mirror runs.
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ffmpeg aria2 zip unzip tar xz-utils gzip rsync curl ca-certificates git procps \
+ python3 python3-venv pipx python3-certifi python3-brotli python3-websockets \
+ python3-mutagen python3-pycryptodome python3-requests \
&& rm -rf /var/lib/apt/lists/*
+# git-filter-repo, PINNED, for `archilyzer source publish` (the homepage's
+# /source mirror). Installed, not left to `pipx run`, so a container publishes
+# with no network fetch at build time. The version is the one
+# common/publish/source.ts names in FILTER_REPO_PIPX_SPEC — a drift test
+# (common/publish/buildImage.test.ts) holds every `pipx install` here to it.
+# Into /opt + /usr/local/bin, not root's home: on PATH for every process.
+RUN PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin pipx install git-filter-repo==2.47.0 \
+ && git filter-repo --version
+
# yt-dlp as the standalone release binary (it bundles its own python), installed
# writable so `yt-dlp -U` works — see YTDLP_AUTO_UPDATE in docker/entrypoint.sh.
# A pinned yt-dlp goes stale fast, and a stale yt-dlp is the single most common
-# reason downloads start failing.
+# reason downloads start failing. It is THE IMAGE'S yt-dlp
+# (ARCHILYZER_IMAGE_YTDLP below); YTDLP_BIN may name another one at run time —
+# RUNNING_IN_DOCKER.md, "Substituting yt-dlp".
ARG TARGETARCH=amd64
RUN set -eux; \
case "${TARGETARCH}" in \
@@ -353,12 +372,21 @@ RUN npm install -g pnpm@9.15.4
WORKDIR /repo
COPY --from=build /repo /repo
+# The yt-dlp substitution hook: a wrapper that runs a yt-dlp SOURCE tree
+# mounted at YTDLP_SOURCE_DIR (default /opt/yt-dlp-src) with the python above.
+# Selected by YTDLP_BIN=/usr/local/bin/yt-dlp-from-source; see
+# docker-compose.ytdlp.yml.
+RUN ln -s /repo/docker/yt-dlp-from-source.sh /usr/local/bin/yt-dlp-from-source
+
# Data lives in volumes, never in the image. See docker-compose.yml.
+# YTDLP_BIN is the one the app runs; ARCHILYZER_IMAGE_YTDLP is the one this
+# image ships, so the entrypoint and `archilyzer doctor` can say "override".
ENV NODE_ENV=production \
NEXT_TELEMETRY_DISABLED=1 \
TRANSCRIPTS_DIR=/data/transcripts \
SETTINGS_FILE=/data/config/settings.json \
YTDLP_BIN=/usr/local/bin/yt-dlp \
+ ARCHILYZER_IMAGE_YTDLP=/usr/local/bin/yt-dlp \
EXPORT_INDEX_DIR=/data/builds/.export-index \
EXPORT_BUILDS_DIR=/data/builds/.export-builds \
ARCHILYZER_SITE_OUT=/data/builds/site
@@ -376,6 +404,15 @@ ENV ARCHILYZER_TRANSCRIBER=whisper-cpp \
WHISPER_BIN=/usr/local/bin/whisper-cli \
WHISPER_MODEL=/data/models/ggml-base.en.bin
+# Which commit and branch this image was built from — the publish stamps'
+# `commit`/`branch` where there is no .git (.dockerignore keeps it out). Last,
+# so a new commit re-runs one ENV layer and nothing else. Passed by compose from
+# the shell: ARCHILYZER_COMMIT=$(git rev-parse HEAD) docker compose build.
+ARG ARCHILYZER_COMMIT=
+ARG ARCHILYZER_BRANCH=
+ENV ARCHILYZER_COMMIT=${ARCHILYZER_COMMIT} \
+ ARCHILYZER_BRANCH=${ARCHILYZER_BRANCH}
+
# ---------------------------------------------------------------------------
# runtime-vulkan — parakeet.cpp on any Vulkan GPU. docker-compose.vulkan.yml.
#
@@ -405,18 +442,32 @@ ENV ARCHILYZER_TRANSCRIBER=parakeet \
WHISPER_BIN=/usr/local/bin/whisper-cli \
WHISPER_MODEL=/data/models/ggml-base.en.bin
+# Which commit and branch this image was built from — the publish stamps'
+# `commit`/`branch` where there is no .git (.dockerignore keeps it out). Last,
+# so a new commit re-runs one ENV layer and nothing else. Passed by compose from
+# the shell: ARCHILYZER_COMMIT=$(git rev-parse HEAD) docker compose build.
+ARG ARCHILYZER_COMMIT=
+ARG ARCHILYZER_BRANCH=
+ENV ARCHILYZER_COMMIT=${ARCHILYZER_COMMIT} \
+ ARCHILYZER_BRANCH=${ARCHILYZER_BRANCH}
+
# ---------------------------------------------------------------------------
# runtime-cuda — whisper.cpp on CUDA. NVIDIA only. docker-compose.gpu.yml.
# ---------------------------------------------------------------------------
FROM ${CUDA_RUNTIME_IMAGE} AS runtime-cuda
ARG NODE_MAJOR=20
+# The same python + pipx + git-filter-repo as runtime-base (read its comments).
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ffmpeg aria2 zip unzip tar xz-utils gzip rsync curl ca-certificates git gnupg procps \
+ python3 python3-venv pipx python3-certifi python3-brotli python3-websockets \
+ python3-mutagen python3-pycryptodome python3-requests \
&& curl -fsSL "https://deb.nodesource.com/setup_${NODE_MAJOR}.x" | bash - \
&& apt-get install -y --no-install-recommends nodejs \
&& rm -rf /var/lib/apt/lists/*
+RUN PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin pipx install git-filter-repo==2.47.0 \
+ && git filter-repo --version
ARG TARGETARCH=amd64
RUN set -eux; \
@@ -461,6 +512,7 @@ RUN ldconfig
WORKDIR /repo
COPY --from=build /repo /repo
+RUN ln -s /repo/docker/yt-dlp-from-source.sh /usr/local/bin/yt-dlp-from-source
ENV NODE_ENV=production \
NEXT_TELEMETRY_DISABLED=1 \
@@ -470,6 +522,7 @@ ENV NODE_ENV=production \
WHISPER_BIN=/usr/local/bin/whisper-cli \
WHISPER_MODEL=/data/models/ggml-base.en.bin \
YTDLP_BIN=/usr/local/bin/yt-dlp \
+ ARCHILYZER_IMAGE_YTDLP=/usr/local/bin/yt-dlp \
EXPORT_INDEX_DIR=/data/builds/.export-index \
EXPORT_BUILDS_DIR=/data/builds/.export-builds \
ARCHILYZER_SITE_OUT=/data/builds/site
@@ -477,3 +530,12 @@ RUN mkdir -p /data/transcripts /data/config /data/models /data/builds
ENTRYPOINT ["/repo/docker/entrypoint.sh"]
CMD ["editor"]
+
+# Which commit and branch this image was built from — the publish stamps'
+# `commit`/`branch` where there is no .git (.dockerignore keeps it out). Last,
+# so a new commit re-runs one ENV layer and nothing else. Passed by compose from
+# the shell: ARCHILYZER_COMMIT=$(git rev-parse HEAD) docker compose build.
+ARG ARCHILYZER_COMMIT=
+ARG ARCHILYZER_BRANCH=
+ENV ARCHILYZER_COMMIT=${ARCHILYZER_COMMIT} \
+ ARCHILYZER_BRANCH=${ARCHILYZER_BRANCH}
diff --git a/ENVIRONMENT.md b/ENVIRONMENT.md
@@ -40,7 +40,7 @@ The one override surface for where things live and which binary runs. Every one
| `ARIA2C_BIN` | `aria2c` on PATH | Fetches an archive.org file over BitTorrent (the item's torrent, archive.org as web seed), then seeds it for a while. Optional: without it archive.org files are downloaded directly. See `archiveOrg` in [SETTINGS.md](SETTINGS.md). | common/lib/paths.ts (getPaths) |
| `OLLAMA_URL` | `http://127.0.0.1:11434` | The local ollama server, the local digest and attribution engine. | common/lib/paths.ts (getPaths) |
| `CLAUDE_BIN` | `claude` on PATH | The `claude` CLI, driving the opt-in metered digest lane. | common/lib/paths.ts (getPaths) |
-| `ARCHILYZER_CONFIG_DIR` | `~/.config/archilyzer` | The operator's private config dir, outside the repo: the two inputs of `archilyzer source publish` below. Never committed. | common/lib/paths.ts (getPaths) |
+| `ARCHILYZER_CONFIG_DIR` | `~/.config/archilyzer` | The operator's private config dir, outside the repo: the two inputs of `archilyzer source publish` below. Never committed. In Docker it is `/data/config/archilyzer`, in the config volume (docker-compose.yml). | common/lib/paths.ts (getPaths) |
| `SOURCE_SCRUB_FILE` | `<ARCHILYZER_CONFIG_DIR>/source-scrub.txt` | git-filter-repo `lhs==>rhs` rules applied to file contents AND commit messages when the source mirror is generated (`<home dir>==>/home/user` is built in and runs first). Every rule's left side is also denied. See [PUBLISH.md](PUBLISH.md). | common/lib/paths.ts (getPaths) |
| `SOURCE_DENYLIST_FILE` | `<ARCHILYZER_CONFIG_DIR>/source-denylist.txt` | Literals the published source must never contain, one per line (`i:` = any case). One hit anywhere in the mirror, the tree or the tarball refuses the publish. | common/lib/paths.ts (getPaths) |
| `ARCHILYZER_SOURCE_SCRATCH` | the OS temp dir | Where `source publish` makes its scratch clone and stage (removed afterwards unless `--keep-scratch`). | common/lib/paths.ts (getPaths) |
@@ -57,9 +57,14 @@ Tokens, credentials and knobs a running process reads. Most configuration is not
| `SYNC_HEARTBEAT_SECONDS` | `settings.syncScheduler.heartbeatSeconds` | Overrides the editor's in-process sync heartbeat. `0` = no internal timer (tick from cron instead). | editor/app/scheduler/heartbeat.ts |
| `SYNC_TICK_URL` | `http://127.0.0.1:3001/api/scheduler/tick` | Where `archilyzer sync tick` (cron's heartbeat) posts. | common/bin/sync-tick.ts |
| `SYNC_TICK_TOKEN` | unset (no auth) | Bearer token for the tick endpoint; set on both the editor and the cron job. | common/bin/sync-tick.ts, editor/app/scheduler/auth.ts |
-| `R2_ACCESS_KEY_ID` | — | R2 S3 credentials for uploading oversize archives at deploy time (with `R2_SECRET_ACCESS_KEY` and `CLOUDFLARE_ACCOUNT_ID`). See [PUBLISH.md](PUBLISH.md). | common/publish/build.ts |
-| `R2_SECRET_ACCESS_KEY` | — | See `R2_ACCESS_KEY_ID`. | common/publish/build.ts |
-| `CLOUDFLARE_ACCOUNT_ID` | — | The account the R2 endpoint belongs to. wrangler reads its own credentials. | common/publish/build.ts |
+| `R2_ACCESS_KEY_ID` | — | R2 S3 credentials for uploading oversize archives at deploy time (with `R2_SECRET_ACCESS_KEY` and `CLOUDFLARE_ACCOUNT_ID`), needed only when `archiveStorage.bucket` is set. In Docker they come from `.env`. See [PUBLISH.md](PUBLISH.md). | common/publish/build.ts, common/bin/doctor.ts (set or not) |
+| `R2_SECRET_ACCESS_KEY` | — | See `R2_ACCESS_KEY_ID`. | common/publish/build.ts, common/bin/doctor.ts (set or not) |
+| `CLOUDFLARE_ACCOUNT_ID` | — | The Cloudflare account: the R2 endpoint's, and the one wrangler deploys to when the token can see more than one. In Docker it comes from `.env`. | common/publish/build.ts, wrangler, common/bin/doctor.ts (set or not) |
+| `CLOUDFLARE_API_TOKEN` | unset (wrangler's own `wrangler login` config, on a host) | The API token every deploy's wrangler authenticates with (Cloudflare Pages: Edit). The way a container deploys — there is no browser for `wrangler login` in one; set it in `.env`. | wrangler (every deploy), common/bin/doctor.ts (set or not, never the value) |
+| `ARCHILYZER_SOURCE_REPO` | this checkout's git common dir | The git DIR `archilyzer source publish` mirrors `main` from, when the checkout has none: in Docker, `/data/source.git`, the host's git common dir mounted read-only by docker-compose.source.yml. A value that names nothing refuses the publish. | common/publish/source.ts, common/bin/doctor.ts, docker/entrypoint.sh |
+| `YTDLP_SOURCE_HOST_DIR` | — (required by the overlay) | Docker: the HOST path of a yt-dlp source checkout (the directory holding `yt_dlp/`), mounted read-only at `/opt/yt-dlp-src` by docker-compose.ytdlp.yml. See [RUNNING_IN_DOCKER.md](RUNNING_IN_DOCKER.md), "Substituting yt-dlp". | docker-compose.ytdlp.yml |
+| `YTDLP_AUTO_UPDATE` | off | Docker: `1` runs `yt-dlp -U` on every editor boot — on the image's yt-dlp only; an override (`YTDLP_BIN` naming another) is left alone, with a warning. | docker/entrypoint.sh, common/bin/doctor.ts |
+| `YTDLP_SOURCE_DIR` | `/opt/yt-dlp-src` | Docker: where `/usr/local/bin/yt-dlp-from-source` finds the yt-dlp source tree it runs with the image's python. | docker/yt-dlp-from-source.sh |
| `DOCKER_BIN` | `docker` | The container engine for docker-mode builds (e.g. `podman`). | common/publish/build.ts |
| `DOCKER_BUILD_MEMORY` | no cap | Per-container memory cap for a docker-mode build (`--memory`). | common/publish/build.ts |
| `DOCKER_BUILD_CPUS` | no cap | Per-container CPU cap for a docker-mode build (`--cpus`). | common/publish/build.ts |
@@ -141,7 +146,7 @@ The publish pipeline sets these for a process it spawns. Listed so a reader know
## Docker
-The container's own set, read by `docker/*.sh`, the compose files and Caddy — not by the apps' code (except `ARCHILYZER_IDLE_BOOT`). See [RUNNING_IN_DOCKER.md](RUNNING_IN_DOCKER.md).
+The container's own set, read by `docker/*.sh`, the compose files and Caddy — not by the apps' code (except `ARCHILYZER_IDLE_BOOT`, and the image's facts `archilyzer doctor` and the publish stamps read). See [RUNNING_IN_DOCKER.md](RUNNING_IN_DOCKER.md).
| Variable | Default | What it does | Read by |
|---|---|---|---|
@@ -150,6 +155,11 @@ The container's own set, read by `docker/*.sh`, the compose files and Caddy —
| `ARCHILYZER_MODELS_DIR` | `/data/models` | Where models live in the container. | docker/entrypoint.sh |
| `ARCHILYZER_BUILDS_DIR` | `/data/builds` | Where the container keeps built sites. | docker/entrypoint.sh |
| `ARCHILYZER_SITE_OUT` | `/data/builds/site` | The built export site the `site` service serves. | docker/entrypoint.sh, docker/publish-site.sh |
+| `ARCHILYZER_HOMEPAGE_OUT` | `/data/builds/homepage` | The locally deployed homepage (`publish homepage --deploy --to local`). The `homepage` service serves it when it is non-empty, else the image's baked build. | docker/entrypoint.sh |
+| `ARCHILYZER_IMAGE_YTDLP` | baked: `/usr/local/bin/yt-dlp` | The yt-dlp the image ships. `YTDLP_BIN` naming anything else is an OVERRIDE: the boot's `yt-dlp:` line and `archilyzer doctor` say so, and `YTDLP_AUTO_UPDATE` leaves it alone. | docker/entrypoint.sh, common/bin/doctor.ts |
+| `ARCHILYZER_COMMIT` | baked: empty unless the build passed it | The commit the image was built from — the publish stamps' `commit` where there is no .git. `ARCHILYZER_COMMIT=$(git rev-parse HEAD) docker compose build`. | the Dockerfile (a build arg), the publish stamps (`IMAGE_COMMIT_ENV`) |
+| `ARCHILYZER_BRANCH` | baked: empty unless the build passed it | The branch the image was built from — the stamps' `branch`, which a production deploy checks. | the Dockerfile (a build arg), the publish stamps (`IMAGE_BRANCH_ENV`) |
+| `ARCHILYZER_SOURCE_HOST_DIR` | `./.git` | The HOST's git common dir docker-compose.source.yml mounts at `/data/source.git`. In a git worktree, the primary checkout's `.git`. | docker-compose.source.yml |
| `ARCHILYZER_IDLE_BOOT` | off | `1` boots the editor without arming the heartbeat or any auto-queue runner. | common/lib/idleBoot.ts (the editor) |
| `ARCHILYZER_AUTH_MODE` | `basic` | `basic`, `forward` or `none` — the only escape hatch from the exposure guard. | docker/guard-exposure.sh, docker/caddy-start.sh |
| `ARCHILYZER_AUTH_USER` | `archilyzer` | Basic-auth user. | docker/Caddyfile |
diff --git a/common/lib/envVars.ts b/common/lib/envVars.ts
@@ -81,7 +81,7 @@ const DECLARED: EnvVarDecl[] = [
paths("ARIA2C_BIN", "`aria2c` on PATH", "Fetches an archive.org file over BitTorrent (the item's torrent, archive.org as web seed), then seeds it for a while. Optional: without it archive.org files are downloaded directly. See `archiveOrg` in [SETTINGS.md](SETTINGS.md)."),
paths("OLLAMA_URL", "`http://127.0.0.1:11434`", "The local ollama server, the local digest and attribution engine."),
paths("CLAUDE_BIN", "`claude` on PATH", "The `claude` CLI, driving the opt-in metered digest lane."),
- paths("ARCHILYZER_CONFIG_DIR", "`~/.config/archilyzer`", "The operator's private config dir, outside the repo: the two inputs of `archilyzer source publish` below. Never committed."),
+ paths("ARCHILYZER_CONFIG_DIR", "`~/.config/archilyzer`", "The operator's private config dir, outside the repo: the two inputs of `archilyzer source publish` below. Never committed. In Docker it is `/data/config/archilyzer`, in the config volume (docker-compose.yml)."),
paths("SOURCE_SCRUB_FILE", "`<ARCHILYZER_CONFIG_DIR>/source-scrub.txt`", "git-filter-repo `lhs==>rhs` rules applied to file contents AND commit messages when the source mirror is generated (`<home dir>==>/home/user` is built in and runs first). Every rule's left side is also denied. See [PUBLISH.md](PUBLISH.md)."),
paths("SOURCE_DENYLIST_FILE", "`<ARCHILYZER_CONFIG_DIR>/source-denylist.txt`", "Literals the published source must never contain, one per line (`i:` = any case). One hit anywhere in the mirror, the tree or the tarball refuses the publish."),
paths("ARCHILYZER_SOURCE_SCRATCH", "the OS temp dir", "Where `source publish` makes its scratch clone and stage (removed afterwards unless `--keep-scratch`)."),
@@ -93,9 +93,14 @@ const DECLARED: EnvVarDecl[] = [
{ name: "SYNC_HEARTBEAT_SECONDS", audience: "runtime", default: "`settings.syncScheduler.heartbeatSeconds`", readBy: "editor/app/scheduler/heartbeat.ts", doc: "Overrides the editor's in-process sync heartbeat. `0` = no internal timer (tick from cron instead)." },
{ name: "SYNC_TICK_URL", audience: "runtime", default: "`http://127.0.0.1:3001/api/scheduler/tick`", readBy: "common/bin/sync-tick.ts", doc: "Where `archilyzer sync tick` (cron's heartbeat) posts." },
{ name: "SYNC_TICK_TOKEN", audience: "runtime", default: "unset (no auth)", readBy: "common/bin/sync-tick.ts, editor/app/scheduler/auth.ts", doc: "Bearer token for the tick endpoint; set on both the editor and the cron job." },
- { name: "R2_ACCESS_KEY_ID", audience: "runtime", default: "—", readBy: "common/publish/build.ts", doc: "R2 S3 credentials for uploading oversize archives at deploy time (with `R2_SECRET_ACCESS_KEY` and `CLOUDFLARE_ACCOUNT_ID`). See [PUBLISH.md](PUBLISH.md)." },
- { name: "R2_SECRET_ACCESS_KEY", audience: "runtime", default: "—", readBy: "common/publish/build.ts", doc: "See `R2_ACCESS_KEY_ID`." },
- { name: "CLOUDFLARE_ACCOUNT_ID", audience: "runtime", default: "—", readBy: "common/publish/build.ts", doc: "The account the R2 endpoint belongs to. wrangler reads its own credentials." },
+ { name: "R2_ACCESS_KEY_ID", audience: "runtime", default: "—", readBy: "common/publish/build.ts, common/bin/doctor.ts (set or not)", doc: "R2 S3 credentials for uploading oversize archives at deploy time (with `R2_SECRET_ACCESS_KEY` and `CLOUDFLARE_ACCOUNT_ID`), needed only when `archiveStorage.bucket` is set. In Docker they come from `.env`. See [PUBLISH.md](PUBLISH.md)." },
+ { name: "R2_SECRET_ACCESS_KEY", audience: "runtime", default: "—", readBy: "common/publish/build.ts, common/bin/doctor.ts (set or not)", doc: "See `R2_ACCESS_KEY_ID`." },
+ { name: "CLOUDFLARE_ACCOUNT_ID", audience: "runtime", default: "—", readBy: "common/publish/build.ts, wrangler, common/bin/doctor.ts (set or not)", doc: "The Cloudflare account: the R2 endpoint's, and the one wrangler deploys to when the token can see more than one. In Docker it comes from `.env`." },
+ { name: "CLOUDFLARE_API_TOKEN", audience: "runtime", default: "unset (wrangler's own `wrangler login` config, on a host)", readBy: "wrangler (every deploy), common/bin/doctor.ts (set or not, never the value)", doc: "The API token every deploy's wrangler authenticates with (Cloudflare Pages: Edit). The way a container deploys — there is no browser for `wrangler login` in one; set it in `.env`." },
+ { name: "ARCHILYZER_SOURCE_REPO", audience: "runtime", default: "this checkout's git common dir", readBy: "common/publish/source.ts, common/bin/doctor.ts, docker/entrypoint.sh", doc: "The git DIR `archilyzer source publish` mirrors `main` from, when the checkout has none: in Docker, `/data/source.git`, the host's git common dir mounted read-only by docker-compose.source.yml. A value that names nothing refuses the publish." },
+ { name: "YTDLP_SOURCE_HOST_DIR", audience: "runtime", default: "— (required by the overlay)", readBy: "docker-compose.ytdlp.yml", doc: "Docker: the HOST path of a yt-dlp source checkout (the directory holding `yt_dlp/`), mounted read-only at `/opt/yt-dlp-src` by docker-compose.ytdlp.yml. See [RUNNING_IN_DOCKER.md](RUNNING_IN_DOCKER.md), \"Substituting yt-dlp\"." },
+ { name: "YTDLP_AUTO_UPDATE", audience: "runtime", default: "off", readBy: "docker/entrypoint.sh, common/bin/doctor.ts", doc: "Docker: `1` runs `yt-dlp -U` on every editor boot — on the image's yt-dlp only; an override (`YTDLP_BIN` naming another) is left alone, with a warning." },
+ { name: "YTDLP_SOURCE_DIR", audience: "runtime", default: "`/opt/yt-dlp-src`", readBy: "docker/yt-dlp-from-source.sh", doc: "Docker: where `/usr/local/bin/yt-dlp-from-source` finds the yt-dlp source tree it runs with the image's python." },
{ name: "DOCKER_BIN", audience: "runtime", default: "`docker`", readBy: "common/publish/build.ts", doc: "The container engine for docker-mode builds (e.g. `podman`)." },
{ name: "DOCKER_BUILD_MEMORY", audience: "runtime", default: "no cap", readBy: "common/publish/build.ts", doc: "Per-container memory cap for a docker-mode build (`--memory`)." },
{ name: "DOCKER_BUILD_CPUS", audience: "runtime", default: "no cap", readBy: "common/publish/build.ts", doc: "Per-container CPU cap for a docker-mode build (`--cpus`)." },
@@ -153,6 +158,11 @@ const DECLARED: EnvVarDecl[] = [
{ name: "ARCHILYZER_MODELS_DIR", audience: "docker", default: "`/data/models`", readBy: "docker/entrypoint.sh", doc: "Where models live in the container." },
{ name: "ARCHILYZER_BUILDS_DIR", audience: "docker", default: "`/data/builds`", readBy: "docker/entrypoint.sh", doc: "Where the container keeps built sites." },
{ name: "ARCHILYZER_SITE_OUT", audience: "docker", default: "`/data/builds/site`", readBy: "docker/entrypoint.sh, docker/publish-site.sh", doc: "The built export site the `site` service serves." },
+ { name: "ARCHILYZER_HOMEPAGE_OUT", audience: "docker", default: "`/data/builds/homepage`", readBy: "docker/entrypoint.sh", doc: "The locally deployed homepage (`publish homepage --deploy --to local`). The `homepage` service serves it when it is non-empty, else the image's baked build." },
+ { name: "ARCHILYZER_IMAGE_YTDLP", audience: "docker", default: "baked: `/usr/local/bin/yt-dlp`", readBy: "docker/entrypoint.sh, common/bin/doctor.ts", doc: "The yt-dlp the image ships. `YTDLP_BIN` naming anything else is an OVERRIDE: the boot's `yt-dlp:` line and `archilyzer doctor` say so, and `YTDLP_AUTO_UPDATE` leaves it alone." },
+ { name: "ARCHILYZER_COMMIT", audience: "docker", default: "baked: empty unless the build passed it", readBy: "the Dockerfile (a build arg), the publish stamps (`IMAGE_COMMIT_ENV`)", doc: "The commit the image was built from — the publish stamps' `commit` where there is no .git. `ARCHILYZER_COMMIT=$(git rev-parse HEAD) docker compose build`." },
+ { name: "ARCHILYZER_BRANCH", audience: "docker", default: "baked: empty unless the build passed it", readBy: "the Dockerfile (a build arg), the publish stamps (`IMAGE_BRANCH_ENV`)", doc: "The branch the image was built from — the stamps' `branch`, which a production deploy checks." },
+ { name: "ARCHILYZER_SOURCE_HOST_DIR", audience: "docker", default: "`./.git`", readBy: "docker-compose.source.yml", doc: "The HOST's git common dir docker-compose.source.yml mounts at `/data/source.git`. In a git worktree, the primary checkout's `.git`." },
{ name: "ARCHILYZER_IDLE_BOOT", audience: "docker", default: "off", readBy: "common/lib/idleBoot.ts (the editor)", doc: "`1` boots the editor without arming the heartbeat or any auto-queue runner." },
{ name: "ARCHILYZER_AUTH_MODE", audience: "docker", default: "`basic`", readBy: "docker/guard-exposure.sh, docker/caddy-start.sh", doc: "`basic`, `forward` or `none` — the only escape hatch from the exposure guard." },
{ name: "ARCHILYZER_AUTH_USER", audience: "docker", default: "`archilyzer`", readBy: "docker/Caddyfile", doc: "Basic-auth user." },
@@ -216,10 +226,21 @@ export const ENV_AUDIENCES: ReadonlyArray<{ id: EnvAudience; title: string; intr
{ id: "runtime", title: "Runtime", intro: "Tokens, credentials and knobs a running process reads. Most configuration is not here but in `settings.json` ([SETTINGS.md](SETTINGS.md))." },
{ id: "port", title: "Ports", intro: "Every local server's default port, from `common/lib/ports.mjs`. The primary checkout uses these; worktree N adds N × 100 (`pnpm wt list`)." },
{ id: "internal", title: "Set by the pipeline", intro: "The publish pipeline sets these for a process it spawns. Listed so a reader knows what they are; nobody sets them by hand." },
- { id: "docker", title: "Docker", intro: "The container's own set, read by `docker/*.sh`, the compose files and Caddy — not by the apps' code (except `ARCHILYZER_IDLE_BOOT`). See [RUNNING_IN_DOCKER.md](RUNNING_IN_DOCKER.md)." },
+ { id: "docker", title: "Docker", intro: "The container's own set, read by `docker/*.sh`, the compose files and Caddy — not by the apps' code (except `ARCHILYZER_IDLE_BOOT`, and the image's facts `archilyzer doctor` and the publish stamps read). See [RUNNING_IN_DOCKER.md](RUNNING_IN_DOCKER.md)." },
{ id: "test", title: "Tests only", intro: "Read only by a test harness, a fake binary or a test-mode branch. Never set one on a real instance." },
];
+// The two build facts the runtime image bakes (Dockerfile build args → ENV), by
+// name: the publish stamps' `commit`/`branch` where there is no .git to ask.
+export const IMAGE_COMMIT_ENV = "ARCHILYZER_COMMIT";
+export const IMAGE_BRANCH_ENV = "ARCHILYZER_BRANCH";
+
+// Those facts, or null each — an image built without the args bakes them empty.
+export function imageBuildFacts(env: NodeJS.ProcessEnv = process.env): { commit: string | null; branch: string | null } {
+ const v = (k: string) => env[k]?.trim() || null;
+ return { commit: v(IMAGE_COMMIT_ENV), branch: v(IMAGE_BRANCH_ENV) };
+}
+
export function envVar(name: string): EnvVarDecl | undefined {
return ENV_VARS.find((v) => v.name === name);
}
diff --git a/common/publish/buildImage.test.ts b/common/publish/buildImage.test.ts
@@ -1,7 +1,9 @@
// The container build's contract with the repo: what Dockerfile.build's
// image bakes into export/public and how docker/build-site.sh puts it in front
-// of each site's `next build`. No docker here — the invariant is read from git,
-// and the shell function is read out of build-site.sh and run with bash.
+// of each site's `next build`; and what the runtime image (the root Dockerfile)
+// must agree with the code about. No docker here — the invariant is read from
+// git or the Dockerfile's text, and the shell function is read out of
+// build-site.sh and run with bash.
//
// Run with:
// pnpm --filter yt-dlp-transcript-common test
@@ -13,6 +15,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
+import { FILTER_REPO_PIPX_SPEC } from "./source";
const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
@@ -88,3 +91,38 @@ test("build-site.sh's asset sync ships a changed svg, drops a removed one, and t
rmSync(root, { recursive: true, force: true });
}
});
+
+// The runtime image (the root Dockerfile) installs git-filter-repo with pipx so
+// a container can publish the homepage's /source mirror without a network fetch.
+// The version it bakes must be the one `source publish` would fetch with
+// `pipx run --spec` on a host (FILTER_REPO_PIPX_SPEC): the filter-repo version
+// is part of the publish's skip key, so a drift between the two is a rebuild
+// nobody asked for, or a mirror rewritten by a version nobody tested.
+test("the runtime image installs the git-filter-repo that source publish names, in every target", () => {
+ const dockerfile = readFileSync(path.join(REPO, "Dockerfile"), "utf8");
+ const installs = [...dockerfile.matchAll(/pipx install (\S+)/g)].map((m) => m[1]);
+ // runtime-base (runtime + runtime-vulkan) and runtime-cuda, which repeats it.
+ assert.equal(installs.length, 2, `expected two \`pipx install\` lines, found ${installs.length}`);
+ for (const spec of installs) {
+ assert.equal(
+ spec,
+ FILTER_REPO_PIPX_SPEC,
+ `the Dockerfile installs ${spec}, source.ts names ${FILTER_REPO_PIPX_SPEC} — move both together`,
+ );
+ }
+});
+
+// The yt-dlp substitution hook: the image ships its own yt-dlp as
+// ARCHILYZER_IMAGE_YTDLP beside YTDLP_BIN (the entrypoint and the doctor tell an
+// override by the difference), and links the from-source wrapper — in every
+// target.
+test("every runtime target names its own yt-dlp and links the from-source wrapper", () => {
+ const dockerfile = readFileSync(path.join(REPO, "Dockerfile"), "utf8");
+ const image = [...dockerfile.matchAll(/^\s+ARCHILYZER_IMAGE_YTDLP=(\S+) \\$/gm)].map((m) => m[1]);
+ const bin = [...dockerfile.matchAll(/^\s+YTDLP_BIN=(\S+) \\$/gm)].map((m) => m[1]);
+ assert.equal(image.length, 2);
+ assert.deepEqual(image, bin, "ARCHILYZER_IMAGE_YTDLP and YTDLP_BIN start equal in each target");
+ const links = dockerfile.match(/ln -s \/repo\/docker\/yt-dlp-from-source\.sh \/usr\/local\/bin\/yt-dlp-from-source/g) ?? [];
+ assert.equal(links.length, 2);
+ assert.ok(existsSync(path.join(REPO, "docker", "yt-dlp-from-source.sh")));
+});
diff --git a/docker-compose.source.yml b/docker-compose.source.yml
@@ -0,0 +1,27 @@
+# Overlay: let the container publish the homepage's /source mirror.
+#
+# docker compose -f docker-compose.yml -f docker-compose.source.yml up -d
+#
+# `archilyzer source publish` (run by the homepage build) mirrors this repo's
+# `main` — but the image has no .git (.dockerignore keeps it out, and it would be
+# stale the day after the build anyway). This mounts the HOST's git common dir,
+# read-only, at /data/source.git and points ARCHILYZER_SOURCE_REPO at it;
+# common/publish/source.ts reads that before it looks for a checkout. Nothing in
+# the container writes to it: the publish clones it into scratch first.
+#
+# ARCHILYZER_SOURCE_HOST_DIR is the host path, default `./.git` — right for a
+# plain clone. In a git WORKTREE `.git` is a file; use the primary checkout's
+# `.git` (`git rev-parse --git-common-dir` prints it).
+#
+# The entrypoint marks /data/source.git as a git safe.directory (the files
+# belong to your host user, the container runs as root). The scrub rules and
+# the denylist the publish needs live in the config volume
+# (ARCHILYZER_CONFIG_DIR, /data/config/archilyzer): `docker compose cp` them in.
+# RUNNING_IN_DOCKER.md, "Publishing".
+
+services:
+ editor:
+ environment:
+ ARCHILYZER_SOURCE_REPO: /data/source.git
+ volumes:
+ - ${ARCHILYZER_SOURCE_HOST_DIR:-./.git}:/data/source.git:ro
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -31,6 +31,13 @@ x-app: &app
build:
context: .
target: runtime
+ # Which commit the image is built from, for the publish stamps (the image
+ # has no .git). Empty unless the shell sets them:
+ # ARCHILYZER_COMMIT=$(git rev-parse HEAD) \
+ # ARCHILYZER_BRANCH=$(git branch --show-current) docker compose build
+ args:
+ ARCHILYZER_COMMIT: ${ARCHILYZER_COMMIT:-}
+ ARCHILYZER_BRANCH: ${ARCHILYZER_BRANCH:-}
image: archilyzer:${ARCHILYZER_TAG:-local}
restart: unless-stopped
networks: [archilyzer]
@@ -38,6 +45,11 @@ x-app: &app
# cosmetic: a bcrypt hash is full of `$`, and compose interpolates `${...}` in
# the YAML but passes env_file values through verbatim. Put the hash in .env
# and it arrives intact, with nothing to escape.
+ #
+ # The same goes for the publish credentials: CLOUDFLARE_API_TOKEN,
+ # CLOUDFLARE_ACCOUNT_ID and the R2_* keys are read from .env by the editor,
+ # and every publish stage it runs (a child process) inherits them. They are
+ # deliberately NOT in x-app-env below — a value there would override .env.
env_file:
- path: .env
required: false
@@ -59,6 +71,13 @@ x-app-env: &app-env
EXPORT_INDEX_DIR: /data/builds/.export-index
EXPORT_BUILDS_DIR: /data/builds/.export-builds
ARCHILYZER_SITE_OUT: /data/builds/site
+ # `publish homepage --deploy --to local` writes here; the homepage service
+ # serves it once it is non-empty (else the image's baked build).
+ ARCHILYZER_HOMEPAGE_OUT: /data/builds/homepage
+ # The operator's private config dir (common/lib/paths.ts): the source
+ # mirror's scrub rules and denylist live here, in the config volume —
+ # `docker compose cp` them in; they are never printed.
+ ARCHILYZER_CONFIG_DIR: /data/config/archilyzer
# Set explicitly so a stray EDITOR_PORT/UMTOOL_PORT in .env cannot move an app
# off the port Caddy proxies to. These are INTERNAL ports; the published ones
# are on the caddy service below.
@@ -133,7 +152,10 @@ services:
- builds:/data/builds
# -------------------------------------------------------------------------
- # The project's own site (marketing + docs). Baked into the image.
+ # The project's own site (marketing + docs). A build is baked into the
+ # image; a local deploy (`publish homepage --deploy --to local`) into the
+ # builds volume replaces it once there is one — restart this service after
+ # the first.
# -------------------------------------------------------------------------
homepage:
<<: *app
@@ -141,6 +163,10 @@ services:
command: ["homepage"]
environment:
<<: *app-env
+ volumes:
+ # Writable for the same reason as the site's: the entrypoint makes the
+ # (empty) directory it looks in.
+ - builds:/data/builds
# -------------------------------------------------------------------------
# umtool: the clip/report bench. Private, like the editor.
@@ -167,12 +193,15 @@ volumes:
# The corpus. This is the one that matters: real media, real transcripts,
# hundreds of GB when it grows up. Back it up.
corpus:
- # settings.json, and anything else operational that must outlive the container.
+ # settings.json, the operator's private config dir (archilyzer/: the source
+ # mirror's rules), and anything else operational that must outlive the
+ # container.
config:
# whisper .bin models — 142 MB to 3 GB, fetched once.
models:
- # Export build staging + the published static site. Reproducible; losing it
- # costs a rebuild, not data.
+ # Export build staging, the shared index, every site's bundle and stamps, and
+ # the locally deployed site and homepage. Reproducible; losing it costs a
+ # rebuild, not data.
builds:
caddy-data:
caddy-config:
diff --git a/docker-compose.ytdlp.yml b/docker-compose.ytdlp.yml
@@ -0,0 +1,25 @@
+# Overlay: run YOUR yt-dlp — a source checkout on the host — instead of the
+# image's release binary. No rebuild: the swap happens at run time.
+#
+# # .env
+# YTDLP_SOURCE_HOST_DIR=/home/you/yt-dlp-patched # holds the yt_dlp/ package
+#
+# docker compose -f docker-compose.yml -f docker-compose.ytdlp.yml up -d
+#
+# The checkout is mounted read-only at /opt/yt-dlp-src and YTDLP_BIN points at
+# /usr/local/bin/yt-dlp-from-source, a wrapper baked into the image
+# (docker/yt-dlp-from-source.sh) that runs `python3 -m yt_dlp` from it with the
+# image's python and yt-dlp's optional modules.
+#
+# Every editor boot prints `yt-dlp: <path> <version> (override)`, and
+# YTDLP_AUTO_UPDATE leaves an override alone — update the checkout instead.
+# A zipapp built on the host (`make yt-dlp` in the checkout) works too, with no
+# overlay: copy it into the config volume and set YTDLP_BIN to its path there.
+# RUNNING_IN_DOCKER.md, "Substituting yt-dlp".
+
+services:
+ editor:
+ environment:
+ YTDLP_BIN: /usr/local/bin/yt-dlp-from-source
+ volumes:
+ - ${YTDLP_SOURCE_HOST_DIR:?set YTDLP_SOURCE_HOST_DIR in .env to your yt-dlp checkout}:/opt/yt-dlp-src:ro
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
@@ -4,7 +4,8 @@
#
# editor the admin app (next start, :3001)
# site the published archive (serve, :3000)
-# homepage the project's own site (serve, :3031)
+# homepage the project's own site (serve, :3031) — the local deploy in the
+# builds volume when there is one, else the image's baked build
# umtool the clip/report bench (next start, :3050)
# shell drop into bash — for `docker compose run --rm editor shell`
#
@@ -22,6 +23,7 @@ SETTINGS_FILE="${SETTINGS_FILE:-/data/config/settings.json}"
MODELS_DIR="${ARCHILYZER_MODELS_DIR:-/data/models}"
BUILDS_DIR="${ARCHILYZER_BUILDS_DIR:-/data/builds}"
SITE_OUT="${ARCHILYZER_SITE_OUT:-${BUILDS_DIR}/site}"
+HOMEPAGE_OUT="${ARCHILYZER_HOMEPAGE_OUT:-${BUILDS_DIR}/homepage}"
WHISPER_MODEL="${WHISPER_MODEL:-${MODELS_DIR}/ggml-base.en.bin}"
# Which transcription backend THIS IMAGE was built with. Set by the Dockerfile
# per target, not by the operator: `runtime`/`runtime-cuda` carry whisper.cpp,
@@ -42,7 +44,19 @@ mkdir -p \
"$(dirname "${SETTINGS_FILE}")" \
"${MODELS_DIR}" \
"${BUILDS_DIR}" \
- "${SITE_OUT}"
+ "${SITE_OUT}" \
+ "${HOMEPAGE_OUT}"
+
+# The host's git common dir, mounted read-only by docker-compose.source.yml for
+# the homepage's source mirror (`archilyzer source publish`). It belongs to the
+# host's user and the container runs as root, so git would refuse it as
+# "dubious ownership" without this. Added once — `--add` on every boot would
+# pile up duplicates in a container that is restarted rather than recreated.
+SOURCE_REPO_DIR="${ARCHILYZER_SOURCE_REPO:-/data/source.git}"
+if ! git config --global --get-all safe.directory 2>/dev/null | grep -qxF "${SOURCE_REPO_DIR}"; then
+ git config --global --add safe.directory "${SOURCE_REPO_DIR}" ||
+ printf '[entrypoint] WARNING: could not mark %s safe for git\n' "${SOURCE_REPO_DIR}"
+fi
# ---------------------------------------------------------------------------
# 2. Seed settings.json — with a worker.
@@ -163,20 +177,67 @@ fetch_model() {
}
# ---------------------------------------------------------------------------
-# 4. yt-dlp self-update.
+# 4. yt-dlp: which one, and its self-update.
+#
+# YTDLP_BIN is the yt-dlp every fetch runs; ARCHILYZER_IMAGE_YTDLP is the one
+# this image ships. They differ when the operator substituted their own (a
+# zipapp, or /usr/local/bin/yt-dlp-from-source over a mounted source tree —
+# RUNNING_IN_DOCKER.md, "Substituting yt-dlp"): an OVERRIDE.
#
# An image pins yt-dlp at build time, and a stale yt-dlp is the most common
# reason downloads suddenly start failing — YouTube changes, yt-dlp ships a fix
-# within days, and an image from last month has none of them. Opt-in because it
-# is a network call on every boot.
+# within days, and an image from last month has none of them. The self-update is
+# opt-in because it is a network call on every boot, and it never touches an
+# override: a patched build is updated where it is built, and `-U` on one either
+# fails or replaces the patch with a release.
# ---------------------------------------------------------------------------
+YTDLP="${YTDLP_BIN:-yt-dlp}"
+IMAGE_YTDLP="${ARCHILYZER_IMAGE_YTDLP:-}"
+
+# "image" or "override", by where each one actually lands.
+ytdlp_origin() {
+ local mine theirs
+ if [ -z "${IMAGE_YTDLP}" ]; then
+ echo image
+ return 0
+ fi
+ mine="$(readlink -f "$(command -v "${YTDLP}" 2>/dev/null || echo "${YTDLP}")" 2>/dev/null || echo "${YTDLP}")"
+ theirs="$(readlink -f "${IMAGE_YTDLP}" 2>/dev/null || echo "${IMAGE_YTDLP}")"
+ if [ "${mine}" = "${theirs}" ]; then echo image; else echo override; fi
+}
+
update_ytdlp() {
case "${YTDLP_AUTO_UPDATE:-0}" in
1 | true | yes | on) ;;
*) return 0 ;;
esac
+ if [ "$(ytdlp_origin)" = "override" ]; then
+ log "WARNING: YTDLP_AUTO_UPDATE is set, but YTDLP_BIN (${YTDLP}) is not the image's yt-dlp"
+ log " (${IMAGE_YTDLP}) — an override is not self-updated; update it where it is built"
+ return 0
+ fi
log "yt-dlp -U (YTDLP_AUTO_UPDATE is set)"
- "${YTDLP_BIN:-yt-dlp}" -U || log "WARNING: yt-dlp self-update failed — continuing with $("${YTDLP_BIN:-yt-dlp}" --version 2>/dev/null || echo unknown)"
+ "${YTDLP}" -U || log "WARNING: yt-dlp self-update failed — continuing with $("${YTDLP}" --version 2>/dev/null || echo unknown)"
+}
+
+# One line on every boot, like the vulkan: line — "which yt-dlp is this, and
+# does it run?" answered before the first download has to. A yt-dlp that is
+# missing or does not run is a WARNING, never a refusal: the editor is still
+# useful (reading, building, publishing) without one.
+ytdlp_line() {
+ local origin version err
+ origin="$(ytdlp_origin)"
+ if ! command -v "${YTDLP}" >/dev/null 2>&1; then
+ log "yt-dlp: MISSING ${YTDLP} (${origin}) — every download will fail until it exists"
+ return 0
+ fi
+ if version="$(timeout 30 "${YTDLP}" --version 2>/dev/null | head -n 1)" && [ -n "${version}" ]; then
+ log "yt-dlp: ${YTDLP} ${version} (${origin})"
+ else
+ err="$(timeout 30 "${YTDLP}" --version 2>&1 >/dev/null | head -n 1 || true)"
+ log "yt-dlp: MISSING — ${YTDLP} (${origin}) does not run: ${err:-no version printed}"
+ log " every download will fail until it does; see RUNNING_IN_DOCKER.md"
+ fi
}
# ---------------------------------------------------------------------------
@@ -203,6 +264,7 @@ case "${APP}" in
editor)
fetch_model
update_ytdlp
+ ytdlp_line
log "corpus: ${TRANSCRIPTS_DIR}"
log "settings: ${SETTINGS_FILE}"
if [ "${TRANSCRIBER}" = "parakeet" ]; then
@@ -271,9 +333,21 @@ HTML
;;
homepage)
- # Corpus-independent, so this one IS baked into the image at build time.
- log "serving /repo/homepage/out on :3031"
- exec "$(serve_bin /repo/homepage)" /repo/homepage/out -l 3031 \
+ # Corpus-independent, so a build IS baked into the image — but that build
+ # has no /source mirror and none of the corpus's numbers (there is no corpus
+ # and no .git at image-build time). `archilyzer publish homepage --deploy
+ # --to local` writes a real one into the builds volume; once that directory
+ # is non-empty it is what is served. Chosen at boot: restart this service
+ # after the FIRST local deploy (later ones are served as they land — serve
+ # reads from disk per request).
+ HOMEPAGE_DIR=/repo/homepage/out
+ if [ -n "$(ls -A "${HOMEPAGE_OUT}" 2>/dev/null)" ]; then
+ HOMEPAGE_DIR="${HOMEPAGE_OUT}"
+ log "serving the locally deployed homepage ${HOMEPAGE_DIR} on :3031"
+ else
+ log "nothing deployed at ${HOMEPAGE_OUT} — serving the image's baked homepage ${HOMEPAGE_DIR} on :3031"
+ fi
+ exec "$(serve_bin /repo/homepage)" "${HOMEPAGE_DIR}" -l 3031 \
-c /repo/homepage/serve.json --no-clipboard --no-port-switching
;;
diff --git a/docker/publish-site.sh b/docker/publish-site.sh
@@ -4,15 +4,19 @@
#
# docker compose exec editor /repo/docker/publish-site.sh <site-id>
#
-# Why this exists rather than a baked build: the export site is a static render
-# OF A CORPUS, and there is no corpus at image-build time — so the image ships
-# the code and this publishes the output. It is the same `archilyzer build site`
-# a host install runs; the only container-specific part is the copy at the end.
+# A wrapper over the three publish stages, nothing more (release 18):
#
-# The copy is a copy and not a symlink on purpose. `next build` REMOVES and
-# recreates export/out (docker/build-site.sh has the same note), so a symlink
-# there survives exactly one build, and a bind mount there breaks the build
-# outright — the rm fails on a busy mount point.
+# archilyzer publish index the shared index, stats, templates
+# archilyzer publish build <id> the site's bundle in the builds volume
+# archilyzer publish deploy <id> --to local copied into ARCHILYZER_SITE_OUT
+#
+# Each stage is a no-op when it is already fresh, so running this twice costs
+# seconds; each takes the publish lock, so it waits for (never races) a stage
+# the editor is running. Run them one at a time for more control — the same
+# commands, through `docker compose exec editor pnpm archilyzer publish …`.
+#
+# Why the export site is built at run time rather than baked: it is a static
+# render OF A CORPUS, and there is no corpus at image-build time.
set -euo pipefail
SITE_ID="${1:-${SITE_ID:-}}"
@@ -28,39 +32,23 @@ if [ -z "${SITE_ID}" ]; then
exit 2
fi
-export SITE_ID
-
# A PRIVATE site (site.json `audience: "private"`) is never deployed, and the
-# volume this script fills is what the `site` service serves: refused before
-# the build, and again over the built corpus.json below (lib/builtExport.ts).
-# Building it without publishing is `archilyzer build site <id>`.
+# volume this fills is what the `site` service serves. The deploy stage refuses
+# it too (over the built corpus.json); refusing here first saves the build.
SITE_JSON="${SITES_DIR:-${TRANSCRIPTS_DIR:-/data/transcripts}/sites}/${SITE_ID}/site.json"
-refuse_private() {
- echo "[publish-site] REFUSED — site '${SITE_ID}' is private (audience: private): it is built for reading on this machine and is never deployed. Nothing was published to ${SITE_OUT}. Build it without publishing: pnpm --filter yt-dlp-transcript-common exec tsx bin/archilyzer.ts build site ${SITE_ID}" >&2
- exit 1
-}
if grep -Eq '"audience"[[:space:]]*:[[:space:]]*"private"' "${SITE_JSON}" 2>/dev/null; then
- refuse_private
+ echo "[publish-site] REFUSED — site '${SITE_ID}' is private (audience: private): it is built for reading on this machine and is never deployed. Nothing was published to ${SITE_OUT}. Build it without publishing: pnpm archilyzer publish build ${SITE_ID}" >&2
+ exit 1
fi
cd /repo
-echo "[publish-site] building '${SITE_ID}'"
-# The full build: the data phase (shared LMDB index + stats + chart
-# templates), then compose:site for THIS site, then next build.
-pnpm --filter yt-dlp-transcript-common exec tsx bin/archilyzer.ts build site "${SITE_ID}"
-
-[ -d /repo/export/out ] || { echo "[publish-site] no export/out after build" >&2; exit 1; }
-if grep -Eq '"audience"[[:space:]]*:[[:space:]]*"private"' /repo/export/out/corpus.json 2>/dev/null; then
- refuse_private
-fi
+archilyzer() {
+ pnpm --filter yt-dlp-transcript-common exec tsx bin/archilyzer.ts "$@"
+}
-echo "[publish-site] publishing -> ${SITE_OUT}"
-mkdir -p "${SITE_OUT}"
-# Empty the directory's CONTENTS, never the directory: it is a volume mount
-# point, and removing it fails.
-find "${SITE_OUT}" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
-cp -a /repo/export/out/. "${SITE_OUT}/"
+archilyzer publish index
+archilyzer publish build "${SITE_ID}"
+archilyzer publish deploy "${SITE_ID}" --to local
-echo "[publish-site] done — $(find "${SITE_OUT}" -type f | wc -l) files"
-echo "[publish-site] the site service serves it immediately; no restart needed."
+echo "[publish-site] the site service serves ${SITE_OUT} immediately; no restart needed."
diff --git a/docker/yt-dlp-from-source.sh b/docker/yt-dlp-from-source.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+# yt-dlp run from a SOURCE tree — the image's hook for substituting your own
+# yt-dlp (a patched checkout) without rebuilding the image.
+#
+# YTDLP_BIN=/usr/local/bin/yt-dlp-from-source (.env)
+# docker compose -f docker-compose.yml -f docker-compose.ytdlp.yml up -d
+#
+# docker-compose.ytdlp.yml mounts YTDLP_SOURCE_HOST_DIR (the checkout: the
+# directory that holds the `yt_dlp/` package) read-only at /opt/yt-dlp-src.
+# YTDLP_SOURCE_DIR moves that mount point. The python is the image's, with
+# yt-dlp's optional modules installed beside it (the Dockerfile's apt line).
+#
+# Linked into /usr/local/bin by the Dockerfile; RUNNING_IN_DOCKER.md,
+# "Substituting yt-dlp".
+src="${YTDLP_SOURCE_DIR:-/opt/yt-dlp-src}"
+if [ ! -f "${src}/yt_dlp/__init__.py" ]; then
+ echo "yt-dlp-from-source: no yt_dlp package in ${src} — mount a yt-dlp checkout there (docker-compose.ytdlp.yml, YTDLP_SOURCE_HOST_DIR)" >&2
+ exit 127
+fi
+PYTHONPATH="${src}${PYTHONPATH:+:${PYTHONPATH}}" exec python3 -m yt_dlp "$@"
