commit d9ce2e6c664ef703fa554d00930eff1a25d773c4
parent a835e96885ddbf6a9817328d914c056f0c5641bd
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 9 Oct 2026 14:06:19 -0400
common: each prepare-playable remux takes the heavy slot (queue-lock --heavy, per video); COMMANDS.md regenerated with the media attach, media playable, seed and tracker rows and the two ops actions
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
4 files changed, 40 insertions(+), 3 deletions(-)
diff --git a/COMMANDS.md b/COMMANDS.md
@@ -45,6 +45,10 @@ Run from the repo root (in the container: `docker compose exec editor pnpm archi
| `archilyzer archive-org refresh` | `<slug> [--dry-run]` | bring a channel's archive.org file records up to their provenance: a name-only mirror's title (where the item gives the file none) and upload date from its file name; offline, through the metadata history; prints old → new |
| `archilyzer wayback refresh` | `<slug> [--titles <json>] [--dry-run]` | bring a channel's Wayback Machine copies up to the Wayback rules: wayback.json (original URL, capture time), the dir renamed to its canonical id through the snapshot's reconcile (roster moved with it), and with --titles (a file of id → {title, upload\_date}) a raw file's title and date; offline, skips a record a live job holds; prints old → new |
| `archilyzer feeds backfill-metadata` | `<slug> [--feed <url>] [--dry-run]` | complete a podcast channel's records (title, date, description, duration) from its RSS feed: one fetch of the feed (default: the channel's url), no media; --dry-run counts matched / unmatched / already complete and writes nothing |
+| `archilyzer media attach` | `<slug> <source> [--items <json\|@file>] [--match <regex>] [--create-records] [--replace] [--dry-run]` | give a channel's held videos their media from a LOCAL archive (a directory, a .zip read in place, a .7z) into the saved-video store, provenance recorded (archive, entry, sha256); the id from the folder's "(<id>)" or the file's yt-dlp suffix; --create-records writes a record for a video not held; --dry-run lists each folder's class and writes nothing; offline, nothing fetched |
+| `archilyzer media playable` | `<slug> [--ids <a,b>] [--trackers <url,url>] [--root <dir>] [--dry-run]` | remux a channel's saved containers losslessly (-c copy) into browser-playable mp4 (+faststart) or webm, one single-file torrent each (no web seed; --trackers is the announce list, outside the infohash), under playable/ beside the saved-video store; keyed by the source's sha256, so a re-run is a no-op; offline |
+| `archilyzer seed` | | the home seeder of last resort: seed the playable torrents of settings.seeder.sites, each only while no other seeder has it (standby otherwise, with hysteresis), TCP + WebRTC, no DHT; runs until interrupted — behind the VPN (docker compose profile `seeder`, docker-compose.seeder.yml) |
+| `archilyzer tracker` | `[--host 127.0.0.1] [--port 8000]` | the self-hosted tracker (HTTP + WebSocket, no UDP) for the seeder's torrents only — any other infohash is refused; runs until interrupted |
| `archilyzer duplicates` | `[--threshold N] [--all-durations] [--blocking title\|duration\|both] [--near F] [--tolerance N] …` | on-demand duplicate detection (after index + stats) *(passthrough)* |
| `archilyzer posts fetch` | `--slug <channel> [--full \| --older [--floor YYYY-MM-DD] [--from YYYY-MM-DD] [--force]] [--limit N] [--pages N]` | fetch a social channel's posts into its posts corpus (--older: walk back below the oldest archived post; --pages: a forum thread's latest N pages) *(passthrough)* |
| `archilyzer posts import-html` | `<slug> <file-or-dir>… [--dry-run]` | import forum thread pages saved from a browser ("Save page as", .html) into a forum-thread channel: new posts appended, edited ones updated, nothing fetched |
@@ -82,6 +86,8 @@ Drives a running editor over HTTP (`/api/ops/*`, the same actions its pages run)
| `refresh-metadata` | refresh-metadata re-reads ONE video's metadata.info.json from its source (no subtitles, no media) on the platform's queue: {"slug", "id"}. The job's log ends with what the source now says — live\_status, formats, audio-only formats and whether any is non-fragmented, English captions, the keys that changed. An id with no data/<id>/ is refused (a refresh re-reads a video already archived), as are archive.org and Wayback records. | `pnpm ops refresh-metadata --json '{"slug":"the-quartering","id":"<videoId>"}' --wait` |
| `import-video` | — | `pnpm ops import-video --json '{"slug":"demo-archive","url":"https://archive.org/details/example-item"}'` |
| `import-archive-org` | — | `pnpm ops import-archive-org --json '{"slug":"demo-archive","item":"example-item","match":"\\.mp4$"}' --wait` |
+| `attach-media` | attach-media copies each held video's file out of a LOCAL archive into the saved-video store, as its source container (nothing fetched): {"slug", "source"} — an absolute path to a directory, a .zip (read in place) or a .7z. The id is the folder's trailing "(<id>)", else the file's yt-dlp suffix; "items": \[{"id", "path"}\] names exact files (path inside the source), "match" narrows the folders by regex. "createRecords": true writes a record for a video the channel does not hold; "replace": true re-attaches over a saved container. "dryRun": true logs each folder's class (attach, not-held, already-attached, lost, no-media, unmatched, ambiguous) and the held videos with no media in it, and writes nothing. The log ends with a summary: line; a re-run resumes. | |
+| `prepare-playable` | prepare-playable remuxes each of a channel's saved containers, losslessly (-c copy), into a browser-playable mp4 (+faststart) or webm, and makes one single-file torrent per copy, under playable/ beside the saved-video store: {"slug"}. "ids" narrows it; "trackers": \[...\] is each .torrent's announce list (none by default; the infohash does not depend on it); "root" names another playable root. A video prepared from the same source (by sha256) is skipped; "dryRun": true logs each decision. | |
| `feed-metadata` | — | `pnpm ops feed-metadata --json '{"slug":"demo-podcast","dryRun":true}' --wait` |
| `refresh-report` | — | `pnpm ops refresh-report --json '{"all":true}'` |
| `sync` | — | `pnpm ops sync --json '{"slug":"the-quartering"}' --wait` |
diff --git a/common/controller/preparePlayable.test.ts b/common/controller/preparePlayable.test.ts
@@ -200,6 +200,27 @@ test("remuxes each container losslessly, makes its torrent, and a re-run is a no
});
});
+test("each remux goes through the heavy slot's gate when the checkout has one", { skip: !tools && "ffmpeg/ffprobe not installed" }, async () => {
+ await withFixture(async ({ dir, paths, root, opts }) => {
+ await make(path.join(dir, "a.mp4"), ["-c:v", "libx264", "-preset", "ultrafast"], ["-c:a", "aac"]);
+ await saved(paths, "aaaaaaaaaaa", path.join(dir, "a.mp4"), "mp4");
+ // A stand-in for scripts/queue-lock.mjs: records its argv, runs the rest.
+ const repo = path.join(dir, "repo");
+ await mkdir(path.join(repo, "scripts"), { recursive: true });
+ const marker = path.join(dir, "gate.json");
+ await writeFile(
+ path.join(repo, "scripts", "queue-lock.mjs"),
+ `import { writeFileSync } from "node:fs";\nimport { spawnSync } from "node:child_process";\n` +
+ `const argv = process.argv.slice(2);\nwriteFileSync(${JSON.stringify(marker)}, JSON.stringify(argv));\n` +
+ `const at = argv.indexOf("--");\nconst r = spawnSync(argv[at + 1], argv.slice(at + 2), { stdio: "inherit" });\nprocess.exit(r.status ?? 1);\n`,
+ );
+ const r = await preparePlayable(opts({ root, paths: { ...paths, monorepoRoot: repo } as Paths }));
+ assert.deepEqual(r.prepared, ["aaaaaaaaaaa"]);
+ const argv = JSON.parse(await readFile(marker, "utf8")) as string[];
+ assert.deepEqual(argv.slice(0, 3), ["--heavy", "--", "ffmpeg"]);
+ });
+});
+
test("dryRun decides and writes nothing", { skip: !tools && "ffmpeg/ffprobe not installed" }, async () => {
await withFixture(async ({ dir, paths, root, opts }) => {
await make(path.join(dir, "a.mp4"), ["-c:v", "libx264", "-preset", "ultrafast"], ["-c:a", "aac"]);
diff --git a/common/controller/preparePlayable.ts b/common/controller/preparePlayable.ts
@@ -170,10 +170,19 @@ async function ffprobe(paths: Paths, file: string, signal?: AbortSignal): Promis
return parseProbe(JSON.parse(String(r.stdout)), path.extname(file));
}
+// EACH REMUX TAKES THE HEAVY SLOT (release 19 B1, scripts/queue-lock.mjs
+// --heavy — the gate `pnpm heavy` and a publish stage's `next build` use):
+// one heavy job machine-wide, above the memory floor. Per video, not per run,
+// so a 36 GB channel interleaves with builds and e2e instead of holding the
+// slot for hours. The gate forwards a cancel to ffmpeg. A checkout without
+// the gate (a test's temp root) runs ffmpeg directly; HEAVY=0 bypasses it.
async function ffmpegRemux(paths: Paths, src: string, dest: string, args: string[], signal?: AbortSignal): Promise<void> {
+ const ffmpegArgs = ["-nostdin", "-hide_banner", "-loglevel", "error", "-y", "-i", src, ...args, dest];
+ const gate = paths.monorepoRoot ? path.join(paths.monorepoRoot, "scripts", "queue-lock.mjs") : null;
+ const gated = gate !== null && (await stat(gate).then((s) => s.isFile()).catch(() => false));
await execa(
- paths.ffmpegBin,
- ["-nostdin", "-hide_banner", "-loglevel", "error", "-y", "-i", src, ...args, dest],
+ gated ? process.execPath : paths.ffmpegBin,
+ gated ? [gate, "--heavy", "--", paths.ffmpegBin, ...ffmpegArgs] : ffmpegArgs,
{ ...(signal ? { cancelSignal: signal } : {}) },
);
}
diff --git a/common/jobs/jobKinds.ts b/common/jobs/jobKinds.ts
@@ -984,7 +984,8 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
// videos. An INGEST kind: the playable manifest it writes is what a site's
// build reads for its torrents (D4). One queue machine-wide (`playable`):
// a remux is disk-bound on the archive drive, and two at once only thrash
- // it. Heavy — it belongs in release 19's heavy slot once that lands.
+ // it. Heavy: each remux takes release 19's heavy slot (`pnpm heavy`'s gate),
+ // one video at a time, so builds and e2e interleave with a long run.
"prepare-playable": {
kind: "prepare-playable",
label: "Prepare playable copies",
