commit 6f39ed08594451fe04fae88633c2c5b91efd3bd0
parent 10b4f62c5918e7a114fc0df16a58483da3a54e47
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 25 Sep 2026 00:48:41 -0400
Merge main (0e9d5dd0, release 6 follow-ups slice) into one-core/phase-4-s1
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
17 files changed, 282 insertions(+), 77 deletions(-)
diff --git a/common/controller/channelSnapshot.ts b/common/controller/channelSnapshot.ts
@@ -582,9 +582,10 @@ export function foldBackfillEntry(
// buildBands keeps its own fold for these two lanes because its numbers are
// different ones: its transcription `reachable` is downloadedNoTranscript
// ALONE — the retry bucket is not in it — and its `blocked` is derived from
-// noTranscript, which is this entry's missingInput. Pointing the bands here
-// would move two rendered numbers. See editor/app/components/pipelines/
-// buildBands.ts.
+// noTranscript (less the untranscribable and the downloaded ones), where this
+// entry states the whole noTranscript count as missingInput. Pointing the
+// bands here would move two rendered numbers. The bands' fold is the
+// transcription branch of common/views/pipeline/buildBands.ts.
export function foldBucketLaneEntry(
lane: AutoQueueKind,
source: BucketSource & { present: number },
diff --git a/common/controller/transcribeOne.ts b/common/controller/transcribeOne.ts
@@ -16,8 +16,9 @@ import { findSourceMedia, isRealAudioFile } from "../lib/videoStatus";
import { pickPreferredAudio } from "../lib/mediaFiles";
import { resolveSavedVideo } from "../lib/savedVideo-server";
import { writeTranscribeOutcome } from "../lib/transcribeOutcome-server";
+import { writeFileAtomic } from "../lib/jsonFile-server";
-const { pathExists, readdir, rename, writeFile } = fs;
+const { pathExists, readdir, rename } = fs;
async function resolveAudioFile(
videoDir: string,
@@ -168,9 +169,11 @@ export async function transcribeOneVideo(
await throwIfAudioVanished(opts.videoDir, resolvedAudio, opts.videoId);
throw err;
}
- // Upload path: write the pulled bytes. Shared-fs path (bytes === null): the
- // remote already wrote transcript.json onto the shared mount at this path.
- if (bytes) await writeFile(transcriptPath, bytes);
+ // Upload path: write the pulled bytes — atomically, so a crash mid-write
+ // never leaves a truncated transcript.json for the indexer to read. Shared-fs
+ // path (bytes === null): the remote already wrote transcript.json onto the
+ // shared mount at this path.
+ if (bytes) await writeFileAtomic(transcriptPath, bytes);
// The remote didn't tell us the engine format; sniff it from the file (the
// same content sniff the indexer uses for mixed corpora).
const raw = await fs.readFile(transcriptPath, "utf8").catch(() => "");
diff --git a/common/lib/availability.test.ts b/common/lib/availability.test.ts
@@ -1,6 +1,6 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { classifyDownloadFailure } from "./availability";
+import { classifyDownloadFailure, parseUnavailableFromStderr } from "./availability";
// Run with:
// pnpm --filter yt-dlp-transcript-common exec tsx --test lib/availability.test.ts
@@ -20,3 +20,18 @@ test("a bare HTTP 403 is a network failure (backs the platform off)", () => {
);
assert.equal(classifyDownloadFailure("ERROR: Unsupported URL", undefined), "unknown");
});
+
+test("HTTP 410 Gone is a removed video, not an error", () => {
+ // The exact line rekietalaw-rumble v7e07us's check has read since 2026-08-21.
+ const stderr =
+ "ERROR: [Rumble] v7e07us: Unable to download webpage: HTTP Error 410: Gone (caused by <HTTPError 410: Gone>)";
+ assert.equal(parseUnavailableFromStderr(stderr), "deleted");
+ // …so a download that hits it skips the video rather than aborting the batch.
+ assert.equal(
+ classifyDownloadFailure(stderr, parseUnavailableFromStderr(stderr)),
+ "per_video",
+ );
+ // Other HTTP errors are unchanged.
+ assert.equal(parseUnavailableFromStderr("HTTP Error 403: Forbidden"), "error");
+ assert.equal(parseUnavailableFromStderr("HTTP Error 500: Internal Server Error"), "error");
+});
diff --git a/common/lib/availability.ts b/common/lib/availability.ts
@@ -187,7 +187,10 @@ export function parseUnavailableFromStderr(stderr: string): Availability {
/account .* has been terminated/.test(s) ||
/this video has been removed/.test(s) ||
/content isn't available/.test(s) ||
- /no longer available/.test(s)
+ /no longer available/.test(s) ||
+ // HTTP 410 Gone is the server saying the video was removed for good —
+ // Rumble answers a taken-down video with it (`HTTP Error 410: Gone`).
+ /http error 410/.test(s)
) {
return "deleted";
}
diff --git a/common/lib/detectPlatform.mjs b/common/lib/detectPlatform.mjs
@@ -0,0 +1,31 @@
+// detectPlatform — which platform a URL is on, by host.
+//
+// Plain JS (with JSDoc types) rather than TS for ONE reason: umtool's
+// report-to-video scripts are `.mjs` run by bare `node` (20 in the runtime
+// image — no type stripping), and they spawn yt-dlp too, so they need the same
+// answer the editor gets. `lib/platform.ts` re-exports this; every TS caller
+// keeps importing it from there. Do not grow a second host table anywhere.
+
+/** @typedef {import("./platform").Platform} Platform */
+
+/**
+ * @param {string | undefined | null} url
+ * @returns {Platform | null}
+ */
+export function detectPlatform(url) {
+ if (!url) return null;
+ try {
+ const host = new URL(url).hostname.toLowerCase();
+ if (host.endsWith("youtube.com") || host === "youtu.be") return "youtube";
+ if (host.endsWith("rumble.com")) return "rumble";
+ if (host.endsWith("odysee.com")) return "odysee";
+ if (host.endsWith("twitch.tv")) return "twitch";
+ if (host.endsWith("kick.com")) return "kick";
+ if (host === "x.com" || host.endsWith(".x.com")) return "twitter";
+ if (host.endsWith("twitter.com")) return "twitter";
+ if (host === "bsky.app" || host.endsWith(".bsky.app")) return "bluesky";
+ } catch {
+ /* fall through */
+ }
+ return null;
+}
diff --git a/common/lib/platform.ts b/common/lib/platform.ts
@@ -33,25 +33,10 @@ export function isSocialPlatform(platform: Platform | null | undefined): boolean
return platform === "twitter" || platform === "bluesky";
}
-export function detectPlatform(
- url: string | undefined | null,
-): Platform | null {
- if (!url) return null;
- try {
- const host = new URL(url).hostname.toLowerCase();
- if (host.endsWith("youtube.com") || host === "youtu.be") return "youtube";
- if (host.endsWith("rumble.com")) return "rumble";
- if (host.endsWith("odysee.com")) return "odysee";
- if (host.endsWith("twitch.tv")) return "twitch";
- if (host.endsWith("kick.com")) return "kick";
- if (host === "x.com" || host.endsWith(".x.com")) return "twitter";
- if (host.endsWith("twitter.com")) return "twitter";
- if (host === "bsky.app" || host.endsWith(".bsky.app")) return "bluesky";
- } catch {
- /* fall through */
- }
- return null;
-}
+// Host → platform. Lives in `detectPlatform.mjs` (plain JS so umtool's `.mjs`
+// scripts can import the same copy); every TS caller imports it from here.
+import { detectPlatform } from "./detectPlatform.mjs";
+export { detectPlatform };
// Best-effort canonical webpage URL for a video given its platform and
// canonical id. Used both as a metadata fallback in summarize() and to
diff --git a/common/package.json b/common/package.json
@@ -31,6 +31,8 @@
"./components/urlState": "./components/urlState.ts",
"./components/virtualizer": "./components/virtualizer.ts",
"./components/*": "./components/*.tsx",
+ "./lib/detectPlatform.mjs": "./lib/detectPlatform.mjs",
+ "./ytdlp/platformArgs.mjs": "./ytdlp/platformArgs.mjs",
"./lib/*": "./lib/*.ts",
"./controller/*": "./controller/*.ts",
"./jobs/*": "./jobs/*.ts",
diff --git a/common/ytdlp/channelArgs.ts b/common/ytdlp/channelArgs.ts
@@ -18,23 +18,10 @@ import type { ChannelConfig } from "../lib/channelConfig";
import { cookieArgs } from "../lib/cookiePolicy";
import { detectPlatform, type Platform } from "../lib/platform";
-// Args a platform needs on EVERY yt-dlp spawn, whatever the job. Code, not a
-// setting: per-channel `ytdlpExtraArgs` is the tweak surface, and it comes
-// AFTER these, so a channel override wins (yt-dlp is last-flag-wins for
-// `--sleep-requests` and `--impersonate`).
-//
-// rumble: every request 403s at Cloudflare without a browser TLS fingerprint
-// (yt-dlp #17496); a probe on 2026-09-24 got 200 with `--impersonate chrome`
-// and 403 without. `--sleep-requests 1` paces the listing walk — the
-// the-quartering-rumble full sweep 429'd at page 155 unpaced the same day.
-export const PLATFORM_ARGS: Partial<Record<Platform, readonly string[]>> = {
- rumble: ["--impersonate", "chrome", "--sleep-requests", "1"],
-};
-
-export function platformArgs(platform: Platform | null | undefined): string[] {
- if (!platform) return [];
- return [...(PLATFORM_ARGS[platform] ?? [])];
-}
+// The platform args table lives in `platformArgs.mjs` (plain JS so umtool's
+// `.mjs` scripts can import the same copy); re-exported here for TS callers.
+import { platformArgs } from "./platformArgs.mjs";
+export { PLATFORM_ARGS, platformArgs, platformArgsForUrl } from "./platformArgs.mjs";
export function channelPlatform(
config: Pick<ChannelConfig, "platform" | "url">,
diff --git a/common/ytdlp/platformArgs.mjs b/common/ytdlp/platformArgs.mjs
@@ -0,0 +1,44 @@
+// The per-platform yt-dlp args table — THE one copy.
+//
+// Plain JS (with JSDoc types) so umtool's report-to-video scripts, which are
+// `.mjs` run by bare `node` (20 in the runtime image — no type stripping), can
+// import it: they spawn yt-dlp for clip fetches and availability checks and hit
+// the same Cloudflare wall. `ytdlp/channelArgs.ts` re-exports it for every TS
+// caller. Never copy these args into another file.
+
+import { detectPlatform } from "../lib/detectPlatform.mjs";
+
+/** @typedef {import("../lib/platform").Platform} Platform */
+
+// Args a platform needs on EVERY yt-dlp spawn, whatever the job. Code, not a
+// setting: per-channel `ytdlpExtraArgs` is the tweak surface, and it comes
+// AFTER these, so a channel override wins (yt-dlp is last-flag-wins for
+// `--sleep-requests` and `--impersonate`).
+//
+// rumble: every request 403s at Cloudflare without a browser TLS fingerprint
+// (yt-dlp #17496); a probe on 2026-09-24 got 200 with `--impersonate chrome`
+// and 403 without. `--sleep-requests 1` paces the listing walk — the
+// the-quartering-rumble full sweep 429'd at page 155 unpaced the same day.
+/** @type {Readonly<Partial<Record<Platform, readonly string[]>>>} */
+export const PLATFORM_ARGS = Object.freeze({
+ rumble: Object.freeze(["--impersonate", "chrome", "--sleep-requests", "1"]),
+});
+
+/**
+ * @param {Platform | null | undefined} platform
+ * @returns {string[]}
+ */
+export function platformArgs(platform) {
+ if (!platform) return [];
+ return [...(PLATFORM_ARGS[platform] ?? [])];
+}
+
+/**
+ * The args for whatever platform `url` is on — for a spawn that has a URL and
+ * no channel config (umtool's clip fetch and availability check).
+ * @param {string | undefined | null} url
+ * @returns {string[]}
+ */
+export function platformArgsForUrl(url) {
+ return platformArgs(detectPlatform(url));
+}
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,9 @@
# Changelog
## [Unreleased]
+- **A video the server answers with HTTP 410 Gone is recorded as removed, not as an error.** Rumble answers a taken-down video with `HTTP Error 410: Gone`; the availability check read that as a generic error (one Rekieta Law Rumble video has said "error" since 2026-08-21), and a download that hit it could stop the batch. It now reads as removed, like "Video unavailable" does, so the check says so and a download skips that one video and carries on. Existing records change the next time the video is checked.
+- **umtool's report videos can fetch Rumble clips again.** The clip fetch and the source availability check in `umtool/report-to-video` ran yt-dlp without the browser fingerprint Rumble now requires, so every Rumble clip failed with 403 and every Rumble source looked missing. They now pass the same Rumble arguments as the editor, from the same single table.
+- **A transcript pulled back from a remote worker is written safely.** It used to be written straight onto `transcript.json`, so a crash part-way through left a truncated transcript; it now goes through the editor's one atomic write (temp file, then rename), like every other file the editor writes.
- **Rumble works again, and a Rumble full sweep that gets rate-limited no longer fails the sync.** Every Rumble request had started coming back 403 from Cloudflare unless yt-dlp presents a browser fingerprint (yt-dlp #17496), so Rumble downloads failed and a Rumble channel could not even be added. Every yt-dlp run for a Rumble channel — sync, download, metadata scan, availability check, the clip-window fetch and the new-channel probe — now passes `--impersonate chrome --sleep-requests 1`, from one table in the code; a channel's own extra yt-dlp arguments still come last and still win. Separately, a full sweep that hits HTTP 429 part-way through the listing used to fail the whole sync and try again on the next one, so a large channel (The Quartering on Rumble, 44 days) never synced at all. What it read is now treated as *incomplete* — not a listing, so nothing is flagged missing and the stored playlist is untouched: the job records the platform's rate-limit cooldown, says "sweep incomplete: 429 at page N of the listing, M entries" in its log, does the ordinary newest-first sync instead, and succeeds. Syncs for that platform are then refused until its cooldown ends, and the full sweep is tried again after that. Any other yt-dlp failure still fails the sync as before.
- **A site can turn off its visitors' per-video transcript downloads.** The transcript viewer on a published site has always offered three ways to take a video's text away: a **Download** menu (txt, srt, json), **Copy MD**, and **Copy download command** (a `yt-dlp` line for a marked clip). A site's settings form now has a checkbox for them, *Per-video transcript downloads*, beside the archive zips one. Unticked, the site's next build shows none of the three; **Share** and the clip marks stay. It is on by default, so a site nobody touches is unchanged, and the file stores `"transcriptDownloads": false` only when it is off (`SITE.md` has the key). The site's machine contract (`/corpus.json`, `llms.txt`, the manifests and shards the MCP server and report-to-video read) is published either way. The hub follows the same switch: the hub form on **Sites** has the same checkbox, stored as `"transcriptDownloads": false` in the hub's `homepage.json`, and it hides the three controls on the hub's Browse and Ask pages. The editor's own video pages are unaffected.
- **Channel rows no longer scroll over a group's controls on `/channels`.** Scrolled down and to the right, the pinned Slug column of every row painted over the pinned group header and its five station buttons (Sync, Download, Transcribe, Digest and the speaker lane), and took the clicks. The pinned Slug cell and the group header sat at the same stacking level, and the later rows won. The rack now has one named layer order, kept in one file: the Advanced panel, then the column header, then the group header, then the pinned checkbox and Slug cells. Nothing ties any more. The screenshot audit found four more problems, fixed as well. A group header's name and buttons now stay on screen however far the columns scroll across (they used to scroll off to the left). An Advanced panel opened near the bottom or the right edge scrolls itself into view instead of being cut off. The rule above a pinned group header moves with it instead of leaving a gap the rows showed through. On a phone, the column header no longer paints over the selection bar pinned to the bottom of the screen.
diff --git a/editor/scripts/measure-nav.mjs b/editor/scripts/measure-nav.mjs
@@ -60,13 +60,18 @@ const ROUTES = [
["/changelog", "changelog"],
];
+// The polled endpoints, by the /api/view/<name> paths the pages actually call
+// (editor/app/api/view/[name]). The six pre-phase-3 paths (/api/pulse,
+// /api/widget/{cleanable,actionable,sync}, /api/jobs/active, /api/workers) are
+// retired — they answer only through editor/next.config.ts's rewrites, so
+// timing them measured rewrite + view, a hop no page pays.
const APIS = [
- "/api/pulse",
- "/api/widget/cleanable",
- "/api/widget/actionable",
- "/api/widget/sync",
- "/api/jobs/active",
- "/api/workers",
+ "/api/view/pulse",
+ "/api/view/cleanable",
+ "/api/view/widgetActionable",
+ "/api/view/widgetSync",
+ "/api/view/activeJobs",
+ "/api/view/workers",
];
// The 4-element form. A 5th element is a 500; `true` in slot 4 means "refetch",
@@ -161,12 +166,12 @@ for (const [route, segment] of ROUTES) {
);
}
-console.log("\n" + "endpoint".padEnd(28) + "ms".padStart(8) + " size");
+console.log("\n" + "endpoint".padEnd(30) + "ms".padStart(8) + " size");
console.log("-".repeat(56));
for (const api of APIS) {
const r = await best(() => timeGet(`${BASE}${api}`));
console.log(
- api.padEnd(28) +
+ api.padEnd(30) +
fmt(r.ms) +
" " +
kb(r.bytes) +
@@ -176,16 +181,16 @@ for (const api of APIS) {
// The idle pulse is the one that has to stay cheap forever — measure it the way
// the client actually calls it, with a rev in hand.
-const first = await timeGet(`${BASE}/api/pulse`);
+const first = await timeGet(`${BASE}/api/view/pulse`);
if (first.status === 200) {
- const res = await fetch(`${BASE}/api/pulse`);
+ const res = await fetch(`${BASE}/api/view/pulse`);
const body = await res.json();
const idle = await best(() =>
- timeGet(`${BASE}/api/pulse?rev=${encodeURIComponent(body.rev)}`),
+ timeGet(`${BASE}/api/view/pulse?rev=${encodeURIComponent(body.rev)}`),
);
console.log(
"\n" +
- "/api/pulse (idle, rev matched)".padEnd(34) +
+ "/api/view/pulse (idle, rev matched)".padEnd(36) +
fmt(idle.ms) +
"ms ← this one is polled forever; keep it tiny",
);
diff --git a/plans/release-6.md b/plans/release-6.md
@@ -0,0 +1,47 @@
+# Release 6 — follow-ups + Phase 4 slice 1
+
+`main` at `93dcb532` (release 5 merged: slice R, Rumble; slice X, visitor exports off). Release 5 is
+live. This release clears release 5's "found and left" items and Phase 3's deferred items, then lands Phase 4 slice 1.
+
+## Record
+
+### Follow-ups slice, as shipped — 410 is removed, umtool's Rumble args, one atomic write, a comment, measure-nav (2026-09-25)
+
+Branch `one-core/r6-followups` off `main` `93dcb532` (`main` had not moved). Five small items,
+one commit each. Four came from release 5's "found and left" list and Phase 3's deferred list. The
+fifth came from a live sidecar.
+
+| sha | what |
+|---|---|
+| `6d5cdbc3` | `common/lib/availability.ts` `parseUnavailableFromStderr`: `/http error 410/` joins the `deleted` group. There was no `404` pattern to copy, so it sits with "video unavailable" / "no longer available". The evidence is `rekietalaw-rumble` `v7e07us`'s sidecar (read only, not re-checked). It has read `error` since 2026-08-21, with `ERROR: [Rumble] v7e07us: Unable to download webpage: HTTP Error 410: Gone (caused by <HTTPError 410: Gone>)`. The classifier now returns `deleted`, so `classifyDownloadFailure` makes a download that hits it `per_video` (skip) where it used to be `unknown` (batch abort). `availability.test.ts` +1 test: that exact line, the per-video consequence, and 403/500 still `error` |
+| `9a88ee4b` | umtool's yt-dlp spawns carry the platform args. `report-to-video/*.mjs` run under bare `node`, and the runtime image is `node:20` (no type stripping), so they cannot import common's TS. **The shim moves the table rather than copying it.** `PLATFORM_ARGS` / `platformArgs` (+ a new `platformArgsForUrl`) now live in `common/ytdlp/platformArgs.mjs`, and `detectPlatform` in `common/lib/detectPlatform.mjs`. Both are plain JS with JSDoc types. `channelArgs.ts` and `platform.ts` re-export them, so no TS caller changed (the base tsconfig already has `allowJs`). common's `package.json` exports the two files by exact path; the `./lib/*` / `./ytdlp/*` patterns map to `.ts`. `umtool-report-to-video` gains a `yt-dlp-transcript-common` dependency. The lockfile gets that importer entry only: a plain `pnpm install` re-resolved ~280 lines of peer suffixes, so the entry was applied by hand and verified with `pnpm install --frozen-lockfile`. `build-video.mjs` exports `clipFetchArgs` (platform args before a retry's `extra`), and `check-availability.mjs` exports `simulateArgs`. New `platform-args.test.mjs` (3): Rumble gets the table's args in position, YouTube gets none |
+| `947f07aa` | `common/controller/transcribeOne.ts`: the remote upload path's `writeFile(transcriptPath, bytes)` is now `writeFileAtomic` (`lib/jsonFile-server.ts` — the brief's `writeJsonAtomic.ts` does not exist; slice W's idiom lives there). The shared-fs path is unchanged. No new test: the path needs a remote worker, and the repo does not module-mock. `transcribeOne.test.ts` (3) is green |
+| `71e2bb41` | `common/controller/channelSnapshot.ts` above `foldBucketLaneEntry`: the comment named the retired `editor/app/components/pipelines/buildBands.ts`. It now names `common/views/pipeline/buildBands.ts`. It also said the band's `blocked` "is this entry's missingInput"; it is `noTranscript` less the untranscribable and downloaded ones (`buildBands.ts:171-175`), so the sentence says that. Comment only |
+| `b4bd58a3` | `editor/scripts/measure-nav.mjs` (the brief said `plans/tools/`; it has always been under `editor/scripts/`). The six retired paths are `/api/pulse`, `/api/widget/{cleanable,actionable,sync}`, `/api/jobs/active` and `/api/workers`, which only reach a handler through `next.config.ts`'s rewrites. They are replaced by the `/api/view/<name>` paths the pages poll (`pulse`, `cleanable`, `widgetActionable`, `widgetSync`, `activeJobs`, `workers`), and the idle-pulse check uses `/api/view/pulse`. The 13 page routes were already current. Run once against the live :3001 (`--runs 1`, GETs only): no route or endpoint flagged, `/api/view/pulse` idle 8 ms |
+| *(this commit)* | this record, `[Unreleased]` bullets |
+
+**Gates** (worktree root). tsc (`pnpm -r --no-bail --workspace-concurrency=1 exec tsc --noEmit`)
+clean before every commit. common **1751/1751** (1750 + 1). test:scripts **159 pass + 1 skip**
+(156 + 1 skip + the 3 new). editor unit **72/72**. mcp **219/219**. `pnpm --filter editor exec
+next build` ok (50 s). `pnpm --filter export exec next build` ok (34 s). Before the builds, the
+worktree's `export/public/archives` link was dangling, because the primary has no `archives/` at
+the moment; it was removed, and no dangling links were left. EDITOR e2e `availability
+availability-backfill maybe-missing` (from `$T/f-specs.txt`; Playwright's substring match also
+picked up `pre-clean-availability`): **15 passed, 0 failed, 2.0 min**, with no wait in the
+queue. Numbers: **none**. No file format changed; `availability.json` keeps its shape, and a
+410 video's value changes only when it is next checked.
+
+**Found and left.**
+- **`v7e07us` still reads `error` on disk.** Nothing here re-checks it (the brief forbids live
+ checks). The next availability check or backfill that reaches it records `deleted`, and the
+ `history` gains that entry.
+- **umtool's `check-availability.mjs` has its own classifier** (`classify`, which already matches
+ a bare `410` as deleted). It differs from common's `parseUnavailableFromStderr`, and its states
+ are different ones (`maybe_missing`, `geo-blocked`, `restricted`). It was left alone because
+ it is umtool's editorial vocabulary, not a copy of the args table.
+- **`--sleep-requests 1` now applies to Rumble clip fetches too.** It comes with the table and
+ adds about a second per request on a clip fetch; that is harmless, and one table is the point.
+- **measure-nav's `nav`/`prefetch` columns read ~0.1 kb on every route of the live editor.** That
+ is the same for routes this commit did not touch. The state-tree encoding may have drifted with
+ Next 16.2; this commit only replaced the paths, so it was not investigated.
+- **Commit trailers** name `Claude Opus 5.5 (1M context)`, as in release 5.
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
@@ -360,7 +360,11 @@ importers:
specifier: ^5.9.3
version: 5.9.3
- umtool/report-to-video: {}
+ umtool/report-to-video:
+ dependencies:
+ yt-dlp-transcript-common:
+ specifier: workspace:*
+ version: link:../../common
packages:
diff --git a/umtool/report-to-video/build-video.mjs b/umtool/report-to-video/build-video.mjs
@@ -63,6 +63,9 @@ import {
cardWidth, contentWidth, reservedFooterHeight,
} from "./render-cards.mjs";
import { createCueSource, siteOriginFromManifest } from "./cues.mjs";
+// The per-platform yt-dlp args (Rumble's `--impersonate chrome`): the ONE table,
+// in common, plain JS so bare `node` can load it.
+import { platformArgsForUrl } from "yt-dlp-transcript-common/ytdlp/platformArgs.mjs";
// The header line, the clock in it and the title cleaner live in one module
// the clip bench imports too -- a preview that shows a line this renderer
// would never draw is worse than no preview.
@@ -351,6 +354,32 @@ export async function findContainingWindow(rawDir, video, from, to) {
return tightestContaining(await cachedWindowsFor(rawDir, video), from, to);
}
+// The yt-dlp argv for one clip window. The platform's fixed args come from
+// common's table (`platformArgsForUrl`) -- the same copy every editor spawn
+// uses, so a Rumble fetch carries `--impersonate chrome` here too instead of
+// 403ing at Cloudflare. They go BEFORE `extra`, so a retry's own flags win.
+export function clipFetchArgs({ url, from, to, fmt, dest, extra = [] }) {
+ return [
+ // The operator's own yt-dlp config redirects output and attaches thumbnail
+ // and metadata post-processors; without this the clips land elsewhere.
+ "--ignore-config",
+ "--no-playlist",
+ "--download-sections", `*${from.toFixed(2)}-${to.toFixed(2)}`,
+ // Without this the cut snaps to the nearest preceding keyframe, which can be
+ // seconds early — fine for scrubbing, not fine when the clip IS the citation.
+ "--force-keyframes-at-cuts",
+ ...platformArgsForUrl(url),
+ ...extra,
+ // Pin H.264/AAC in mp4. Left alone yt-dlp picks VP9+Opus at these heights,
+ // and since --force-keyframes-at-cuts re-encodes, that means libvpx-vp9 —
+ // 27s to cut a 5s clip. It also writes .webm and appends that to -o.
+ "-f", fmt,
+ "--merge-output-format", "mp4",
+ "-o", dest,
+ "--", url,
+ ];
+}
+
/**
* WHERE A CLIP IS CUT OUT OF A CACHED WINDOW, as ffmpeg input arguments.
*
@@ -415,24 +444,8 @@ async function fetchClip(entry, meta, render, rawDir, opts) {
`b[height<=${maxH}]`,
].join("/");
- const argsWith = (extra) => [
- // The operator's own yt-dlp config redirects output and attaches thumbnail
- // and metadata post-processors; without this the clips land elsewhere.
- "--ignore-config",
- "--no-playlist",
- "--download-sections", `*${from.toFixed(2)}-${to.toFixed(2)}`,
- // Without this the cut snaps to the nearest preceding keyframe, which can be
- // seconds early — fine for scrubbing, not fine when the clip IS the citation.
- "--force-keyframes-at-cuts",
- ...extra,
- // Pin H.264/AAC in mp4. Left alone yt-dlp picks VP9+Opus at these heights,
- // and since --force-keyframes-at-cuts re-encodes, that means libvpx-vp9 —
- // 27s to cut a 5s clip. It also writes .webm and appends that to -o.
- "-f", fmt,
- "--merge-output-format", "mp4",
- "-o", dest,
- "--", meta.webpageUrl,
- ];
+ const argsWith = (extra) =>
+ clipFetchArgs({ url: meta.webpageUrl, from, to, fmt, dest, extra });
const attempt = async (extra) => {
try {
diff --git a/umtool/report-to-video/check-availability.mjs b/umtool/report-to-video/check-availability.mjs
@@ -26,6 +26,9 @@ import { mkdir, readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import { DEFAULT_CHANNELS_DIR } from "./cues.mjs";
+// The per-platform yt-dlp args (Rumble's `--impersonate chrome`): the ONE table,
+// in common, plain JS so bare `node` can load it.
+import { platformArgsForUrl } from "yt-dlp-transcript-common/ytdlp/platformArgs.mjs";
const execFileP = promisify(execFile);
@@ -50,6 +53,17 @@ function classify(stderr) {
return "maybe_missing";
}
+// `--simulate` resolves formats without downloading. It carries the platform's
+// fixed args like every other spawn: without `--impersonate` every Rumble source
+// 403s here and reads as missing when it is not.
+export function simulateArgs(url) {
+ return [
+ "--ignore-config", "--no-playlist", "--simulate", "--quiet", "--no-warnings",
+ ...platformArgsForUrl(url),
+ "--", url,
+ ];
+}
+
async function cueMeta(videoId, channelSlug) {
const p = path.join(CHANNELS_DIR, channelSlug, "data", videoId, "transcript.cues.json");
const d = JSON.parse(await readFile(p, "utf8"));
@@ -118,7 +132,7 @@ export async function checkAvailability(manifestPath, { outDir, maxAgeDays = 0 }
try {
await execFileP(
YTDLP,
- ["--ignore-config", "--no-playlist", "--simulate", "--quiet", "--no-warnings", "--", meta.webpageUrl],
+ simulateArgs(meta.webpageUrl),
{ maxBuffer: 1 << 24 },
);
sources.push({
diff --git a/umtool/report-to-video/package.json b/umtool/report-to-video/package.json
@@ -4,6 +4,9 @@
"private": true,
"type": "module",
"description": "Turn a cited sweep report into a narrated-by-text video.",
+ "dependencies": {
+ "yt-dlp-transcript-common": "workspace:*"
+ },
"bin": {
"report-build-video": "./build-video.mjs",
"report-resolve-windows": "./resolve-windows.mjs",
diff --git a/umtool/report-to-video/platform-args.test.mjs b/umtool/report-to-video/platform-args.test.mjs
@@ -0,0 +1,45 @@
+// Tests that umtool's two yt-dlp spawns carry the platform's fixed args from
+// common's ONE table — a Rumble clip fetch or availability check without
+// `--impersonate chrome` 403s at Cloudflare (yt-dlp #17496).
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { PLATFORM_ARGS } from "yt-dlp-transcript-common/ytdlp/platformArgs.mjs";
+import { clipFetchArgs } from "./build-video.mjs";
+import { simulateArgs } from "./check-availability.mjs";
+
+const RUMBLE = "https://rumble.com/v7e07us-some-title.html";
+const YOUTUBE = "https://www.youtube.com/watch?v=dQw4w9WgXcQ";
+
+/** The args between two markers, so the assertion names the position too. */
+const slice = (argv, from, to) => argv.slice(argv.indexOf(from) + 1, argv.indexOf(to));
+
+test("a Rumble clip fetch carries the rumble platform args, before the retry's extra", () => {
+ const argv = clipFetchArgs({
+ url: RUMBLE, from: 12, to: 20.5, fmt: "b", dest: "/x/out.mp4",
+ extra: ["--downloader-args", "ffmpeg_i:-extension_picky 0"],
+ });
+ assert.deepEqual(
+ slice(argv, "--force-keyframes-at-cuts", "-f"),
+ [...PLATFORM_ARGS.rumble, "--downloader-args", "ffmpeg_i:-extension_picky 0"],
+ );
+ assert.ok(argv.includes("--impersonate"));
+ assert.deepEqual(argv.slice(-2), ["--", RUMBLE]);
+ assert.equal(argv[argv.indexOf("--download-sections") + 1], "*12.00-20.50");
+});
+
+test("a YouTube clip fetch gets no platform args", () => {
+ const argv = clipFetchArgs({ url: YOUTUBE, from: 1, to: 2, fmt: "b", dest: "/x/out.mp4" });
+ assert.deepEqual(slice(argv, "--force-keyframes-at-cuts", "-f"), []);
+ assert.ok(!argv.includes("--impersonate"));
+});
+
+test("the availability simulate carries the platform args for Rumble only", () => {
+ const rumble = simulateArgs(RUMBLE);
+ assert.deepEqual(slice(rumble, "--no-warnings", "--"), [...PLATFORM_ARGS.rumble]);
+ assert.deepEqual(rumble.slice(-2), ["--", RUMBLE]);
+ assert.ok(rumble.includes("--simulate"));
+ assert.ok(!simulateArgs(YOUTUBE).includes("--impersonate"));
+});
