commit 5c877b01967c1338342230720ac6c4b78b8e3799
parent 12353c3cc6b1c40d6bace8529aa5ec0e88d144a6
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 6 Oct 2026 11:29:57 -0400
imports: a one-off Odysee or BitChute import waits a floor after the last one there
Each import is its own job, so nothing spaced them: Odysee answered 429 to
imports queued 10-50 s apart. PLATFORM_IMPORT_MIN_GAP_SECONDS (odysee: 60)
and platformImportMinGapSeconds (the larger of it and the batch floor) say
how long; jobs/platformGap.ts keeps each platform's next start process-wide
and waits it out, re-reading after each sleep, abortable. importVideoAction
gives every URL on such a platform what a BitChute import already had: its
own queue and args, a refusal while held or cooling down, a wait for a
cooldown that began after it was queued, and its answer recorded. Batch
downloads and syncs of an Odysee channel keep the operator's gap.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
7 files changed, 245 insertions(+), 12 deletions(-)
diff --git a/common/jobs/platformGap.test.ts b/common/jobs/platformGap.test.ts
@@ -0,0 +1,72 @@
+import { test, beforeEach } from "node:test";
+import assert from "node:assert/strict";
+import {
+ clearPlatformGaps,
+ notePlatformGap,
+ platformGapRemainingMs,
+ waitForPlatformGap,
+} from "./platformGap";
+
+beforeEach(() => clearPlatformGaps());
+
+test("a platform with no import yet is asked at once", () => {
+ assert.equal(platformGapRemainingMs("odysee", 1_000), 0);
+});
+
+test("the next import waits the gap from when the last one settled, per platform", () => {
+ notePlatformGap("odysee", 60_000, 1_000);
+ assert.equal(platformGapRemainingMs("odysee", 1_000), 60_000);
+ assert.equal(platformGapRemainingMs("odysee", 31_000), 30_000);
+ assert.equal(platformGapRemainingMs("odysee", 61_000), 0);
+ assert.equal(platformGapRemainingMs("bitchute", 1_000), 0);
+});
+
+test("a later start is never shortened by a shorter gap", () => {
+ notePlatformGap("odysee", 90_000, 0);
+ notePlatformGap("odysee", 10_000, 5_000);
+ assert.equal(platformGapRemainingMs("odysee", 5_000), 85_000);
+});
+
+test("the gap lives on globalThis, so every bundle shares it", () => {
+ notePlatformGap("odysee", 1_000, 0);
+ assert.equal(globalThis.__yttPlatformGap__?.get("odysee"), 1_000);
+});
+
+test("waiting re-reads the remaining time after each sleep and logs each wait", async () => {
+ const left = [45_000, 2_000, 0];
+ const slept: number[] = [];
+ const lines: string[] = [];
+ const waited = await waitForPlatformGap({
+ label: "Odysee",
+ remainingMs: () => left.shift() ?? 0,
+ onLog: (l) => lines.push(l),
+ sleep: async (ms) => {
+ slept.push(ms);
+ },
+ });
+ assert.deepEqual(slept, [45_000, 2_000]);
+ assert.equal(waited, 47_000);
+ assert.deepEqual(lines, [
+ "Odysee asks for a gap between videos: waiting 45s.",
+ "Odysee asks for a gap between videos: waiting 2s.",
+ ]);
+});
+
+test("nothing to wait: no sleep, no line", async () => {
+ const lines: string[] = [];
+ const waited = await waitForPlatformGap({
+ label: "Odysee",
+ remainingMs: async () => 0,
+ onLog: (l) => lines.push(l),
+ sleep: async () => assert.fail("must not sleep"),
+ });
+ assert.equal(waited, 0);
+ assert.deepEqual(lines, []);
+});
+
+test("an aborted job stops waiting", async () => {
+ const ac = new AbortController();
+ const p = waitForPlatformGap({ label: "Odysee", remainingMs: () => 60_000, signal: ac.signal });
+ ac.abort();
+ await assert.rejects(p, { name: "AbortError" });
+});
diff --git a/common/jobs/platformGap.ts b/common/jobs/platformGap.ts
@@ -0,0 +1,82 @@
+// THE GAP BETWEEN TWO ONE-OFF IMPORTS ON ONE PLATFORM. A batch download, a
+// sync, a persist and the auto-download lane each wait their platform's floor
+// between two videos — but every import is its own job, so ten imports queued
+// on `platform:odysee` used to ask Odysee as fast as each job could finish
+// (Odysee answered 429 at 10–50 s apart, 2026-10-06). An import on a platform
+// with an import floor (platformImportMinGapSeconds, ytdlp/platformArgs.mjs)
+// waits here first, and sets the platform's next start when it settles.
+//
+// Process-wide (on globalThis, as the job registry is: a route bundle and the
+// job runner must share one view) and in memory: a restart is itself a gap.
+
+declare global {
+ // eslint-disable-next-line no-var
+ var __yttPlatformGap__: Map<string, number> | undefined;
+}
+
+function nextStartAt(): Map<string, number> {
+ if (!globalThis.__yttPlatformGap__) globalThis.__yttPlatformGap__ = new Map();
+ return globalThis.__yttPlatformGap__;
+}
+
+// Milliseconds before `platform` may be asked again (0 = now).
+export function platformGapRemainingMs(platform: string, now: number = Date.now()): number {
+ const at = nextStartAt().get(platform);
+ return at !== undefined && at > now ? at - now : 0;
+}
+
+// A unit on `platform` settled at `now`: the next one starts no sooner than
+// `gapMs` later. A later start already set is kept (never shortened).
+export function notePlatformGap(platform: string, gapMs: number, now: number = Date.now()): void {
+ if (gapMs <= 0) return;
+ const at = now + gapMs;
+ const map = nextStartAt();
+ if ((map.get(platform) ?? 0) < at) map.set(platform, at);
+}
+
+export function clearPlatformGaps(): void {
+ nextStartAt().clear();
+}
+
+// Wait out `remainingMs()` — re-read after each wait, since another import or
+// a 429 may have moved it — logging once per wait. Rejects with an AbortError
+// when `signal` aborts.
+export async function waitForPlatformGap(opts: {
+ label: string;
+ remainingMs: () => Promise<number> | number;
+ signal?: AbortSignal;
+ onLog?: (line: string) => void;
+ sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
+}): Promise<number> {
+ const sleep = opts.sleep ?? abortableSleep;
+ let waited = 0;
+ for (;;) {
+ if (opts.signal?.aborted) throw abortError();
+ const ms = await opts.remainingMs();
+ if (ms <= 0) return waited;
+ opts.onLog?.(`${opts.label} asks for a gap between videos: waiting ${Math.ceil(ms / 1000)}s.`);
+ await sleep(ms, opts.signal);
+ waited += ms;
+ }
+}
+
+function abortError(): Error {
+ const err = new Error("aborted while waiting for the platform gap");
+ err.name = "AbortError";
+ return err;
+}
+
+function abortableSleep(ms: number, signal?: AbortSignal): Promise<void> {
+ return new Promise((resolve, reject) => {
+ if (signal?.aborted) return reject(abortError());
+ const t = setTimeout(() => {
+ signal?.removeEventListener("abort", onAbort);
+ resolve();
+ }, ms);
+ const onAbort = () => {
+ clearTimeout(t);
+ reject(abortError());
+ };
+ signal?.addEventListener("abort", onAbort, { once: true });
+ });
+}
diff --git a/common/ytdlp/channelArgs.test.ts b/common/ytdlp/channelArgs.test.ts
@@ -6,6 +6,8 @@ import {
pacedPlatformArgs,
platformArgs,
platformArgsForUrl,
+ platformImportMinGapSeconds,
+ PLATFORM_IMPORT_MIN_GAP_SECONDS,
PLATFORM_ARGS,
PLATFORM_MIN_GAP_SECONDS,
platformMinGapSeconds,
@@ -185,3 +187,14 @@ test("BitChute is paced hardest: 3 s between requests, exponential retry sleeps,
assert.equal(platformMinGapSeconds(p), 0, String(p));
}
});
+
+test("one-off imports have their own floor: Odysee's 60 s, BitChute's batch floor, none elsewhere", () => {
+ assert.equal(PLATFORM_IMPORT_MIN_GAP_SECONDS.odysee, 60);
+ assert.equal(platformImportMinGapSeconds("odysee"), 60);
+ assert.equal(platformImportMinGapSeconds("bitchute"), 60);
+ // Odysee's batch gap is untouched: the import floor is imports only.
+ assert.equal(platformMinGapSeconds("odysee"), 0);
+ for (const p of ["youtube", "rumble", "archiveorg", "unknown", null]) {
+ assert.equal(platformImportMinGapSeconds(p), 0, String(p));
+ }
+});
diff --git a/common/ytdlp/channelArgs.ts b/common/ytdlp/channelArgs.ts
@@ -41,6 +41,8 @@ export {
platformArgsForUrl,
PLATFORM_MIN_GAP_SECONDS,
platformMinGapSeconds,
+ PLATFORM_IMPORT_MIN_GAP_SECONDS,
+ platformImportMinGapSeconds,
staticSleepRequestsSeconds,
withSleepRequests,
} from "./platformArgs.mjs";
diff --git a/common/ytdlp/platformArgs.mjs b/common/ytdlp/platformArgs.mjs
@@ -94,6 +94,18 @@ export const PLATFORM_MIN_GAP_SECONDS = Object.freeze({
bitchute: 60,
});
+// THE FLOOR UNDER THE GAP BETWEEN TWO ONE-OFF IMPORTS on a platform, in
+// seconds, where it must be higher than the batch floor above. Each import is
+// its own job, so nothing else spaces them: Odysee's API answered HTTP 429 to
+// imports queued 10–50 s apart (2026-10-06). An import waits the larger of
+// this and PLATFORM_MIN_GAP_SECONDS after the last import there, plus up to
+// half again at random (jobs/platformGap.ts). Batch downloads and syncs of an
+// Odysee channel keep the operator's gap.
+/** @type {Readonly<Partial<Record<Platform, number>>>} */
+export const PLATFORM_IMPORT_MIN_GAP_SECONDS = Object.freeze({
+ odysee: 60,
+});
+
/**
* @param {string | null | undefined} platform
* @returns {number}
@@ -106,6 +118,17 @@ export function platformMinGapSeconds(platform) {
}
/**
+ * @param {string | null | undefined} platform
+ * @returns {number}
+ */
+export function platformImportMinGapSeconds(platform) {
+ if (!platform) return 0;
+ /** @type {Record<string, number | undefined>} */
+ const table = PLATFORM_IMPORT_MIN_GAP_SECONDS;
+ return Math.max(platformMinGapSeconds(platform), table[platform] ?? 0);
+}
+
+/**
* @param {Platform | null | undefined} platform
* @returns {string[]}
*/
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,7 @@
# Changelog
## [Unreleased]
+- **One-off imports from Odysee and BitChute are spaced, as a batch download is.** Each import is its own job, so nothing used to space them: imports queued on Odysee went out as fast as each finished, and Odysee answered 429. An imported Odysee or BitChute URL now waits at least 60 s after the last import from the same platform, plus up to half again at random; the job's log says "Odysee asks for a gap between videos: waiting Ns." An Odysee import, as a BitChute one already did, runs on that platform's own queue whatever the channel's platform, is refused while the platform is held or in a rate-limit cooldown, waits out a cooldown that began after it was queued, and records what the platform answered, so a 429 backs every path on that platform off. Batch downloads and syncs of an Odysee channel keep the operator's gap. The wait is held in memory, and a restart clears it. Needs a restart of the editor.
- **A Wayback Machine capture is a copy, and says of what.** A capture URL (`web.archive.org/web/<timestamp>[id_|im_|…]/<original>`) names its record by what it is a capture of: an archived YouTube page by its YouTube id (no longer `watch`), a JW Player file by its media id (no longer `<id>-<rendition>.mp4`). Every download of a capture writes `wayback.json` (the original URL, the capture's timestamp, the capture page and its raw bytes); the video page says "Archived copy (Wayback Machine, <date>) of <original>"; a citation links the original, marked as possibly gone, and the Wayback copy, and its moment link is the capture, which plays (a capture URL never takes a time param). An existing record whose page is a capture is renamed to its id by the next snapshot.
- **`archilyzer wayback refresh <slug> [--titles <file>] [--dry-run]`** brings a channel's Wayback copies up to that offline: `wayback.json`, the dir renamed through the snapshot's own reconcile pass with its roster entry moved, and with `--titles` (`id → {title, upload_date}`) the title and date of a raw file that has none, recorded in the metadata history as `wayback-provenance`. A record a live job holds is skipped and named; a second run changes nothing.
- **A video's captions are read from its original-audio track first, and a track with no text never hides one that has it.** Where YouTube serves both, `transcript.en-orig.vtt` (the captions of the original audio) is read before `transcript.en.vtt`, whose text can be a rewrite of what was said; then regional tracks (`en-US`, `en-GB`, …), then auto-translated `en-en-*` ones. The transcript is the first track in that order that has cues. One rule (`englishVttsByPreference` / `readEnglishVttCues` in `common/lib/videoStatus.ts`) serves the index, normalize and report compose, so search, the export, the MCP and report videos read the same words. **Set as transcript** on a video's page copies the chosen track to `transcript.en.vtt` and pins it there with `transcript-pin.json`, which ranks it first; deleting `transcript-pin.json` returns the video to the automatic pick. The served `en` track is listed as an alternate subtitle track where `en-orig` is the transcript. Videos with a human-made `en` track are not counted as auto-captions-only, so the replace-auto-captions lane still leaves them alone.
diff --git a/editor/app/channels/[slug]/pipelineActions.ts b/editor/app/channels/[slug]/pipelineActions.ts
@@ -30,6 +30,13 @@ import {
recordDownloadBackoff,
recordPlatformClean,
} from "yt-dlp-transcript-common/jobs/downloadBackoff";
+import { downloadGapMs } from "yt-dlp-transcript-common/jobs/platformBackoff";
+import {
+ notePlatformGap,
+ platformGapRemainingMs,
+ waitForPlatformGap,
+} from "yt-dlp-transcript-common/jobs/platformGap";
+import { platformImportMinGapSeconds } from "yt-dlp-transcript-common/ytdlp/channelArgs";
import {
countNotYetDownloaded,
readChannelConfig,
@@ -531,6 +538,15 @@ export async function importVideoAction(
// again when on disk — and what BitChute answers is recorded on its pacing
// state below.
const bitchute = urlPlatform === "bitchute";
+ // ANY URL ON A PLATFORM WITH AN IMPORT FLOOR (platformImportMinGapSeconds:
+ // BitChute, Odysee) is paced though each import is its own job: it runs
+ // on the platform's own queue with the platform's args, is refused while the
+ // platform is held or cooling down, waits out the floor after the last import
+ // there (jobs/platformGap.ts) and any cooldown that began since it was
+ // queued, and records what the platform answered.
+ const pacedPlatform =
+ urlPlatform && platformImportMinGapSeconds(urlPlatform) > 0 ? urlPlatform : null;
+ const pacedLabel = pacedPlatform ? (PACED_LABELS[pacedPlatform] ?? pacedPlatform) : "";
let downloadConfig = channelConfig;
if (archiveOrg) {
const refused = await archiveOrgRefusal(paths, "The import");
@@ -555,9 +571,14 @@ export async function importVideoAction(
downloadConfig = { ...channelConfig, platform: "archiveorg" };
}
}
- if (bitchute) {
- const refused = await platformRefusal(paths, "bitchute", "BitChute", "The import");
+ if (pacedPlatform) {
+ const refused = await platformRefusal(paths, pacedPlatform, pacedLabel, "The import");
if (refused) return { ok: false, info: true, error: refused };
+ if (channelConfig.platform !== pacedPlatform) {
+ downloadConfig = { ...channelConfig, platform: pacedPlatform };
+ }
+ }
+ if (bitchute) {
const resolved = resolveBitchuteImportUrl(videoUrl);
if (!resolved.ok) return { ok: false, error: resolved.error };
videoUrl = resolved.url;
@@ -574,9 +595,6 @@ export async function importVideoAction(
error: `Already downloaded: data/${resolved.id}/ — BitChute is not asked for it again.`,
};
}
- if (channelConfig.platform !== "bitchute") {
- downloadConfig = { ...channelConfig, platform: "bitchute" };
- }
}
// Best-effort canonical id: used only for revalidation/labels. When null,
// downloadOneManaged falls back to %(id)s and the reconcile pass repairs the
@@ -589,7 +607,7 @@ export async function importVideoAction(
// queue control starts at the CHANNEL's queue, which is therefore read as
// "no choice made"; any other queue the operator picks still wins.
const channelQueue = downloadQueueKey(channelConfig);
- const ownQueue = archiveOrg || bitchute ? platformQueueKey(urlPlatform) : null;
+ const ownQueue = archiveOrg || pacedPlatform ? platformQueueKey(urlPlatform) : null;
const override =
ownQueue && queueKey !== undefined && queueKey.trim() === channelQueue
? undefined
@@ -607,6 +625,18 @@ export async function importVideoAction(
kind: "download",
});
try {
+ if (pacedPlatform) {
+ await waitForPlatformGap({
+ label: pacedLabel,
+ remainingMs: async () =>
+ Math.max(
+ platformGapRemainingMs(pacedPlatform),
+ await platformCooldownRemainingMs(pacedPlatform, paths).catch(() => 0),
+ ),
+ signal,
+ onLog: task.onLog,
+ });
+ }
const outcome = await downloadOneManaged({
channelSlug: slug,
channelConfig: downloadConfig,
@@ -619,16 +649,23 @@ export async function importVideoAction(
globalSkipLiveDownloads: settings.skipLiveDownloads,
appendArchive: true,
});
- if (bitchute) {
- // BitChute's shared pacing state learns what it said: a 429 backs
- // every BitChute path off (the lane, a sync, the next import).
- // Best-effort, as the batch downloads' bookkeeping is.
+ if (pacedPlatform) {
+ // The platform's shared pacing state learns what it said: a 429
+ // backs every path on it off (the lane, a sync, the next import).
+ // Best-effort, as the batch downloads' bookkeeping is. The next
+ // import there waits the floor from now, whatever the answer.
+ notePlatformGap(
+ pacedPlatform,
+ downloadGapMs(settings.sleepBetweenDownloadsSeconds, 0, 0, {
+ minSeconds: platformImportMinGapSeconds(pacedPlatform),
+ }),
+ );
const answer = importPlatformSignal(outcome);
try {
if (answer === "rate_limit" || answer === "network") {
- await recordDownloadBackoff("bitchute", paths, answer);
+ await recordDownloadBackoff(pacedPlatform, paths, answer);
} else if (answer === "clean") {
- const line = await recordPlatformClean("bitchute", paths);
+ const line = await recordPlatformClean(pacedPlatform, paths);
if (line) task.onLog(line);
}
} catch {
@@ -660,6 +697,9 @@ export async function importVideoAction(
});
}
+// How an import names a paced platform in what it says.
+const PACED_LABELS: Partial<Record<string, string>> = { bitchute: "BitChute", odysee: "Odysee" };
+
// A platform held, or in a rate-limit cooldown: a sentence, else null. An
// import asks a platform nothing while it has asked us to wait.
async function platformRefusal(
