commit 558efa98ab0e8bf9e986662dd84839d89b155896
parent 3d6eaab7d85126f7811b8d5792b932b5878d25cf
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 17 Jul 2026 18:40:30 -0400
Audio-check: adaptive (AIMD) probe interval
The audio-integrity probe ran on a fixed cadence (default 60s) for the whole
download, so up to ~60s of bytes were downloaded and discarded between a
corruption event and the checkpoint that caught it.
Make the interval adaptive (AIMD, inverted TCP congestion control): each
malformed checkpoint halves the live interval down to the existing 10s floor,
so a misbehaving source is probed more aggressively and wastes fewer bytes per
rollback; a run of clean checkpoints then steps it back up additively toward
the configured interval. The reduced cadence persists across yt-dlp relaunches
for the rest of the download run. A clean download never leaves the configured
interval, so the change is inert on the happy path.
The AIMD math lives in a pure, unit-tested module (audioCheckCadence.ts);
audioCheckedDownload.ts owns the run-scoped state, logging, and setTimeout, and
stamps the live interval onto each CheckpointRecord for observability. Tunable
via constants in channelConfig.ts plus test-only env overrides.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Diffstat:
5 files changed, 281 insertions(+), 6 deletions(-)
diff --git a/common/lib/channelConfig.ts b/common/lib/channelConfig.ts
@@ -118,6 +118,14 @@ export const AUDIO_CHECK_INTERVAL_DEFAULT_SECONDS = 60;
export const AUDIO_CHECK_INTERVAL_MIN_SECONDS = 10;
export const AUDIO_CHECK_INTERVAL_MAX_SECONDS = 600;
+// Adaptive (AIMD) probe cadence. Each malformed checkpoint multiplies the live
+// interval by this factor (fast tightening) down to AUDIO_CHECK_INTERVAL_MIN_SECONDS
+// as the floor; each run of clean checkpoints steps it back up additively toward
+// the configured interval (slow relaxation). See runAudioCheckedYtdlp.
+export const AUDIO_CHECK_INTERVAL_BACKOFF_FACTOR_DEFAULT = 0.5;
+export const AUDIO_CHECK_INTERVAL_RECOVER_STEP_SECONDS = 15;
+export const AUDIO_CHECK_INTERVAL_RECOVER_AFTER_CLEAN = 2;
+
export const AUDIO_CHECK_MAX_ROLLBACKS_DEFAULT = 5;
export const AUDIO_CHECK_MAX_ROLLBACKS_MIN = 1;
export const AUDIO_CHECK_MAX_ROLLBACKS_MAX = 20;
diff --git a/common/ytdlp/audioCheckCadence.test.ts b/common/ytdlp/audioCheckCadence.test.ts
@@ -0,0 +1,112 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ backoffInterval,
+ formatInterval,
+ recoverInterval,
+ type CadenceParams,
+} from "./audioCheckCadence";
+
+// Mirrors the production defaults: 60s ceiling, 10s floor, halve, +15s recovery
+// after 2 clean checkpoints.
+const P: CadenceParams = {
+ ceilingMs: 60_000,
+ floorMs: 10_000,
+ factor: 0.5,
+ recoverStepMs: 15_000,
+ recoverAfterClean: 2,
+};
+
+test("backoff halves the interval toward the floor and stops there", () => {
+ assert.equal(backoffInterval(60_000, P), 30_000);
+ assert.equal(backoffInterval(30_000, P), 15_000);
+ // 15000 * 0.5 = 7500, clamped up to the 10000 floor.
+ assert.equal(backoffInterval(15_000, P), 10_000);
+ // Already at the floor: no further shrink.
+ assert.equal(backoffInterval(10_000, P), 10_000);
+});
+
+test("backoff is a no-op for a degenerate factor (>= 1)", () => {
+ assert.equal(backoffInterval(60_000, { ...P, factor: 1 }), 60_000);
+ assert.equal(backoffInterval(60_000, { ...P, factor: 1.5 }), 60_000);
+});
+
+test("backoff rounds to whole milliseconds", () => {
+ // 2001 * 0.5 = 1000.5 -> 1001; well above a 200ms floor.
+ assert.equal(backoffInterval(2001, { ...P, floorMs: 200 }), 1001);
+});
+
+test("recovery needs a clean streak before stepping up", () => {
+ // Start reduced at 15s. First clean: streak 1 < 2, no step, streak carried.
+ let r = recoverInterval(15_000, 0, P);
+ assert.deepEqual(r, { intervalMs: 15_000, cleanStreak: 1 });
+ // Second clean: streak reaches 2 -> +15s and streak resets.
+ r = recoverInterval(15_000, 1, P);
+ assert.deepEqual(r, { intervalMs: 30_000, cleanStreak: 0 });
+});
+
+test("recovery caps at the configured ceiling and is a no-op there", () => {
+ // 50s + 15s step would be 65s, capped to the 60s ceiling.
+ assert.deepEqual(recoverInterval(50_000, 1, P), {
+ intervalMs: 60_000,
+ cleanStreak: 0,
+ });
+ // Already at the ceiling: streak still counts but interval is unchanged.
+ assert.deepEqual(recoverInterval(60_000, 1, P), {
+ intervalMs: 60_000,
+ cleanStreak: 2,
+ });
+});
+
+test("AIMD is asymmetric: fast multiplicative decrease, slow additive increase", () => {
+ // Three malformed checkpoints drive 60 -> 30 -> 15 -> 10 (floor).
+ let interval = 60_000;
+ interval = backoffInterval(interval, P);
+ interval = backoffInterval(interval, P);
+ interval = backoffInterval(interval, P);
+ assert.equal(interval, 10_000);
+
+ // Recovery from the floor climbs additively, one +15s step per 2 cleans, and
+ // never overshoots the ceiling.
+ let streak = 0;
+ const steps: number[] = [];
+ for (let i = 0; i < 12; i += 1) {
+ const r = recoverInterval(interval, streak, P);
+ interval = r.intervalMs;
+ streak = r.cleanStreak;
+ steps.push(interval);
+ }
+ // 10 -> (clean,clean)25 -> (clean,clean)40 -> 55 -> 60 (capped), then steady.
+ assert.deepEqual(steps, [
+ 10_000, 25_000, 25_000, 40_000, 40_000, 55_000, 55_000, 60_000, 60_000,
+ 60_000, 60_000, 60_000,
+ ]);
+});
+
+test("a malformed checkpoint mid-recovery re-tightens immediately", () => {
+ // Recovered partway to 40s, then a malformed check halves straight back down.
+ assert.equal(backoffInterval(40_000, P), 20_000);
+});
+
+test("test-scale params (300ms interval, 50ms floor) still shrink", () => {
+ const T: CadenceParams = {
+ ceilingMs: 300,
+ floorMs: 50,
+ factor: 0.5,
+ recoverStepMs: 100,
+ recoverAfterClean: 2,
+ };
+ assert.equal(backoffInterval(300, T), 150);
+ assert.equal(backoffInterval(150, T), 75);
+ // 75 * 0.5 = 37.5, clamped up to the 50ms floor.
+ assert.equal(backoffInterval(75, T), 50);
+ assert.equal(backoffInterval(50, T), 50);
+});
+
+test("formatInterval prints seconds at/above 1s and milliseconds below", () => {
+ assert.equal(formatInterval(60_000), "60s");
+ assert.equal(formatInterval(10_000), "10s");
+ assert.equal(formatInterval(1_000), "1s");
+ assert.equal(formatInterval(300), "300ms");
+ assert.equal(formatInterval(50), "50ms");
+});
diff --git a/common/ytdlp/audioCheckCadence.ts b/common/ytdlp/audioCheckCadence.ts
@@ -0,0 +1,60 @@
+// Pure AIMD (additive-increase / multiplicative-decrease) math for the
+// audio-check probe cadence. Kept free of IO/logging so the curve is unit
+// testable in isolation; runAudioCheckedYtdlp owns the surrounding state,
+// logging, and the actual setTimeout.
+//
+// The live interval starts at the configured value (the ceiling). A malformed
+// checkpoint multiplies it down toward `floorMs` (fast tightening); a run of
+// clean checkpoints steps it back up toward the ceiling (slow relaxation). This
+// mirrors TCP congestion control, inverted for our purpose: probe harder the
+// moment a source misbehaves, ease off only once it has proven stable again.
+
+export type CadenceParams = {
+ // Configured interval — the ceiling and starting point; recovery never
+ // exceeds it.
+ ceilingMs: number;
+ // Lower bound the multiplicative decrease clamps to.
+ floorMs: number;
+ // Multiplicative-decrease factor, expected in (0, 1]. A factor >= 1 (or an
+ // interval already at the floor) makes backoff a no-op.
+ factor: number;
+ // Additive-increase step applied on recovery.
+ recoverStepMs: number;
+ // Consecutive clean checkpoints required before each recovery step.
+ recoverAfterClean: number;
+};
+
+// Multiplicative decrease on a malformed checkpoint. Returns the new interval,
+// clamped to the floor; returns `currentMs` unchanged when there's no room to
+// shrink (already at/below the floor, or a degenerate factor).
+export function backoffInterval(currentMs: number, p: CadenceParams): number {
+ if (!(p.factor < 1) || currentMs <= p.floorMs) return currentMs;
+ return Math.max(p.floorMs, Math.round(currentMs * p.factor));
+}
+
+// Additive increase driven by a clean checkpoint. `cleanStreakBefore` is the
+// streak prior to this checkpoint; the checkpoint itself counts as one more.
+// Once the streak reaches `recoverAfterClean` the interval steps up by
+// `recoverStepMs` (capped at the ceiling) and the streak resets; otherwise the
+// interval is unchanged and the incremented streak carries forward.
+export function recoverInterval(
+ currentMs: number,
+ cleanStreakBefore: number,
+ p: CadenceParams,
+): { intervalMs: number; cleanStreak: number } {
+ const streak = cleanStreakBefore + 1;
+ if (streak < p.recoverAfterClean || currentMs >= p.ceilingMs) {
+ return { intervalMs: currentMs, cleanStreak: streak };
+ }
+ return {
+ intervalMs: Math.min(p.ceilingMs, currentMs + p.recoverStepMs),
+ cleanStreak: 0,
+ };
+}
+
+// Human-readable interval for log lines. Sub-second (test) intervals print as
+// milliseconds so they stay legible; whole-second production intervals print as
+// seconds.
+export function formatInterval(ms: number): string {
+ return ms >= 1000 ? `${Math.round(ms / 1000)}s` : `${ms}ms`;
+}
diff --git a/common/ytdlp/audioCheckedDownload.ts b/common/ytdlp/audioCheckedDownload.ts
@@ -30,13 +30,23 @@ import path from "node:path";
import { execa } from "execa";
import {
AUDIO_CHECK_COPY_TIMEOUT_DEFAULT_SECONDS,
+ AUDIO_CHECK_INTERVAL_BACKOFF_FACTOR_DEFAULT,
AUDIO_CHECK_INTERVAL_DEFAULT_SECONDS,
+ AUDIO_CHECK_INTERVAL_MIN_SECONDS,
+ AUDIO_CHECK_INTERVAL_RECOVER_AFTER_CLEAN,
+ AUDIO_CHECK_INTERVAL_RECOVER_STEP_SECONDS,
AUDIO_CHECK_MAX_ROLLBACKS_DEFAULT,
type AudioFormat,
type ChannelConfig,
} from "../lib/channelConfig";
import type { Paths } from "../lib/paths";
import { transcodeAudio } from "../controller/transcode";
+import {
+ backoffInterval,
+ formatInterval,
+ recoverInterval,
+ type CadenceParams,
+} from "./audioCheckCadence";
import type { StreamVerdict } from "./ffmpegStreamClassify";
import { probeAudioStream } from "./ffmpegStreamProbe";
import { DLOM_PROBE_MARKER } from "../jobs/progressParsers";
@@ -58,6 +68,15 @@ function envIntOverride(name: string): number | null {
return Number.isFinite(n) && n >= 0 ? n : null;
}
+// Float env override for the backoff factor. Only accepts a value in (0, 1] —
+// a factor >= 1 would never shrink the interval, and <= 0 is nonsensical.
+function envFloatOverride(name: string): number | null {
+ const raw = process.env[name];
+ if (!raw) return null;
+ const n = Number.parseFloat(raw);
+ return Number.isFinite(n) && n > 0 && n <= 1 ? n : null;
+}
+
// Tri-state boolean env override: unset -> null (use config), "1"/"true" ->
// true, anything else -> false. Lets comparison runs flip the resume-during-
// probe behavior without editing channel config.
@@ -84,6 +103,12 @@ export type CheckpointRecord = {
// checkpoints where a real probe ran (advance / rollback / restart /
// skip-copy-timeout); undefined for size-gate / missing-part skips.
durationMs?: number;
+ // The live adaptive-cadence interval (ms) that was in effect when this
+ // checkpoint fired — i.e. the wait before it, and the value the malformed /
+ // clean decision then adjusts. Stamped on advance / rollback / restart so the
+ // AIMD curve is observable (and deterministically assertable) from the
+ // outcome record. Undefined on skip records that don't touch the cadence.
+ intervalMs?: number;
};
export type AudioCheckOutcomeKind =
@@ -137,11 +162,21 @@ export type AudioCheckedOpts = {
// Resolved knobs.
type Knobs = {
+ // The configured probe interval — the AIMD ceiling and starting point. The
+ // live cadence (currentIntervalMs in runAudioCheckedYtdlp) shrinks below this
+ // on malformed checkpoints and relaxes back up toward it, never past it.
intervalMs: number;
maxRollbacks: number;
copyTimeoutMs: number;
debugPauseMs: number;
sizeGateBytes: number;
+ // Adaptive-cadence knobs (AIMD). backoffFactor multiplies the live interval on
+ // each malformed checkpoint (down to backoffFloorMs); after recoverAfterClean
+ // consecutive clean checkpoints the interval steps back up by recoverStepMs.
+ backoffFactor: number;
+ backoffFloorMs: number;
+ recoverStepMs: number;
+ recoverAfterClean: number;
// Legacy behavior when true: SIGCONT immediately after the snapshot copy
// and probe concurrently. Default false: hold the child stopped across the
// probe so no would-be-discarded bytes are downloaded.
@@ -156,16 +191,35 @@ function resolveKnobs(opts: AudioCheckedOpts): Knobs {
const resumeDuringProbeOverride = envBoolOverride(
"AUDIO_CHECK_RESUME_DURING_PROBE",
);
+ const backoffFactorOverride = envFloatOverride("AUDIO_CHECK_BACKOFF_FACTOR");
+ const floorMsOverride = envIntOverride("AUDIO_CHECK_INTERVAL_FLOOR_MS_OVERRIDE");
+ const recoverStepMsOverride = envIntOverride("AUDIO_CHECK_RECOVER_STEP_MS_OVERRIDE");
+ const recoverAfterOverride = envIntOverride("AUDIO_CHECK_RECOVER_AFTER_OVERRIDE");
+ const intervalMs =
+ intervalMsOverride ??
+ (cfg?.intervalSeconds ?? AUDIO_CHECK_INTERVAL_DEFAULT_SECONDS) * 1000;
+ // Floor defaults to the configured MIN, but never exceed the starting
+ // interval itself — otherwise a small (test) interval would have no room to
+ // shrink, and the very first backoff would clamp *up*.
+ const backoffFloorMs = Math.min(
+ floorMsOverride ?? AUDIO_CHECK_INTERVAL_MIN_SECONDS * 1000,
+ intervalMs,
+ );
return {
- intervalMs:
- intervalMsOverride ??
- (cfg?.intervalSeconds ?? AUDIO_CHECK_INTERVAL_DEFAULT_SECONDS) * 1000,
+ intervalMs,
maxRollbacks: cfg?.maxRollbacks ?? AUDIO_CHECK_MAX_ROLLBACKS_DEFAULT,
copyTimeoutMs:
(cfg?.copyTimeoutSeconds ?? AUDIO_CHECK_COPY_TIMEOUT_DEFAULT_SECONDS) *
1000,
debugPauseMs: debugPauseOverride ?? opts.debugPauseMs ?? 0,
sizeGateBytes: sizeGateOverride ?? DEFAULT_SIZE_GATE_BYTES,
+ backoffFactor:
+ backoffFactorOverride ?? AUDIO_CHECK_INTERVAL_BACKOFF_FACTOR_DEFAULT,
+ backoffFloorMs,
+ recoverStepMs:
+ recoverStepMsOverride ?? AUDIO_CHECK_INTERVAL_RECOVER_STEP_SECONDS * 1000,
+ recoverAfterClean:
+ recoverAfterOverride ?? AUDIO_CHECK_INTERVAL_RECOVER_AFTER_CLEAN,
resumeDuringProbe:
resumeDuringProbeOverride ?? cfg?.resumeDuringProbe ?? false,
};
@@ -567,6 +621,19 @@ export async function runAudioCheckedYtdlp(
let rollbacks = 0;
let restarts = 0;
let consecutiveRollbacks = 0;
+ // Adaptive (AIMD) probe cadence, run-scoped so it persists across yt-dlp
+ // relaunches: a malformed checkpoint multiplies it down toward knobs.backoffFloorMs,
+ // and recoverAfterClean consecutive clean checkpoints step it back up toward
+ // the configured knobs.intervalMs. cleanStreak counts the clean run driving recovery.
+ let currentIntervalMs = knobs.intervalMs;
+ let cleanStreak = 0;
+ const cadence: CadenceParams = {
+ ceilingMs: knobs.intervalMs,
+ floorMs: knobs.backoffFloorMs,
+ factor: knobs.backoffFactor,
+ recoverStepMs: knobs.recoverStepMs,
+ recoverAfterClean: knobs.recoverAfterClean,
+ };
// Counts malformed verdicts on the FINAL probe (a complete yt-dlp exit-0
// download), as opposed to mid-download checkpoint rollbacks. Re-downloading
// a complete file yields identical bytes and thus the same verdict, so we
@@ -867,7 +934,7 @@ export async function runAudioCheckedYtdlp(
// the "probing" phase, carrying the estimated duration (last probe plus
// its observed increase). finishProbe/skipProbe close the phase on every
// exit path below; durationMs from a real probe feeds the parser's trend.
- const intervalSec = Math.round(knobs.intervalMs / 1000);
+ const intervalSec = Math.round(currentIntervalMs / 1000);
const probeStartedAt = Date.now();
const estMs =
lastProbeMs > 0 ? lastProbeMs + Math.max(0, lastProbeMs - prevProbeMs) : 0;
@@ -983,10 +1050,23 @@ export async function runAudioCheckedYtdlp(
verdict: probe.verdict,
action: "advance",
durationMs: Date.now() - probeStartedAt,
+ intervalMs: currentIntervalMs,
});
opts.onLog(
`Checkpoint OK at ${snap.bytes} bytes (verdict=${probe.verdict}).\n`,
);
+ // AIMD additive increase: after a run of clean checkpoints, relax the
+ // cadence one step back toward the configured interval (never past it).
+ {
+ const rec = recoverInterval(currentIntervalMs, cleanStreak, cadence);
+ if (rec.intervalMs > currentIntervalMs) {
+ opts.onLog(
+ `Audio-check interval recovering: ${formatInterval(currentIntervalMs)} -> ${formatInterval(rec.intervalMs)} after clean checkpoints\n`,
+ );
+ }
+ currentIntervalMs = rec.intervalMs;
+ cleanStreak = rec.cleanStreak;
+ }
finishProbe(false);
return;
}
@@ -1001,10 +1081,24 @@ export async function runAudioCheckedYtdlp(
verdict: probe.verdict,
action: pendingDecision,
durationMs: Date.now() - probeStartedAt,
+ intervalMs: currentIntervalMs,
});
opts.onLog(
`Checkpoint MALFORMED at ${snap.bytes} bytes. ${haveGood ? "Rolling back to .good." : "No .good baseline; restarting from 0."}\n`,
);
+ // AIMD multiplicative decrease: a malformed source gets probed more
+ // aggressively for the rest of the run (persists across relaunches),
+ // down to the floor. Resets the clean streak driving recovery.
+ cleanStreak = 0;
+ {
+ const next = backoffInterval(currentIntervalMs, cadence);
+ if (next < currentIntervalMs) {
+ opts.onLog(
+ `Audio-check interval backoff: ${formatInterval(currentIntervalMs)} -> ${formatInterval(next)} after malformed checkpoint\n`,
+ );
+ currentIntervalMs = next;
+ }
+ }
// reset=true: the rollback/restart shrinks the .part, so the next
// probe is cheaper — the parser must drop its duration trend.
finishProbe(true);
@@ -1058,9 +1152,9 @@ export async function runAudioCheckedYtdlp(
}
if (!partFile || watcherStop || childExited) return;
while (!watcherStop && !childExited && !opts.signal.aborted) {
- // Wait the interval (cancellable on abort/exit).
+ // Wait the (adaptive) interval (cancellable on abort/exit).
await new Promise<void>((resolve) => {
- const t = setTimeout(resolve, knobs.intervalMs);
+ const t = setTimeout(resolve, currentIntervalMs);
const onAbort = () => {
clearTimeout(t);
resolve();
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,7 @@
# Changelog
## [Unreleased]
+- **The audio-integrity check now tightens its probe interval when a source starts serving corruption, then relaxes as it stabilises.** Previously the integrity probe ran on a fixed cadence (default 60s) for the whole download, so up to ~60s of bytes were downloaded — and discarded — between a corruption event and the checkpoint that caught it. The interval is now adaptive (AIMD, like TCP congestion control, inverted): each **malformed** checkpoint **halves** the live interval (60→30→15→10s, floored at the existing `AUDIO_CHECK_INTERVAL_MIN_SECONDS` of 10s), so a misbehaving source gets probed more aggressively and wastes fewer bytes per rollback; a run of clean checkpoints then **steps it back up** additively (+15s after every 2 clean probes) toward the configured interval. The reduced cadence persists across yt-dlp relaunches for the rest of the download run. Fully backward compatible — a clean download never leaves the configured interval. Tunable via constants in `common/lib/channelConfig.ts` (`AUDIO_CHECK_INTERVAL_BACKOFF_FACTOR_DEFAULT`, `AUDIO_CHECK_INTERVAL_RECOVER_STEP_SECONDS`, `AUDIO_CHECK_INTERVAL_RECOVER_AFTER_CLEAN`) plus test-only env overrides. See `common/ytdlp/audioCheckCadence.ts` (pure AIMD math + `audioCheckCadence.test.ts`) and `common/ytdlp/audioCheckedDownload.ts` (`resolveKnobs`, the watcher loop, and the advance/malformed checkpoint branches).
- **Docker build mode is now real: build every site in parallel, then deploy them serially.** The `Docker` build mode (Settings → Build pipeline) was previously a stub that fell back to the basic build. It now runs a proper pipeline, driven by a new **Build all sites** control on the Deploy page (one job, one log, one Cancel). The shared, corpus-scale work — the search index, the per-site staging, and the downloadable archive zips — runs **once on the host**; then each site's `compose + next build` runs in its **own container in parallel** (capped by the **Max parallel builds** setting), each writing an isolated per-site `out/` under `export/.export-builds/<siteId>/`; then the built sites **deploy serially** on the host (R2 upload + `wrangler pages deploy`), tolerant of a single site failing. Containers are read-only over the shared corpus/index/archive cache and run as your host user so outputs aren't root-owned. The image (`Dockerfile.build`, tag from **Build image**) is built/reused via Docker layer caching; when no container engine is available the action falls back to a serial host build+deploy. New env knobs: `DOCKER_BIN` (e.g. `podman`), `DOCKER_BUILD_MEMORY`/`DOCKER_BUILD_CPUS` (per-container caps). See `editor/app/deploy/buildDeployCore.ts` (`runDockerBuildAllPhase`/`runDockerDeployAllPhase`), `editor/app/build/buildAction.ts` (`buildAndDeployAllSitesAction`/`buildAllSitesAction`), `editor/app/deploy/components/BuildAllSitesButton.tsx`, `Dockerfile.build`, `docker/build-site.sh`, `common/bin/build-archives.ts`, and **[DEPLOY_DOCKER.md](../DEPLOY_DOCKER.md)**.
## [0.7.3] - 2026-07-07
