commit 04b16332096449e83ef60ca3718a7af44f061fa4
parent e610eca55b821e29641f56e3e9d2222154cc310e
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sun, 31 May 2026 21:30:01 -0400
Show full yt-dlp logs; drive progress bars from a progress template
Managed downloads capture the per-video archive marker with yt-dlp's
--print, which implies --quiet -- so they ran nearly silent: download.log
held little more than the archive line, and the per-video progress bars
on /jobs/active never advanced because the [download] lines their parser
needs were suppressed.
Re-enable full output (--no-quiet) on all four managed invocations and
emit a structured --progress-template line (throttled to ~1/sec) that the
parser reads directly via the new parseStructuredDownloadProgress(),
instead of scraping yt-dlp's human progress text. The legacy [download]
regex parser is kept as a fallback for non-managed single-video/subs-only
downloads and the e2e fake.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Diffstat:
3 files changed, 109 insertions(+), 0 deletions(-)
diff --git a/common/jobs/progressParsers.ts b/common/jobs/progressParsers.ts
@@ -18,11 +18,98 @@ function formatClock(totalSeconds: number): string {
return hh > 0 ? `${hh}:${pad(mm)}:${pad(ss)}` : `${mm}:${pad(ss)}`;
}
+// Bytes/sec → yt-dlp-style human rate, e.g. 4533988 → "4.33MiB/s".
+function formatBytesPerSec(bytesPerSec: number): string {
+ const units = ["B", "KiB", "MiB", "GiB", "TiB"];
+ let v = bytesPerSec;
+ let i = 0;
+ while (v >= 1024 && i < units.length - 1) {
+ v /= 1024;
+ i += 1;
+ }
+ const digits = v >= 100 || i === 0 ? 0 : 2;
+ return `${v.toFixed(digits)}${units[i]}/s`;
+}
+
+// Marker + template for yt-dlp's --progress-template. We emit a structured,
+// numeric line (rather than scrape the human "[download] …%" line) because the
+// fields are far easier and more robust to parse — see parseStructuredDownloadProgress.
+// The marker is deliberately distinct from downloadOneManaged's DLOM_ARCHIVE so
+// the archive-line parser never confuses the two.
+export const DOWNLOAD_PROGRESS_MARKER = "DLOM_PROGRESS";
+export const DOWNLOAD_PROGRESS_TEMPLATE =
+ `download:${DOWNLOAD_PROGRESS_MARKER}` +
+ " status=%(progress.status)s" +
+ " downloaded=%(progress.downloaded_bytes)s" +
+ " total=%(progress.total_bytes,progress.total_bytes_estimate)s" +
+ " frag=%(progress.fragment_index)s/%(progress.fragment_count)s" +
+ " speed=%(progress.speed)s" +
+ " eta=%(progress.eta)s";
+
+// yt-dlp renders unavailable template fields as the literal "NA". Treat that
+// (and empty / non-finite) as "unknown".
+function num(value: string | undefined): number | undefined {
+ if (value === undefined || value === "" || value === "NA") return undefined;
+ const n = Number.parseFloat(value);
+ return Number.isFinite(n) ? n : undefined;
+}
+
+// Structured progress line emitted by DOWNLOAD_PROGRESS_TEMPLATE, e.g.
+// DLOM_PROGRESS status=downloading downloaded=31744 total=117526 frag=NA/NA speed=7519285.37 eta=0
+// Returns null for any other line so the legacy parser can have a turn.
+export function parseStructuredDownloadProgress(
+ line: string,
+): ProgressUpdate | null {
+ if (!line.includes(DOWNLOAD_PROGRESS_MARKER)) return null;
+ const field = (key: string): string | undefined =>
+ line.match(new RegExp(`\\b${key}=(\\S+)`))?.[1];
+
+ const status = field("status");
+ const downloaded = num(field("downloaded"));
+ const total = num(field("total"));
+ const speed = num(field("speed"));
+ const eta = num(field("eta"));
+ const fragStr = field("frag"); // e.g. "3/6" or "NA/NA"
+ const [fragIndex, fragCount] = (fragStr ?? "").split("/").map(num);
+
+ let fraction: number | undefined;
+ if (status === "finished") {
+ fraction = 1;
+ } else if (total !== undefined && total > 0 && downloaded !== undefined) {
+ fraction = clamp01(downloaded / total);
+ } else if (
+ fragIndex !== undefined &&
+ fragCount !== undefined &&
+ fragCount > 0
+ ) {
+ fraction = clamp01(fragIndex / fragCount);
+ }
+
+ const detailParts: string[] = [];
+ if (speed !== undefined) detailParts.push(formatBytesPerSec(speed));
+ if (eta !== undefined) detailParts.push(`ETA ${formatClock(eta)}`);
+ if (fragIndex !== undefined && fragCount !== undefined) {
+ detailParts.push(`frag ${fragIndex}/${fragCount}`);
+ }
+
+ if (fraction === undefined && detailParts.length === 0) return null;
+ const out: ProgressUpdate = {};
+ if (fraction !== undefined) out.fraction = fraction;
+ if (detailParts.length > 0) out.detail = detailParts.join(" · ");
+ return out;
+}
+
// yt-dlp download progress, e.g.
// [download] 0.1% of ~ 1.04GiB at 2.31MiB/s ETA Unknown (frag 3/2599)
// [download] 100% of 123.95KiB in 00:00:01
// Returns null for non-progress lines (Destination:, Downloading item, …).
export function parseDownloadProgress(line: string): ProgressUpdate | null {
+ // Managed downloads emit a structured --progress-template line; prefer it.
+ const structured = parseStructuredDownloadProgress(line);
+ if (structured) return structured;
+
+ // Legacy fallback: scrape yt-dlp's human "[download] …%" line. Still used by
+ // non-managed paths (single-video / subs-only) and the e2e fake.
if (!line.includes("[download]")) return null;
const pctMatch = line.match(/\[download\]\s+([\d.]+)%/);
const fragMatch = line.match(/\(frag\s+(\d+)\/(\d+)\)/);
diff --git a/common/ytdlp/downloadOneManaged.ts b/common/ytdlp/downloadOneManaged.ts
@@ -21,10 +21,27 @@ import type { Paths } from "../lib/paths";
import { transcribeOneVideo } from "../controller/transcribeOne";
import { extractVideoId, outputArgsForUrl } from "./runYtdlp";
import { runAudioCheckedYtdlp } from "./audioCheckedDownload";
+import { DOWNLOAD_PROGRESS_TEMPLATE } from "../jobs/progressParsers";
const STDERR_TAIL_BYTES = 64 * 1024;
const ARCHIVE_MARKER = "DLOM_ARCHIVE";
+// The `--print` archive marker below implies `--quiet`, which otherwise
+// suppresses every extraction log and the [download] progress lines. Re-enable
+// full output and emit a structured, throttled progress line we can parse
+// directly into the per-video progress bars (human log readability is secondary
+// to reliable progress parsing). --progress-delta keeps it to ~1 line/sec,
+// matching the /jobs/active poll.
+const FULL_LOG_PROGRESS_ARGS = [
+ "--no-quiet",
+ "--progress",
+ "--newline",
+ "--progress-delta",
+ "1",
+ "--progress-template",
+ DOWNLOAD_PROGRESS_TEMPLATE,
+];
+
export type ManagedDownloadOpts = {
channelSlug: string;
channelConfig: ChannelConfig;
@@ -293,6 +310,7 @@ async function runManagedDownload(
const primaryArgs = [
"--ignore-config",
"--restrict-filenames",
+ ...FULL_LOG_PROGRESS_ARGS,
...outputArgsForUrl(opts.videoUrl),
...transcribeHandlingArgsForAudioCheck(opts.channelConfig),
"--print",
@@ -336,6 +354,7 @@ async function runManagedDownload(
const primaryArgs = [
"--ignore-config",
"--restrict-filenames",
+ ...FULL_LOG_PROGRESS_ARGS,
...outputArgsForUrl(opts.videoUrl),
...(opts.channelConfig.handling === "youtube"
? youtubeHandlingArgs(opts.channelConfig)
@@ -388,6 +407,7 @@ async function runManagedDownload(
const retryArgs = [
"--ignore-config",
"--restrict-filenames",
+ ...FULL_LOG_PROGRESS_ARGS,
...outputArgsForUrl(opts.videoUrl),
...(opts.channelConfig.handling === "youtube"
? youtubeHandlingArgs(opts.channelConfig)
@@ -454,6 +474,7 @@ async function runManagedDownload(
const fallbackArgs = [
"--ignore-config",
"--restrict-filenames",
+ ...FULL_LOG_PROGRESS_ARGS,
...outputArgsForUrl(opts.videoUrl),
...transcribeHandlingArgs(fallbackConfig),
"--print",
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,7 @@
# Changelog
## [Unreleased]
+- **Managed downloads stream yt-dlp's full output again, and progress bars now read a structured progress template.** The per-video archive marker is captured with yt-dlp's `--print`, which silently implies `--quiet` — so managed downloads ran nearly silent: `download.log` held little more than the archive line, and the per-video progress bars on `/jobs/active` never advanced (the `[download]` lines they parsed were suppressed). Managed downloads now re-enable full logging (`--no-quiet`) and emit a machine-readable `--progress-template` line (throttled to ~1/sec) that the progress parser reads directly instead of scraping the human progress text — so the bars advance reliably during real downloads (with speed/ETA detail) and `download.log` captures the whole run. The legacy `[download]` line parser is kept as a fallback for non-managed single-video/subs-only downloads.
- **One global site selector scopes the editor to a single site.** The sidebar gained a single **site** dropdown (with an **All sites** option) that scopes the **Dashboard**, **Channels**, **Charts**, and **Deploy** views to the chosen site: Dashboard stats and the channel table, and the Channels list, now show only that site's member channels; Charts edits that site's dashboard; and Build static export / Deploy target it. Views that act on the shared pool — **Jobs**, **Active**, **Build**, and **Actionable** — always show every site regardless of the selector. The nav is regrouped to match: a **Site** group first, then **Pool**, then **Manage**. The selection persists in `localStorage` and is mirrored into the `?site=` URL param. Under "All sites", Charts and Deploy ask you to pick a specific site; creating a channel while a specific site is active also adds it to that site's membership. This replaces the per-page site tabs on Charts and the per-button site dropdowns on Deploy.
- **Date-range filter on the public site's search.** The exported site's search filters gained a **Date** row (From / To pickers) that restricts results to videos uploaded within an inclusive range. The range saves in profiles and shareable search links and carries over to "Chart this search", reusing the same date mechanism the charts dashboard already uses. No editor-chrome change; it ships in every built site.
- **Bulk checkbox transcribe/retry now queue like every other batch.** Selecting videos in the channel's list and clicking **Transcribe** (or **Retry download**) used to fire one queued single-video job per selection onto the channel's *platform* queue — diverging from "Transcribe missing"/"Download", which submit one batch job on the shared `transcription`/platform queue. The checkbox actions now submit a **single** batch job through the same path (so bulk and the stage buttons serialize together instead of contending), and the resulting job shows up in the page's running-jobs list. The selection bar gained the matching controls: a **queue** selector for each action (defaulting to `transcription` for transcribe and the platform queue for retry), a **Parallel** concurrency input for transcribe, and an **Abort on error** toggle for retry. Default-queue resolution for all batch features now lives in one shared helper (`common/lib/queueKeys.ts`) so they can't drift apart again. "Mark untranscribable" is unchanged (instant metadata write).
