commit 0c1a529882eb6bdb7037de9f8d1b346a2f019169 parent 5a2b1667e8d3ba644ef5b166f0d4208dd56a6f1c Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st> Date: Sun, 4 Oct 2026 16:17:50 -0400 Merge video-720-preset (video_720 download preset: ≤720p H.264 with 480 and any-codec rungs; sourceVideoQuality setting global + per channel; persists honour it and record the format they took) Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> # Conflicts: # editor/CHANGELOG.md Diffstat:
30 files changed, 939 insertions(+), 49 deletions(-)
diff --git a/CHANNEL.md b/CHANNEL.md @@ -23,6 +23,7 @@ Regenerate this file with `pnpm --filter yt-dlp-transcript-common exec tsx bin/f | `url` | config | The channel / playlist / account URL syncs enumerate. Absent = the channel is never auto-synced. | | `audioFormat` | config | `"m4a"`, `"mp3"` or `"opus"`: the audio a transcribe-handling download keeps. | | `downloadFormat` | config | Per-channel override for the yt-dlp `-f` download format preset. Absent = inherit the global `downloadFormat`, which itself falls back to the per-source "auto" selector. Lets a channel whose source serves full-length audio only in its `original` format (e.g. Odysee) force it. | +| `sourceVideoQuality` | config | Per-channel override for the quality of the source container a full persist keeps ("Persist source video", the whole-recording fetch, "Persist kept now"). `"original"` (best video + audio) or `"video_720"` (≤720p H.264, for clips/editing). Absent = inherit the global `sourceVideoQuality`. | | `keepSourceVideo` | config | Keep the downloaded source video beside the audio. | | `keepLatest` | config | Keep-latest window: the newest N videos (by upload date) are protected from the Clean-audio sweep AND have their source video persisted to the saved-video store. 0 or absent = disabled; positives clamp to [1, 100000]. A kept video later found deleted at the source is pinned permanently via the do-not-clean marker. | | `extractionMode` | config | `"ytdlp"` (default — yt-dlp's own `-x --audio-format` postprocessor, no source container kept) or `"app"` (yt-dlp downloads the source container and the app runs ffmpeg). The keep-latest persistence rule forces `"app"` for the videos it persists. | diff --git a/SETTINGS.md b/SETTINGS.md @@ -23,6 +23,7 @@ A copied example PINS every default it spells — including each lane's `autoQue | [`sleepBetweenDownloadsSeconds`](#sleepbetweendownloadsseconds) | `10` | | [`pacing`](#pacing) | object — see below | | [`downloadFormat`](#downloadformat) | `"auto"` | +| [`sourceVideoQuality`](#sourcevideoquality) | `"original"` | | [`minFreeDiskGB`](#minfreediskgb) | `5` | | [`resumeMarginGB`](#resumemargingb) | `2` | | [`parallelTranscriptions`](#paralleltranscriptions) | `2` | @@ -220,6 +221,12 @@ Default yt-dlp `-f` download format for every channel that doesn't set its own ( Default: `"auto"` +## `sourceVideoQuality` + +Quality of the source container a full persist keeps — "Persist source video", the whole-recording fetch (`full: true`) and "Persist kept now" — for every channel that doesn't set its own (ChannelConfig.sourceVideoQuality). "original" (default) = `bestvideo*+bestaudio/best`; "video_720" = ≤720p H.264/AAC mp4 for clip and editing work, falling back to 480p and then to anything (logged). A single persist can override it. See common/ytdlp/downloadFormat.ts. + +Default: `"original"` + ## `minFreeDiskGB` Minimum free disk space (GB) required on the transcripts data directory for downloads to run. When free space is below this floor, a download job is prevented from starting and a running batch stops launching new videos (the in-flight one finishes). 0 disables the gate. See common/lib/diskSpace.ts. diff --git a/common/controller/persistKept.ts b/common/controller/persistKept.ts @@ -11,6 +11,10 @@ import { downloadOneManaged, sourceFetchFailure, } from "../ytdlp/downloadOneManaged"; +import { + resolveSourceVideoQuality, + type SourceVideoQuality, +} from "../ytdlp/downloadFormat"; // Bulk "persist kept now" pass (Phase 5). Ensures every video currently in the // channel's keep-latest window has its source container saved to the store, for @@ -48,12 +52,16 @@ export async function persistKept({ channelConfig, onLog, signal, + quality, }: { paths: Paths; channelSlug: string; channelConfig: ChannelConfig; onLog?: (line: string) => void; signal?: AbortSignal; + // Per-run override of the source-video quality; absent = the channel's, + // else the global setting (resolveSourceVideoQuality). + quality?: SourceVideoQuality; }): Promise<PersistKeptResult> { const log = (line: string) => onLog?.(line.endsWith("\n") ? line : `${line}\n`); // downloadOneManaged requires a non-optional onLog/signal; supply inert @@ -83,6 +91,12 @@ export async function persistKept({ return result; } const settings = getSettings(); + const persistFormatPreset = resolveSourceVideoQuality({ + override: quality, + channel: channelConfig.sourceVideoQuality, + global: settings.sourceVideoQuality, + }); + log(`Persist kept: source video quality ${persistFormatPreset}.`); const dataDir = path.join(paths.channelsDir, channelSlug, "data"); // Newest-first so the freshest videos are secured even if the pass is cancelled // partway through. computeKeptVideoIds returns an unordered set; sort by id desc @@ -138,6 +152,7 @@ export async function persistKept({ // pass is otherwise skipped for any video with a transcript or // captions, which is every kept one (release 10 slice N). forceMedia: true, + persistFormatPreset, }); // A download that RETURNED is not a persisted source (release 11 slice // O3, review low 2): a failed forced media pass leaves the download `ok` diff --git a/common/lib/channelConfig.ts b/common/lib/channelConfig.ts @@ -1,7 +1,9 @@ import { PLATFORM_VALUES, type Platform } from "./platform"; import { isDownloadFormatPreset, + isSourceVideoQuality, type DownloadFormatPreset, + type SourceVideoQuality, } from "../ytdlp/downloadFormat"; import { isCookieMode, type CookieMode } from "./cookiePolicy"; import type { FieldDocs } from "./fieldDocs"; @@ -121,6 +123,7 @@ export type ChannelConfig = { url?: string; audioFormat?: AudioFormat; downloadFormat?: DownloadFormatPreset; + sourceVideoQuality?: SourceVideoQuality; keepSourceVideo?: boolean; keepLatest?: number; extractionMode?: ExtractionMode; @@ -160,6 +163,8 @@ export const CHANNEL_CONFIG_FIELD_DOCS: FieldDocs<ChannelConfig> = { audioFormat: '`"m4a"`, `"mp3"` or `"opus"`: the audio a transcribe-handling download keeps.', downloadFormat: "Per-channel override for the yt-dlp `-f` download format preset. Absent = inherit the global `downloadFormat`, which itself falls back to the per-source \"auto\" selector. Lets a channel whose source serves full-length audio only in its `original` format (e.g. Odysee) force it.", + sourceVideoQuality: + 'Per-channel override for the quality of the source container a full persist keeps ("Persist source video", the whole-recording fetch, "Persist kept now"). `"original"` (best video + audio) or `"video_720"` (≤720p H.264, for clips/editing). Absent = inherit the global `sourceVideoQuality`.', keepSourceVideo: "Keep the downloaded source video beside the audio.", keepLatest: "Keep-latest window: the newest N videos (by upload date) are protected from the Clean-audio sweep AND have their source video persisted to the saved-video store. 0 or absent = disabled; positives clamp to [1, 100000]. A kept video later found deleted at the source is pinned permanently via the do-not-clean marker.", @@ -366,6 +371,7 @@ export const CHANNEL_CONFIG_COERCIONS: { url: str, audioFormat: (v) => (v === "m4a" || v === "mp3" || v === "opus" ? v : undefined), downloadFormat: (v) => (isDownloadFormatPreset(v) ? v : undefined), + sourceVideoQuality: (v) => (isSourceVideoQuality(v) ? v : undefined), keepSourceVideo: bool, keepLatest: zeroOrClamped(1, KEEP_LATEST_MAX), extractionMode: (v) => (v === "ytdlp" || v === "app" ? v : undefined), diff --git a/common/lib/channelConfigSchema.test.ts b/common/lib/channelConfigSchema.test.ts @@ -39,7 +39,7 @@ test("one key list: docs = coercions = schema shape, sync-state keys inside it", assert.deepEqual(Object.keys(CHANNEL_CONFIG_COERCIONS), [...CHANNEL_CONFIG_KEYS]); assert.deepEqual(Object.keys(channelConfigObjectSchema.shape), [...CHANNEL_CONFIG_KEYS]); assert.deepEqual(Object.keys(CHANNEL_CONFIG_FIELD_DOCS), [...CHANNEL_CONFIG_KEYS]); - assert.equal(CHANNEL_CONFIG_KEYS.length, 30); + assert.equal(CHANNEL_CONFIG_KEYS.length, 31); assert.equal(sameKeys, true); assert.equal(fits, true); for (const k of CHANNEL_SYNC_STATE_KEYS) assert.ok(CHANNEL_CONFIG_KEYS.includes(k), k); @@ -69,6 +69,17 @@ test("an invalid optional key is OMITTED, not defaulted", () => { } }); +test("sourceVideoQuality: the two qualities parse, anything else is omitted", () => { + for (const q of ["original", "video_720"] as const) { + assert.equal(parse({ handling: "youtube", sourceVideoQuality: q })!.sourceVideoQuality, q); + assert.equal(parseChannelConfig({ handling: "youtube", sourceVideoQuality: q })!.sourceVideoQuality, q); + } + for (const junk of ["720", "bestvideo_audio", "", 720, null]) { + const cfg = parse({ handling: "youtube", sourceVideoQuality: junk })!; + assert.equal("sourceVideoQuality" in cfg, false, JSON.stringify(junk)); + } +}); + test("clamps: 0 is preserved as the disabled sentinel, positives clamp, fractions floor", () => { const cfg = parse({ handling: "transcribe", @@ -142,6 +153,7 @@ test("the zod schema and the client-safe parser agree, key order included", () = handling: ["youtube", "transcribe"], platform: ["youtube", "rumble", "nope"], cookieMode: ["always", "never", "nope"], + sourceVideoQuality: ["original", "video_720", "nope"], }; let seed = 7; const rand = () => ((seed = (seed * 1103515245 + 12345) % 2 ** 31) / 2 ** 31); diff --git a/common/lib/channelConfigSchema.ts b/common/lib/channelConfigSchema.ts @@ -52,6 +52,7 @@ export const channelConfigObjectSchema = z.object({ url: field("url"), audioFormat: field("audioFormat"), downloadFormat: field("downloadFormat"), + sourceVideoQuality: field("sourceVideoQuality"), keepSourceVideo: field("keepSourceVideo"), keepLatest: field("keepLatest"), extractionMode: field("extractionMode"), diff --git a/common/lib/savedVideo-server.ts b/common/lib/savedVideo-server.ts @@ -11,6 +11,7 @@ import { SAVED_VIDEO_POINTER_FILENAME, parseSavedVideoPointer, savedVideoPath, + type SavedVideoFormat, type SavedVideoKeepReason, type SavedVideoOrigin, type SavedVideoPointer, @@ -107,6 +108,8 @@ export async function persistSourceVideo(opts: { // SavedVideoOrigin — it records the requester WITHOUT changing keepReason, // which is what keeps the container out of the retention prune. origin?: SavedVideoOrigin; + // What the persist asked for and what yt-dlp took; see SavedVideoPointer.format. + format?: SavedVideoFormat; // WHEN THE STORE IS BEING MOVED, THIS DOES NOT RUN. Passed by the download // path, which has them; a caller that omits them opts out, which is right for // a store that is not the corpus one. See lib/savedVideoStore.ts for the @@ -130,6 +133,7 @@ export async function persistSourceVideo(opts: { bytes: st.size, ...(opts.keepReason ? { keepReason: opts.keepReason } : {}), ...(opts.origin ? { origin: opts.origin } : {}), + ...(opts.format ? { format: opts.format } : {}), }; await writePointer(opts.videoDir, pointer); return pointer; diff --git a/common/lib/savedVideo.test.ts b/common/lib/savedVideo.test.ts @@ -7,6 +7,7 @@ import type { Paths } from "./paths"; import { parseSavedVideoPointer, savedVideoDir, + savedVideoFormatFromLine, savedVideoRoot, } from "./savedVideo"; import { @@ -80,6 +81,68 @@ test("parseSavedVideoPointer rejects malformed and reads keepReason", () => { assert.equal(noReason?.keepReason, undefined); }); +test("format: an old pointer parses as before, a new one carries its format", () => { + const base = { storedAt: "t", dir: "/a", file: "source-media.mp4", bytes: 10 }; + // Every pointer written before the field: no `format` key at all. + const old = parseSavedVideoPointer(base); + assert.deepEqual(old, base); + assert.equal(old && "format" in old, false); + // The new shape round-trips. + assert.deepEqual( + parseSavedVideoPointer({ + ...base, + format: { preset: "video_720", height: 720, vcodec: "avc1.4d401f" }, + })?.format, + { preset: "video_720", height: 720, vcodec: "avc1.4d401f" }, + ); + // Height and vcodec are optional; junk in them is dropped, not carried. + assert.deepEqual( + parseSavedVideoPointer({ + ...base, + format: { preset: "original", height: "720", vcodec: "" }, + })?.format, + { preset: "original" }, + ); + // A format with no valid preset is dropped whole; the pointer still parses. + const bad = parseSavedVideoPointer({ ...base, format: { preset: "4k", height: 2160 } }); + assert.ok(bad); + assert.equal(bad && "format" in bad, false); +}); + +test("savedVideoFormatFromLine reads the printed format line", () => { + assert.deepEqual(savedVideoFormatFromLine("720 avc1.4d401f 136+140", "video_720"), { + preset: "video_720", + height: 720, + vcodec: "avc1.4d401f", + }); + // yt-dlp prints NA for a field it does not have. + assert.deepEqual(savedVideoFormatFromLine("NA NA original", "original"), { + preset: "original", + }); + // No line: the asked-for preset alone. + assert.deepEqual(savedVideoFormatFromLine(null, "video_720"), { preset: "video_720" }); + assert.deepEqual(savedVideoFormatFromLine("", "original"), { preset: "original" }); +}); + +test("persist writes the format it is given onto the pointer", async () => { + await withTmp(async (root) => { + const videoDir = path.join(root, "data", "vid1"); + const storeDir = path.join(root, "store", "chan", "vid1"); + await mkdir(videoDir, { recursive: true }); + await writeFile(path.join(videoDir, "source-media.mp4"), "video-bytes"); + const format = { preset: "video_720" as const, height: 480, vcodec: "avc1.4d401e" }; + const pointer = await persistSourceVideo({ + videoDir, + sourceFilename: "source-media.mp4", + storeDir, + keepReason: "override", + format, + }); + assert.deepEqual(pointer.format, format); + assert.deepEqual((await loadSavedVideo(videoDir))?.format, format); + }); +}); + test("persist moves the container into the store and writes a pointer", async () => { await withTmp(async (root) => { const videoDir = path.join(root, "data", "vid1"); diff --git a/common/lib/savedVideo.ts b/common/lib/savedVideo.ts @@ -1,6 +1,10 @@ import path from "node:path"; import type { Paths } from "./paths"; import type { ChannelConfig } from "./channelConfig"; +import { + isSourceVideoQuality, + type SourceVideoQuality, +} from "../ytdlp/downloadFormat"; // The saved-video store (Phase 3 of the video-persistence feature). // @@ -50,8 +54,51 @@ export type SavedVideoPointer = { // somebody asked for. Absent on every pointer written before this and on // every automatic persist. origin?: SavedVideoOrigin; + // WHAT WAS DOWNLOADED: the quality the persist asked for and the format + // yt-dlp actually took (its `requested_downloads[0]`, printed at download + // time). A "video_720" persist can fall through to its last-resort rung, so + // the asked-for preset alone does not say how tall the file is. Absent on + // every pointer written before this; `height`/`vcodec` absent when yt-dlp + // did not report them. + format?: SavedVideoFormat; }; +export type SavedVideoFormat = { + preset: SourceVideoQuality; + height?: number; + vcodec?: string; +}; + +function parseFormat(raw: unknown): SavedVideoFormat | null { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null; + const r = raw as Record<string, unknown>; + if (!isSourceVideoQuality(r.preset)) return null; + const out: SavedVideoFormat = { preset: r.preset }; + if (typeof r.height === "number" && Number.isInteger(r.height) && r.height > 0) { + out.height = r.height; + } + if (typeof r.vcodec === "string" && r.vcodec !== "") out.vcodec = r.vcodec; + return out; +} + +// The format a persist pass printed (runOneYtdlp's FORMAT_MARKER line: +// "<height> <vcodec> <format_id>", yt-dlp's "NA" for a missing field), as the +// pointer's `format`. With no line, the asked-for preset alone. +export function savedVideoFormatFromLine( + line: string | null | undefined, + preset: SourceVideoQuality, +): SavedVideoFormat { + const out: SavedVideoFormat = { preset }; + if (!line) return out; + const [heightRaw, vcodecRaw] = line.trim().split(/\s+/); + const height = Number(heightRaw); + if (Number.isInteger(height) && height > 0) out.height = height; + if (vcodecRaw && vcodecRaw !== "NA" && vcodecRaw !== "none") { + out.vcodec = vcodecRaw; + } + return out; +} + // The requester of a manually-sourced container: the tool, and the report clip // it was wanted for. Deliberately the same vocabulary as ClipProvenance // (lib/clipWindow.ts) so the video page renders both the same way. @@ -123,5 +170,8 @@ export function parseSavedVideoPointer(raw: unknown): SavedVideoPointer | null { // TOLERANT: a legacy pointer has no `origin` and must parse exactly as it did. const origin = parseOrigin(r.origin); if (origin) pointer.origin = origin; + // Same rule: a pointer without `format` (every one before it) parses as before. + const format = parseFormat(r.format); + if (format) pointer.format = format; return pointer; } diff --git a/common/lib/settingsSchema.test.ts b/common/lib/settingsSchema.test.ts @@ -24,7 +24,10 @@ import type { Worker } from "./workers"; import type { AutoQueueSettings } from "./autoQueueTypes"; import type { ChannelPriority } from "./channelPriority"; import type { CookieMode } from "./cookiePolicy"; -import type { DownloadFormatPreset } from "../ytdlp/downloadFormat"; +import type { + DownloadFormatPreset, + SourceVideoQuality, +} from "../ytdlp/downloadFormat"; import type { StorageSettings } from "./storageLocations"; import type { AttributionSettings, @@ -70,6 +73,8 @@ type PreSchemaSiteSettings = { // Release 17 slice RL — the adaptive pace and the hold. pacing: PacingSettingsBlock; downloadFormat: DownloadFormatPreset; + // The source-video quality a full persist keeps. + sourceVideoQuality: SourceVideoQuality; minFreeDiskGB: number; resumeMarginGB: number; parallelTranscriptions: number; @@ -98,7 +103,7 @@ type PreSchemaSiteSettings = { type Same<A, B> = [A] extends [B] ? ([B] extends [A] ? true : false) : false; const shapeUnchanged: Same<SiteSettings, PreSchemaSiteSettings> = true; -test("SiteSettings keeps its 33 fields, in file order", () => { +test("SiteSettings keeps its 34 fields, in file order", () => { assert.equal(shapeUnchanged, true); assert.deepEqual(Object.keys(siteSettingsSchema.shape), [ "adminTitle", @@ -112,6 +117,7 @@ test("SiteSettings keeps its 33 fields, in file order", () => { "sleepBetweenDownloadsSeconds", "pacing", "downloadFormat", + "sourceVideoQuality", "minFreeDiskGB", "resumeMarginGB", "parallelTranscriptions", @@ -338,12 +344,19 @@ test("enum fields fall back to their default on junk", () => { const s = siteSettingsSchema.parse({ cookieMode: "sometimes", downloadFormat: "best", + sourceVideoQuality: "1080p", reportDebouncePreset: "instant", transcriptionApp: "no-such-app", }); const d = defaults(); assert.equal(s.cookieMode, d.cookieMode); assert.equal(s.downloadFormat, "auto"); + assert.equal(s.sourceVideoQuality, "original"); + assert.equal(d.sourceVideoQuality, "original"); + assert.equal( + siteSettingsSchema.parse({ sourceVideoQuality: "video_720" }).sourceVideoQuality, + "video_720", + ); assert.equal(s.reportDebouncePreset, d.reportDebouncePreset); assert.equal(s.transcriptionApp, d.transcriptionApp); }); diff --git a/common/lib/settingsSchema.ts b/common/lib/settingsSchema.ts @@ -44,8 +44,11 @@ import { } from "./settingsFieldSchemas"; import { SYNC_INTERVAL_MAX_MINUTES } from "./channelConfig"; import { + DEFAULT_SOURCE_VIDEO_QUALITY, isDownloadFormatPreset, + isSourceVideoQuality, type DownloadFormatPreset, + type SourceVideoQuality, } from "../ytdlp/downloadFormat"; import { type AppInstanceConfig, @@ -1566,6 +1569,10 @@ export const siteSettingsSchema = z.object({ isDownloadFormatPreset(v) ? v : "auto").describe( "Default yt-dlp `-f` download format for every channel that doesn't set its own (ChannelConfig.downloadFormat). \"auto\" picks per-source: `original` for Odysee (whose HLS rungs are CDN-truncated), `bestaudio/worst` elsewhere. See common/ytdlp/downloadFormat.ts.", ), + sourceVideoQuality: settingsField((v): SourceVideoQuality => + isSourceVideoQuality(v) ? v : DEFAULT_SOURCE_VIDEO_QUALITY).describe( + "Quality of the source container a full persist keeps — \"Persist source video\", the whole-recording fetch (`full: true`) and \"Persist kept now\" — for every channel that doesn't set its own (ChannelConfig.sourceVideoQuality). \"original\" (default) = `bestvideo*+bestaudio/best`; \"video_720\" = ≤720p H.264/AAC mp4 for clip and editing work, falling back to 480p and then to anything (logged). A single persist can override it. See common/ytdlp/downloadFormat.ts.", + ), minFreeDiskGB: settingsField((v): number => clampMinFreeDiskGB(v)).describe( "Minimum free disk space (GB) required on the transcripts data directory for downloads to run. When free space is below this floor, a download job is prevented from starting and a running batch stops launching new videos (the in-flight one finishes). 0 disables the gate. See common/lib/diskSpace.ts.", ), diff --git a/common/ytdlp/downloadFormat.test.ts b/common/ytdlp/downloadFormat.test.ts @@ -0,0 +1,94 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + DOWNLOAD_FORMAT_LABELS, + DOWNLOAD_FORMAT_PRESETS, + ORIGINAL_SOURCE_FORMAT_SELECTOR, + SOURCE_VIDEO_QUALITIES, + VIDEO_720_PRESET, + clipFormatSelector, + isDownloadFormatPreset, + isSourceVideoQuality, + resolveDownloadFormatSelector, + resolveSourceVideoQuality, + sourceVideoFormatSelector, +} from "./downloadFormat"; +import { clipFormatSelector as reexported } from "./fetchWindowManaged"; + +// Run with: +// pnpm --filter yt-dlp-transcript-common exec tsx --test ytdlp/downloadFormat.test.ts + +const CLIP_720 = + "bv*[vcodec^=avc1][height<=720]+ba[acodec^=mp4a]/" + + "bv*[ext=mp4][height<=720]+ba[ext=m4a]/" + + "b[ext=mp4][height<=720]/" + + "b[height<=720]"; + +test("clipFormatSelector is the selector umtool uses, and the old import path still serves it", () => { + assert.equal(clipFormatSelector(720), CLIP_720); + assert.equal(reexported, clipFormatSelector); +}); + +test("video_720: split H.264 at 720, then 480, then any codec split ≤720, then single-file ≤720, then the last resort", () => { + assert.equal(VIDEO_720_PRESET, "video_720"); + assert.ok(isDownloadFormatPreset("video_720")); + assert.ok(DOWNLOAD_FORMAT_PRESETS.includes("video_720")); + assert.equal(DOWNLOAD_FORMAT_LABELS.video_720, "Video 720p (H.264, for clips/editing)"); + const expected = [ + "bv*[vcodec^=avc1][height<=720]+ba[acodec^=mp4a]", + "bv*[ext=mp4][height<=720]+ba[ext=m4a]", + "bv*[vcodec^=avc1][height<=480]+ba[acodec^=mp4a]", + "bv*[ext=mp4][height<=480]+ba[ext=m4a]", + "bv*[height<=720]+ba", + "b[ext=mp4][height<=720]", + "b[height<=720]", + "bv*+ba/b", + ].join("/"); + assert.equal(resolveDownloadFormatSelector("video_720", null), expected); + // An explicit preset ignores the platform, as every explicit preset does. + assert.equal(resolveDownloadFormatSelector("video_720", "odysee"), expected); +}); + +test("the existing presets resolve exactly as before", () => { + assert.equal(resolveDownloadFormatSelector("auto", "youtube"), "bestaudio/worst"); + assert.equal(resolveDownloadFormatSelector("auto", "odysee"), "original/bestaudio/worst"); + assert.equal(resolveDownloadFormatSelector("original", null), "original/bestaudio/worst"); + assert.equal(resolveDownloadFormatSelector("bestaudio", null), "bestaudio/worst"); + assert.equal(resolveDownloadFormatSelector("bestvideo_audio", null), "bestvideo*+bestaudio/best"); +}); + +test("sourceVideoFormatSelector: original is today's persist selector, byte for byte", () => { + assert.deepEqual([...SOURCE_VIDEO_QUALITIES], ["original", "video_720"]); + assert.equal(ORIGINAL_SOURCE_FORMAT_SELECTOR, "bestvideo*+bestaudio/best"); + assert.equal(sourceVideoFormatSelector("original"), "bestvideo*+bestaudio/best"); + assert.equal( + sourceVideoFormatSelector("video_720"), + resolveDownloadFormatSelector("video_720", null), + ); +}); + +test("resolveSourceVideoQuality: override beats channel beats global; original when unset", () => { + assert.equal(resolveSourceVideoQuality({}), "original"); + assert.equal(resolveSourceVideoQuality({ global: "video_720" }), "video_720"); + // The channel wins over the global, either way round. + assert.equal( + resolveSourceVideoQuality({ channel: "original", global: "video_720" }), + "original", + ); + assert.equal( + resolveSourceVideoQuality({ channel: "video_720", global: "original" }), + "video_720", + ); + assert.equal( + resolveSourceVideoQuality({ override: "original", channel: "video_720", global: "video_720" }), + "original", + ); + // Junk at any level falls through to the next, never reaching yt-dlp. + assert.equal( + resolveSourceVideoQuality({ override: "1080p", channel: undefined, global: "video_720" }), + "video_720", + ); + assert.equal(resolveSourceVideoQuality({ override: "", channel: 720 }), "original"); + assert.ok(isSourceVideoQuality("video_720")); + assert.ok(!isSourceVideoQuality("bestvideo_audio")); +}); diff --git a/common/ytdlp/downloadFormat.ts b/common/ytdlp/downloadFormat.ts @@ -9,13 +9,20 @@ export type DownloadFormatPreset = | "auto" | "original" | "bestaudio" - | "bestvideo_audio"; + | "bestvideo_audio" + | "video_720"; + +// The clip/editing preset's id, named once for every caller that asks for it +// by name (the persist control, the source-video quality setting, the batch +// persist and the clip fetch). +export const VIDEO_720_PRESET = "video_720" as const; export const DOWNLOAD_FORMAT_PRESETS: ReadonlyArray<DownloadFormatPreset> = [ "auto", "original", "bestaudio", "bestvideo_audio", + VIDEO_720_PRESET, ]; export function isDownloadFormatPreset( @@ -33,8 +40,64 @@ export const DOWNLOAD_FORMAT_LABELS: Record<DownloadFormatPreset, string> = { original: "Original (full file)", bestaudio: "Best audio only", bestvideo_audio: "Best video + audio", + video_720: "Video 720p (H.264, for clips/editing)", }; +// THE FORMAT SELECTOR A CLIP IS CUT FROM, and why it is not "bestvideo_audio". +// +// "bestvideo_audio" answers "what should we ARCHIVE" — best available, +// container-agnostic. This answers "what can ffmpeg cut and re-encode cheaply, +// right now". Those differ on one specific trap: left alone yt-dlp picks +// VP9+Opus at these heights, and since --force-keyframes-at-cuts re-encodes, +// that means libvpx-vp9 — 27 s to cut a 5 s clip, measured — and it writes +// .webm, which it then appends to the -o name. So H.264/AAC in mp4 is pinned, +// with three progressively looser fallbacks. Byte-identical to the selector +// umtool's build-video.mjs uses, on purpose: a window this fetches and a window +// that fetched must be the same file, or the two caches diverge. +// +// Lives here (not in fetchWindowManaged.ts, which re-exports it) because the +// "video_720" preset below is built from its rungs, and a persist must not +// import the clip-window fetch to learn a selector. +export function clipFormatSelector(maxHeight: number): string { + return [...clipSplitRungs(maxHeight), ...clipSingleFileRungs(maxHeight)].join( + "/", + ); +} + +// The clip selector's two halves. SPLIT: separate H.264 video + AAC audio +// streams, merged. SINGLE-FILE: one file carrying both (mp4 first). Split +// first, because a single-file format at these heights is often a low rung +// (YouTube's 360p mp4). +function clipSplitRungs(maxHeight: number): string[] { + return [ + `bv*[vcodec^=avc1][height<=${maxHeight}]+ba[acodec^=mp4a]`, + `bv*[ext=mp4][height<=${maxHeight}]+ba[ext=m4a]`, + ]; +} + +function clipSingleFileRungs(maxHeight: number): string[] { + return [`b[ext=mp4][height<=${maxHeight}]`, `b[height<=${maxHeight}]`]; +} + +// The height "video_720" asks for, and the rung it falls back to. +export const VIDEO_720_MAX_HEIGHT = 720; +export const VIDEO_720_FALLBACK_HEIGHT = 480; + +// The codec-agnostic split rung of "video_720", after the H.264 split ones: a +// source whose only video at or under 720p is VP9 or AV1 still lands at or +// under 720p, at its best such height — not on a 360p single-file mp4, and not +// on the best (possibly 4K) file. +export const VIDEO_720_ANY_CODEC_RUNG = `bv*[height<=${VIDEO_720_MAX_HEIGHT}]+ba`; + +// The last rung of "video_720": anything at all. Reached only when the source +// has nothing at or under 720p, so a file above 720p is possible — the +// download logs it (downloadOneManaged, persistedFormatLog) rather than +// letting it pass silently. +export const LAST_RESORT_FORMAT_SELECTOR = "bv*+ba/b"; + +// The source container a persist has always taken: best video + best audio. +export const ORIGINAL_SOURCE_FORMAT_SELECTOR = "bestvideo*+bestaudio/best"; + // Resolve a preset into the concrete yt-dlp `-f` selector string. The platform // only matters for "auto"; explicit presets apply literally regardless of // source (so an explicit choice always wins over the per-source default). @@ -48,7 +111,19 @@ export function resolveDownloadFormatSelector( case "bestaudio": return "bestaudio/worst"; case "bestvideo_audio": - return "bestvideo*+bestaudio/best"; + return ORIGINAL_SOURCE_FORMAT_SELECTOR; + case "video_720": + // NOT clipFormatSelector(720) whole: its single-file rungs would take a + // 360p mp4 before a 720p VP9 split pair is ever tried. Split H.264 at + // 720, then at 480, then any codec split at or under 720, then the + // single-file rungs, then anything (logged). + return [ + ...clipSplitRungs(VIDEO_720_MAX_HEIGHT), + ...clipSplitRungs(VIDEO_720_FALLBACK_HEIGHT), + VIDEO_720_ANY_CODEC_RUNG, + ...clipSingleFileRungs(VIDEO_720_MAX_HEIGHT), + LAST_RESORT_FORMAT_SELECTOR, + ].join("/"); case "auto": default: return platform === "odysee" @@ -66,3 +141,60 @@ export function resolveDownloadFormatPreset(opts: { }): DownloadFormatPreset { return opts.override ?? opts.channel ?? opts.global ?? "auto"; } + +// ── SOURCE-VIDEO QUALITY ─────────────────────────────────────────────────── +// +// What a FULL-SOURCE persist downloads: "original" (best video + best audio, +// the behaviour every persist had before this setting) or "video_720" (≤720p +// H.264, for clip and editing work). A separate axis from DownloadFormatPreset: +// that one picks what a download fetches to get AUDIO from; this one picks the +// container a persist keeps. Set globally (settings `sourceVideoQuality`), per +// channel (ChannelConfig.sourceVideoQuality) and per persist (the editor's +// "Persist source video" control). +export type SourceVideoQuality = "original" | typeof VIDEO_720_PRESET; + +export const SOURCE_VIDEO_QUALITIES: ReadonlyArray<SourceVideoQuality> = [ + "original", + VIDEO_720_PRESET, +]; + +export const DEFAULT_SOURCE_VIDEO_QUALITY: SourceVideoQuality = "original"; + +export const SOURCE_VIDEO_QUALITY_LABELS: Record<SourceVideoQuality, string> = { + original: "Original (best video + audio)", + video_720: DOWNLOAD_FORMAT_LABELS.video_720, +}; + +export function isSourceVideoQuality( + value: unknown, +): value is SourceVideoQuality { + return ( + typeof value === "string" && + (SOURCE_VIDEO_QUALITIES as ReadonlyArray<string>).includes(value) + ); +} + +// The `-f` selector a persist at this quality passes. "original" is the +// selector a persist has always used, byte for byte. +export function sourceVideoFormatSelector(quality: SourceVideoQuality): string { + return quality === VIDEO_720_PRESET + ? resolveDownloadFormatSelector(VIDEO_720_PRESET, null) + : ORIGINAL_SOURCE_FORMAT_SELECTOR; +} + +// THE ONE EFFECTIVE-QUALITY RESOLVER. Every surface that persists a full +// source — the persist control, the whole-recording fetch, "Persist kept now", +// the batch persist — asks this, so a channel's choice cannot win on one and +// lose on another. Per-persist override beats the channel beats the global; +// "original" when nothing is set. Each input is checked, so a stale or +// hand-typed value falls through to the next rather than reaching yt-dlp. +export function resolveSourceVideoQuality(opts: { + override?: unknown; + channel?: unknown; + global?: unknown; +}): SourceVideoQuality { + for (const v of [opts.override, opts.channel, opts.global]) { + if (isSourceVideoQuality(v)) return v; + } + return DEFAULT_SOURCE_VIDEO_QUALITY; +} diff --git a/common/ytdlp/downloadOneManaged.ts b/common/ytdlp/downloadOneManaged.ts @@ -33,7 +33,12 @@ import { import { isDoNotClean } from "../lib/doNotClean-server"; import { transcodeAudio } from "../controller/transcode"; import { findSourceMedia } from "../lib/videoStatus"; -import { savedVideoDir, type SavedVideoOrigin } from "../lib/savedVideo"; +import { + savedVideoDir, + savedVideoFormatFromLine, + type SavedVideoFormat, + type SavedVideoOrigin, +} from "../lib/savedVideo"; import { persistSourceVideo } from "../lib/savedVideo-server"; import { type AudioCheckAttemptStats, @@ -70,13 +75,20 @@ import { extractVideoId, outputArgsForUrl } from "./runYtdlp"; import { channelExtraArgs, channelPaceSeconds } from "./channelArgs"; import { ARCHIVE_MARKER, + FORMAT_MARKER, runOneYtdlp as runOneYtdlpRaw, type AttemptOutcome, } from "./runOneYtdlp"; import { runAudioCheckedYtdlp } from "./audioCheckedDownload"; import { + DEFAULT_SOURCE_VIDEO_QUALITY, + LAST_RESORT_FORMAT_SELECTOR, + VIDEO_720_MAX_HEIGHT, + VIDEO_720_PRESET, resolveDownloadFormatSelector, + sourceVideoFormatSelector, type DownloadFormatPreset, + type SourceVideoQuality, } from "./downloadFormat"; import { DOWNLOAD_PROGRESS_TEMPLATE } from "../jobs/progressParsers"; @@ -164,6 +176,14 @@ export type ManagedDownloadOpts = { // selector here, where the per-video source platform is known (so "auto" // picks `original` for Odysee). Defaults to "auto" when omitted. downloadFormatPreset?: DownloadFormatPreset; + // The quality of the source container a PERSIST keeps (a pass whose plan + // persists in app mode: "Persist source video", the whole-recording fetch, + // "Persist kept now", a keep-latest download). Resolved by the caller through + // resolveSourceVideoQuality (override > channel > global). Unset or + // "original" = `bestvideo*+bestaudio/best`, as every persist has always been; + // "video_720" = the ≤720p H.264 selector (downloadFormat.ts). Never touches + // the audio-only selector above. + persistFormatPreset?: SourceVideoQuality; }; // When `reuseInfoJson` is true, the real download reuses the metadata the @@ -216,6 +236,7 @@ function audioFormatSelectionArgs( fmt: AudioFormat, config: ChannelConfig, selector: string, + persistSelector: string, ): string[] { if (plan.extractionMode === "ytdlp") { const args = ["-f", selector, "-x", "--audio-format", fmt]; @@ -223,12 +244,48 @@ function audioFormatSelectionArgs( return args; } // Persisting in app mode keeps the full source container (always a complete - // video), so the truncation-prone audio-only selector doesn't apply. + // video), so the truncation-prone audio-only selector doesn't apply: the + // persist quality's selector does (sourceVideoFormatSelector). The format + // yt-dlp actually took is printed for the saved-video pointer. return plan.persist - ? ["-f", "bestvideo*+bestaudio/best"] + ? ["-f", persistSelector, "--print", PERSIST_FORMAT_PRINT] : ["-f", selector]; } +// After the video, the downloaded format: `requested_downloads[0]` is the +// merged (or single) format, carrying the video stream's height and vcodec. +// runOneYtdlp scrapes it as `formatLine`; savedVideoFormatFromLine parses it. +const PERSIST_FORMAT_PRINT = + `after_video:${FORMAT_MARKER} %(requested_downloads.0.height)s ` + + `%(requested_downloads.0.vcodec)s %(requested_downloads.0.format_id)s`; + +// The log line for a persisted container's format, and the LOUD one for a +// "video_720" persist that the last-resort rung answered: a file above 720p +// (or one whose height yt-dlp did not report) is never silent. +export function persistedFormatLog( + format: SavedVideoFormat, + formatLine: string | null | undefined, +): string { + if (!formatLine) { + return `Source format: not reported by yt-dlp (quality ${format.preset}).\n`; + } + const formatId = formatLine?.trim().split(/\s+/)[2]; + const what = + `${format.height ? `${format.height}p` : "unknown height"}` + + `${format.vcodec ? ` ${format.vcodec}` : ""}` + + `${formatId && formatId !== "NA" ? ` (format ${formatId})` : ""}`; + if ( + format.preset === VIDEO_720_PRESET && + (format.height === undefined || format.height > VIDEO_720_MAX_HEIGHT) + ) { + return ( + `Source quality ${VIDEO_720_PRESET}: no format at or under ${VIDEO_720_MAX_HEIGHT}p matched ` + + `any rung; the last resort (${LAST_RESORT_FORMAT_SELECTOR}) took ${what}.\n` + ); + } + return `Source format: ${what} (quality ${format.preset}).\n`; +} + // The media output + info-json + format args for a transcribe-handling download. // In app mode the main output is source-media.<ext> (distinct from audio.<ext> // so it's never treated as cleanable audio); otherwise the historical audio.<ext>. @@ -240,12 +297,13 @@ function transcribeMediaArgs( fmt: AudioFormat, reuseInfoJson: boolean, selector: string, + persistSelector: string, ): string[] { const mediaName = plan.extractionMode === "app" ? "source-media" : "audio"; return [ ...outputArgsForUrl(url, { mediaName }), ...(reuseInfoJson ? [] : ["--write-info-json"]), - ...audioFormatSelectionArgs(plan, fmt, config, selector), + ...audioFormatSelectionArgs(plan, fmt, config, selector, persistSelector), ]; } @@ -276,6 +334,9 @@ async function finalizeAppExtraction(opts: { // `keepTranscript`) wants the source kept, and an audio.<fmt> beside an // existing transcript is bytes nothing will read. Default true. extractAudio?: boolean; + // What the persist asked for and what yt-dlp took (the pass's FORMAT_MARKER + // line), recorded on the pointer. Absent = not reported. + format?: SavedVideoFormat | null; onLog: (s: string) => void; signal: AbortSignal; }): Promise<void> { @@ -333,6 +394,7 @@ async function finalizeAppExtraction(opts: { storeDir, keepReason: opts.category === "none" ? undefined : opts.category, origin: opts.origin, + ...(opts.format ? { format: opts.format } : {}), // The store-in-transition guard. A move of the saved-video store is a // multi-hour copy, and a container landing in the middle of it is lost // three different ways — see lib/savedVideoStore.ts. The catch below is @@ -709,6 +771,8 @@ async function runManagedDownload( // recorded on the outcome so the UI can explain it without re-probing. let shortAudioInfo: NonNullable<DownloadOutcomeRecord["shortAudio"]> | undefined; let lastArchiveLine: string | null = null; + // The format the last media pass printed (FORMAT_MARKER), for the pointer. + let lastFormatLine: string | null = null; // Full (untruncated) stderr tail of the most recent attempt, so a failed // download can be classified (rate_limit/network) against everything yt-dlp // printed — not just the last 3 lines stored on the attempt record. @@ -1051,6 +1115,10 @@ async function runManagedDownload( opts.downloadFormatPreset ?? "auto", resolvedPlatform, ); + // The container a persisting pass keeps. Platform-independent: a persist + // wants a whole video, which "auto"'s Odysee rule is not about. + const persistQuality = opts.persistFormatPreset ?? DEFAULT_SOURCE_VIDEO_QUALITY; + const persistSelector = sourceVideoFormatSelector(persistQuality); const pinned = canonicalId ? await isDoNotClean(path.join(channelDir, "data", canonicalId)) : false; @@ -1086,6 +1154,7 @@ async function runManagedDownload( fmt, reuseInfoJson, downloadFormatSelector, + persistSelector, ); if (opts.channelConfig.handling === "transcribe") { opts.onLog( @@ -1213,6 +1282,7 @@ async function runManagedDownload( ...(audioCheckStats ? { audioCheck: audioCheckStats } : {}), }); if (primaryRes.archiveLine) lastArchiveLine = primaryRes.archiveLine; + if (primaryRes.formatLine) lastFormatLine = primaryRes.formatLine; // ---------- A subtitle 429 is not a failed download (release 17, RL) ---------- // YouTube's timedtext endpoint refuses per video while the media requests @@ -1295,6 +1365,7 @@ async function runManagedDownload( : trimError(noSubsRes.stderrTail), }); if (noSubsRes.archiveLine) lastArchiveLine = noSubsRes.archiveLine; + if (noSubsRes.formatLine) lastFormatLine = noSubsRes.formatLine; primaryRes = noSubsRes; if (!attemptSucceeded(noSubsRes.exitCode)) subsRateLimited = false; } @@ -1365,6 +1436,7 @@ async function runManagedDownload( : trimError(retryRes.stderrTail), }); if (retryRes.archiveLine) lastArchiveLine = retryRes.archiveLine; + if (retryRes.formatLine) lastFormatLine = retryRes.formatLine; if (attemptSucceeded(retryRes.exitCode)) { lastSucceeded = true; status = "ok-with-cookies"; @@ -1376,6 +1448,16 @@ async function runManagedDownload( path.join(channelDir, "data", canonicalId ?? "unknown"); const videoId = path.basename(videoDir); + // The pointer's `format`, read off the media pass that just ran, logged as it + // is recorded (persistedFormatLog). Only a persisting plan keeps a container. + const persistedFormat = (): SavedVideoFormat | null => { + if (!plan.persist) return null; + const format = savedVideoFormatFromLine(lastFormatLine, persistQuality); + const line = persistedFormatLog(format, lastFormatLine); + if (line) opts.onLog(line); + return format; + }; + // Download-time duration guard: probe the produced audio.<fmt> and compare its // actual length to the metadata duration. A large shortfall means the source // served a truncated stream (e.g. a CDN-truncated HLS rung) even though yt-dlp @@ -1435,6 +1517,7 @@ async function runManagedDownload( persist: plan.persist, category: plan.category, origin: opts.persistOrigin, + format: persistedFormat(), onLog: opts.onLog, signal: opts.signal, }); @@ -1522,6 +1605,7 @@ async function runManagedDownload( fmt, true, downloadFormatSelector, + persistSelector, ), "--print", `after_video:${ARCHIVE_MARKER} %(extractor)s %(id)s`, @@ -1554,6 +1638,7 @@ async function runManagedDownload( : trimError(fallbackRes.stderrTail), }); if (fallbackRes.archiveLine) lastArchiveLine = fallbackRes.archiveLine; + if (fallbackRes.formatLine) lastFormatLine = fallbackRes.formatLine; if (attemptSucceeded(fallbackRes.exitCode) && keepTranscript) { // Persist only. Not a fallback to transcription — the transcript is @@ -1570,6 +1655,7 @@ async function runManagedDownload( persist: plan.persist, category: plan.category, origin: opts.persistOrigin, + format: persistedFormat(), extractAudio: false, onLog: opts.onLog, signal: opts.signal, @@ -1590,6 +1676,7 @@ async function runManagedDownload( persist: plan.persist, category: plan.category, origin: opts.persistOrigin, + format: persistedFormat(), onLog: opts.onLog, signal: opts.signal, }); diff --git a/common/ytdlp/fetchWindowManaged.ts b/common/ytdlp/fetchWindowManaged.ts @@ -42,31 +42,16 @@ import { import { channelExtraArgs } from "./channelArgs"; import { runOneYtdlp } from "./runOneYtdlp"; import { FULL_LOG_PROGRESS_ARGS } from "./downloadOneManaged"; +import { clipFormatSelector } from "./downloadFormat"; // The source height a clip is worth fetching at. 720 is umtool's number and the // reason is the render: the deliverable is 1080p with a clip inset, so pixels // above 720 are thrown away after paying for them. export const DEFAULT_CLIP_MAX_HEIGHT = 720; -// THE FORMAT SELECTOR, and why it is not downloadFormatPreset's. -// -// resolveDownloadFormatSelector answers "what should we ARCHIVE" — best -// available, container-agnostic. This answers "what can ffmpeg cut and -// re-encode cheaply, right now". Those differ on one specific trap: left alone -// yt-dlp picks VP9+Opus at these heights, and since --force-keyframes-at-cuts -// re-encodes, that means libvpx-vp9 — 27 s to cut a 5 s clip, measured — and it -// writes .webm, which it then appends to the -o name. So H.264/AAC in mp4 is -// pinned, with three progressively looser fallbacks. Byte-identical to the -// selector umtool's build-video.mjs uses, on purpose: a window this fetches and -// a window that fetched must be the same file, or the two caches diverge. -export function clipFormatSelector(maxHeight: number): string { - return [ - `bv*[vcodec^=avc1][height<=${maxHeight}]+ba[acodec^=mp4a]`, - `bv*[ext=mp4][height<=${maxHeight}]+ba[ext=m4a]`, - `b[ext=mp4][height<=${maxHeight}]`, - `b[height<=${maxHeight}]`, - ].join("/"); -} +// The clip format selector lives with the download presets (the "video_720" +// preset is built from it); re-exported so existing importers keep working. +export { clipFormatSelector }; export type FetchWindowProvenance = { requestedBy: string; diff --git a/common/ytdlp/persistFormatPreset.test.ts b/common/ytdlp/persistFormatPreset.test.ts @@ -0,0 +1,216 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { Paths } from "../lib/paths"; +import type { ChannelConfig } from "../lib/channelConfig"; +import { downloadOneManaged, type ManagedDownloadOpts } from "./downloadOneManaged"; +import { resolveDownloadFormatSelector } from "./downloadFormat"; + +// Run with: +// pnpm --filter yt-dlp-transcript-common exec tsx --test ytdlp/persistFormatPreset.test.ts +// +// THE PERSIST QUALITY. A pass that persists a source container takes +// `persistFormatPreset`'s selector — unset is `bestvideo*+bestaudio/best`, as +// every persist always was — prints the format yt-dlp actually took, and that +// lands on the saved-video pointer. A "video_720" persist the last-resort rung +// answered (a file above 720p) says so in the log. The yt-dlp here is a temp +// node script; no network. + +const ID = "abc123def45"; +const VIDEO = `https://www.youtube.com/watch?v=${ID}`; + +// The prefetch writes meta.json as the info json; the subtitle pass writes +// nothing; the media pass (the `source-media` output) writes the container and +// prints `format-line` (when present) the way `--print after_video:DLOM_FORMAT` +// would. Every spawn appends its argv to `argv.log`. +const FAKE_YTDLP = `#!/usr/bin/env node +const fs = require("node:fs"); +const path = require("node:path"); +const argv = process.argv.slice(2); +const root = process.env.FAKE_ROOT; +fs.appendFileSync(path.join(root, "argv.log"), JSON.stringify(argv) + "\\n"); +const has = (f) => argv.includes(f); +const arg = (f) => { const i = argv.indexOf(f); return i < 0 ? undefined : argv[i + 1]; }; +const info = arg("--load-info-json"); +const id = info ? path.basename(path.dirname(info)) : "${ID}"; +const dir = path.join("data", id); +fs.mkdirSync(dir, { recursive: true }); +if (has("--skip-download") && has("--write-info-json") && has("--no-write-subs")) { + fs.copyFileSync(path.join(root, "meta.json"), path.join(dir, "metadata.info.json")); + process.exit(0); +} +if (has("--skip-download") && has("--write-auto-subs")) { + console.log("DLOM_ARCHIVE youtube " + id); + process.exit(0); +} +if ((arg("-o") || "").includes("source-media")) { + fs.writeFileSync(path.join(dir, "source-media.mp4"), "container bytes"); + const fl = path.join(root, "format-line"); + if (fs.existsSync(fl)) console.log("DLOM_FORMAT " + fs.readFileSync(fl, "utf8").trim()); + console.log("DLOM_ARCHIVE youtube " + id); + process.exit(0); +} +console.error("fake: unknown invocation " + argv.join(" ")); +process.exit(2); +`; + +const FAKE_FFMPEG = `#!/bin/sh +for last; do :; done +echo "extracted audio" > "$last" +`; + +const META = { + id: ID, + title: "A video", + duration: 60, + webpage_url: VIDEO, + subtitles: {}, + automatic_captions: { en: [{ url: "https://example/cap" }] }, +}; + +async function runWith(opts: { + formatLine?: string; + config?: Partial<ChannelConfig>; + managed?: Partial<ManagedDownloadOpts>; +}) { + const root = await mkdtemp(path.join(tmpdir(), "persist-format-")); + const ytdlp = path.join(root, "fake-ytdlp.cjs"); + const ffmpeg = path.join(root, "fake-ffmpeg.sh"); + await writeFile(ytdlp, FAKE_YTDLP); + await writeFile(ffmpeg, FAKE_FFMPEG); + await chmod(ytdlp, 0o755); + await chmod(ffmpeg, 0o755); + await writeFile(path.join(root, "meta.json"), JSON.stringify(META)); + if (opts.formatLine) await writeFile(path.join(root, "format-line"), opts.formatLine); + process.env.FAKE_ROOT = root; + const paths = { + transcriptsDir: root, + channelsDir: path.join(root, "channels"), + savedVideosDir: path.join(root, "saved-videos"), + ytdlpBin: ytdlp, + ffmpegBin: ffmpeg, + ffprobeBin: path.join(root, "no-ffprobe"), + } as Paths; + const videoDir = path.join(paths.channelsDir, "demo-channel", "data", ID); + await mkdir(videoDir, { recursive: true }); + // A transcript on disk: the forced pass persists and does nothing else. + await writeFile(path.join(videoDir, "transcript.json"), '{"segments":[]}'); + let log = ""; + const record = await downloadOneManaged({ + channelSlug: "demo-channel", + channelConfig: { + handling: "youtube", + url: "https://www.youtube.com/@demo/videos", + ...opts.config, + } as ChannelConfig, + paths, + videoUrl: VIDEO, + onLog: (s) => { + log += s; + }, + signal: new AbortController().signal, + keepSourceVideoOverride: true, + forceMedia: true, + ...opts.managed, + }); + const argvs = (await readFile(path.join(root, "argv.log"), "utf8")) + .split("\n") + .filter(Boolean) + .map((l) => JSON.parse(l) as string[]); + const media = argvs.find((a) => (a[a.indexOf("-o") + 1] ?? "").includes("source-media")); + const pointer = JSON.parse( + await readFile(path.join(videoDir, "saved-video.json"), "utf8"), + ) as Record<string, unknown>; + return { + record, + log, + media: media ?? [], + pointer, + cleanup: () => rm(root, { recursive: true, force: true }), + }; +} + +const selectorOf = (argv: string[]) => argv[argv.indexOf("-f") + 1]; + +test("unset: the persist selector is bestvideo*+bestaudio/best, as it always was", async () => { + const r = await runWith({ formatLine: "1080 avc1.640028 137+140" }); + try { + assert.equal(r.record.status, "ok"); + assert.equal(selectorOf(r.media), "bestvideo*+bestaudio/best"); + // The format yt-dlp took is asked for, and recorded under "original". + assert.ok(r.media.some((a) => a.startsWith("after_video:DLOM_FORMAT ")), r.media.join(" ")); + assert.deepEqual(r.pointer.format, { + preset: "original", + height: 1080, + vcodec: "avc1.640028", + }); + assert.match(r.log, /Source format: 1080p avc1\.640028 \(format 137\+140\) \(quality original\)\.\n/); + } finally { + await r.cleanup(); + } +}); + +test("video_720: the persist takes the 720 selector and records what came back", async () => { + const r = await runWith({ + formatLine: "720 avc1.4d401f 136+140", + managed: { persistFormatPreset: "video_720" }, + }); + try { + assert.equal(selectorOf(r.media), resolveDownloadFormatSelector("video_720", null)); + assert.deepEqual(r.pointer.format, { + preset: "video_720", + height: 720, + vcodec: "avc1.4d401f", + }); + assert.doesNotMatch(r.log, /last resort/); + } finally { + await r.cleanup(); + } +}); + +test("video_720 answered by the last resort: a file above 720p is logged, not silent", async () => { + const r = await runWith({ + formatLine: "1080 vp9 248+251", + managed: { persistFormatPreset: "video_720" }, + }); + try { + assert.match( + r.log, + /Source quality video_720: no format at or under 720p matched any rung; the last resort \(bv\*\+ba\/b\) took 1080p vp9 \(format 248\+251\)\.\n/, + ); + assert.deepEqual(r.pointer.format, { preset: "video_720", height: 1080, vcodec: "vp9" }); + } finally { + await r.cleanup(); + } +}); + +test("original asked explicitly is the unset selector; no printed format → preset only", async () => { + const r = await runWith({ managed: { persistFormatPreset: "original" } }); + try { + assert.equal(selectorOf(r.media), "bestvideo*+bestaudio/best"); + assert.deepEqual(r.pointer.format, { preset: "original" }); + assert.match(r.log, /Source format: not reported by yt-dlp \(quality original\)\.\n/); + } finally { + await r.cleanup(); + } +}); + +test("transcribe handling: a persisting primary takes the preset too; the audio selector is untouched", async () => { + const r = await runWith({ + formatLine: "480 avc1.4d401e 135+140", + config: { handling: "transcribe", extractionMode: "app" }, + managed: { persistFormatPreset: "video_720", forceMedia: false }, + }); + try { + assert.equal(selectorOf(r.media), resolveDownloadFormatSelector("video_720", null)); + assert.deepEqual(r.pointer.format, { + preset: "video_720", + height: 480, + vcodec: "avc1.4d401e", + }); + } finally { + await r.cleanup(); + } +}); diff --git a/common/ytdlp/runOneYtdlp.ts b/common/ytdlp/runOneYtdlp.ts @@ -17,6 +17,14 @@ export const STDERR_TAIL_BYTES = 64 * 1024; // learn the archive line without re-reading the archive file. export const ARCHIVE_MARKER = "DLOM_ARCHIVE"; +// A persist pass also `--print`s this marker after the video, with the format +// yt-dlp actually downloaded (height, vcodec, format_id of +// `requested_downloads[0]`). The info json on disk cannot say it: the prefetch +// wrote it with --skip-download, and the download reuses it via +// --load-info-json, so its top-level format is the prefetch's, and +// `requested_downloads` is only set after yt-dlp has written the file. +export const FORMAT_MARKER = "DLOM_FORMAT"; + export type YtdlpRunOpts = { ytdlpBin: string; onLog: (s: string) => void; @@ -30,6 +38,9 @@ export type AttemptOutcome = { // (lib/availability.ts classifyDownloadFailure). stderrTail: string; archiveLine: string | null; + // The text after FORMAT_MARKER, when the invocation printed one. Optional so + // an outcome assembled elsewhere (the audio-checked primary) need not name it. + formatLine?: string | null; }; export async function runOneYtdlp( @@ -55,6 +66,14 @@ export async function runOneYtdlp( let stdoutBuf = ""; let archiveLine: string | null = null; + let formatLine: string | null = null; + const scrape = (line: string) => { + if (line.startsWith(`${ARCHIVE_MARKER} `)) { + archiveLine = line.slice(ARCHIVE_MARKER.length + 1).trim(); + } else if (line.startsWith(`${FORMAT_MARKER} `)) { + formatLine = line.slice(FORMAT_MARKER.length + 1).trim(); + } + }; child.stdout?.on("data", (c: Buffer) => { const chunk = c.toString("utf8"); opts.onLog(chunk); @@ -64,21 +83,17 @@ export async function runOneYtdlp( while ((nl = stdoutBuf.indexOf("\n")) !== -1) { const line = stdoutBuf.slice(0, nl).trim(); stdoutBuf = stdoutBuf.slice(nl + 1); - if (line.startsWith(`${ARCHIVE_MARKER} `)) { - archiveLine = line.slice(ARCHIVE_MARKER.length + 1).trim(); - } + scrape(line); } }); const result = await child; // Flush any final partial line. - const trailing = stdoutBuf.trim(); - if (trailing.startsWith(`${ARCHIVE_MARKER} `)) { - archiveLine = trailing.slice(ARCHIVE_MARKER.length + 1).trim(); - } + scrape(stdoutBuf.trim()); return { exitCode: result.exitCode ?? null, stderrTail, archiveLine, + formatLine, }; } diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] - **Capture specific X posts: a screenshot of each, and its attached media.** `pnpm ops capture-posts --json '{"slug":"<channel>","ids":["<post id>", …]}'` shoots each post as X shows it, through the connected X profile, and downloads its pictures and videos with gallery-dl, into the channel's `posts-media/<post id>/` beside a `capture.json` that records when, from which URLs, and each file's size and SHA-256. Every id must already be in the channel's posts archive; one that is not is refused by name and nothing runs. `"shots": false` or `"media": false` skips that half, and posts already captured are skipped unless `"force": true`. The job runs on the X queue with a post fetch, so the two never run at once, and waits a random 4–10 seconds before each request to X, as fetches do. A deleted post, or one behind its account's wall (protected, suspended, gone), is recorded as such in the channel's deleted-post record; a post behind a sensitive-media warning is opened and shot. If X asks to log in, or answers "Something went wrong", the job stops at that post and leaves the rest for a later run. Captures are never published: the export does not read them. +- **A source video can be saved at 720p for clip and editing work.** **Persist source video** on a video's page has a **Quality** select: **Original** (the best video and audio, as every persist has been) or **Video 720p (H.264, for clips/editing)**, which saves an H.264 mp4 at most 720p tall — smaller, and quick to cut. When a source has nothing at or under 720p in H.264 it takes 480p, and when it has neither it takes whatever is best and says so in the job's log, with the height it got. The default is the new **Source video quality** under **Settings**, which a channel can override in its Advanced settings; the whole-recording fetch (`full: true`) and **Persist kept now** follow the channel's, else the global, choice. A persisted video's **Source video** card now shows the format that was saved (height and codec) and the quality asked for; videos persisted before this show nothing new. "Video 720p" is also offered as a download format. - **X posts are fetched more slowly, with random gaps.** Every read of X now waits a random 4 to 10 seconds before each request to X, where it used to page as fast as X answered, and always waits out a rate limit rather than pushing through. When fetching older posts, the pause between one three-month window and the next is a random 45 to 120 seconds instead of a fixed 15. A deep walk of an account's history takes longer; a routine fetch of new posts takes a few seconds more. - **The MCP's search tools take `date_from` and `date_to` as `2024-10-26` as well as `20241026`, and refuse a date they cannot read.** `search_transcripts` and `enumerate_matches` used to accept only `YYYYMMDD`: any other spelling was dropped with a footer warning and the search ran with no date bound, so a whole-corpus count could be read as the bounded one. Dashed, slashed and dotted dates and ISO timestamps are now normalised, and anything else is an error and nothing is searched. - **An X channel can fetch posts older than its timeline reaches.** X's timeline only pages back so far, so a fetch could end, and call the history done, well short of an account's first post. The new **Fetch older posts** button on an X channel's page (or `pnpm ops fetch-posts --json '{"slug":"<channel>","older":true}'`) walks back from the oldest archived post through X search, three months at a time, and saves posts the same way a normal fetch does; posts already archived are skipped. It needs a login, as search does: without one it stops at once and the channel shows **Needs credentials**. A run saves its place as it goes and stops after three hours; the next run continues from there. The walk ends at the account's creation date, after a year of windows with no posts, or at a date you give as `"floor": "YYYY-MM-DD"`, and the page's **Older posts** line then says it is complete; running it again says so and fetches nothing. A normal **Fetch posts** is unaffected and still fetches new posts from the top. Bluesky channels have no such button: their fetch already reads the whole history. diff --git a/editor/app/channels/[slug]/videos/[id]/components/VideoPanel.tsx b/editor/app/channels/[slug]/videos/[id]/components/VideoPanel.tsx @@ -14,6 +14,7 @@ import type { } from "yt-dlp-transcript-common/lib/channelConfig"; import type { DownloadOutcomeRecord } from "yt-dlp-transcript-common/lib/downloadOutcome"; import type { SavedVideoPointer } from "yt-dlp-transcript-common/lib/savedVideo"; +import type { SourceVideoQuality } from "yt-dlp-transcript-common/ytdlp/downloadFormat"; import type { AvailabilityHistoryEntry } from "yt-dlp-transcript-common/lib/availability"; import type { SubtitleProvenance } from "yt-dlp-transcript-common/lib/subtitleProvenance"; import { formatBytes } from "yt-dlp-transcript-common/lib/format"; @@ -73,6 +74,9 @@ type Props = { // This video's saved-video pointer when its source container is persisted to // the store, else null. Drives the Source-video persistence card (Phase 5). savedVideo?: SavedVideoPointer | null; + // The effective source-video quality for this channel (channel over global, + // resolveSourceVideoQuality): where the persist control's select starts. + sourceVideoQuality?: SourceVideoQuality; // Windows of this video's source media another tool asked the editor to // fetch, into data/<id>/clips/. Empty for almost every video. clipWindows?: FetchedWindow[]; @@ -108,6 +112,7 @@ export function VideoPanel({ doNotClean = false, excludedFromTruncatedCheck = false, savedVideo = null, + sourceVideoQuality, clipWindows = [], coverage = null, prevHref, @@ -374,6 +379,7 @@ export function VideoPanel({ defaultQueueKey={defaultQueueKey} existingQueues={existingQueues} savedVideo={savedVideo} + defaultQuality={sourceVideoQuality} /> </PipelineStageCard> diff --git a/editor/app/channels/[slug]/videos/[id]/components/cards/SourceVideoSection.tsx b/editor/app/channels/[slug]/videos/[id]/components/cards/SourceVideoSection.tsx @@ -5,6 +5,12 @@ import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActio import type { ChannelHandling } from "yt-dlp-transcript-common/lib/channelConfig"; import type { SavedVideoPointer } from "yt-dlp-transcript-common/lib/savedVideo"; import { formatBytes } from "yt-dlp-transcript-common/lib/format"; +import { + DEFAULT_SOURCE_VIDEO_QUALITY, + SOURCE_VIDEO_QUALITIES, + SOURCE_VIDEO_QUALITY_LABELS, + type SourceVideoQuality, +} from "yt-dlp-transcript-common/ytdlp/downloadFormat"; import { QueueControl } from "../../../../../../components/QueueControl"; import { cancelJobAction } from "../../../../../../jobs/actions"; import { @@ -19,6 +25,7 @@ export function SourceVideoSection({ defaultQueueKey, existingQueues, savedVideo, + defaultQuality = DEFAULT_SOURCE_VIDEO_QUALITY, }: { slug: string; videoId: string; @@ -26,8 +33,12 @@ export function SourceVideoSection({ defaultQueueKey: string; existingQueues: string[]; savedVideo: SavedVideoPointer | null; + // The effective source-video quality (channel over global), resolved by the + // page through resolveSourceVideoQuality: the select starts on it. + defaultQuality?: SourceVideoQuality; }) { const [queueKey, setQueueKey] = useState(defaultQueueKey); + const [quality, setQuality] = useState<SourceVideoQuality>(defaultQuality); // Set the moment a persist run STARTS from this surface. That run's own // router.refresh() lands the saved-video pointer, and the persisted view used // to REPLACE this whole subtree — destroying the StreamActionLog below and the @@ -74,20 +85,44 @@ export function SourceVideoSection({ disabled={!!savedVideo} trigger={() => { setRanHere(true); - return redownloadToArchiveAction(slug, videoId, queueKey); + return redownloadToArchiveAction( + slug, + videoId, + queueKey, + quality, + ); }} cancelAction={cancelJobAction} buttonLabel="Persist source video" runningLabel="Archiving…" label={`Persist source video for ${videoId}`} extraControls={ - <QueueControl - value={queueKey} - onChange={setQueueKey} - defaultQueueKey={defaultQueueKey} - existingQueues={existingQueues} - actionLabel={`Persist source video for ${videoId}`} - /> + <> + <label className="flex items-center gap-2 text-sm"> + Quality + <select + value={quality} + onChange={(e) => + setQuality(e.target.value as SourceVideoQuality) + } + aria-label="source video quality" + className="rounded border border-border bg-card px-2 py-1 text-sm" + > + {SOURCE_VIDEO_QUALITIES.map((q) => ( + <option key={q} value={q}> + {SOURCE_VIDEO_QUALITY_LABELS[q]} + </option> + ))} + </select> + </label> + <QueueControl + value={queueKey} + onChange={setQueueKey} + defaultQueueKey={defaultQueueKey} + existingQueues={existingQueues} + actionLabel={`Persist source video for ${videoId}`} + /> + </> } /> )} @@ -123,6 +158,20 @@ function PersistedSourceVideo({ <dd className="tabular-nums">{formatBytes(savedVideo.bytes)}</dd> <dt className="text-muted-foreground">Stored</dt> <dd>{new Date(savedVideo.storedAt).toLocaleString()}</dd> + {/* WHAT WAS KEPT: the quality asked for and the format yt-dlp took. + Absent on a pointer written before the quality setting. */} + {savedVideo.format && ( + <> + <dt className="text-muted-foreground">Format</dt> + <dd> + {savedVideo.format.height + ? `${savedVideo.format.height}p` + : "unknown height"} + {savedVideo.format.vcodec ? ` ${savedVideo.format.vcodec}` : ""} + {` (${SOURCE_VIDEO_QUALITY_LABELS[savedVideo.format.preset]})`} + </dd> + </> + )} {savedVideo.keepReason && ( <> <dt className="text-muted-foreground">Reason</dt> diff --git a/editor/app/channels/[slug]/videos/[id]/page.tsx b/editor/app/channels/[slug]/videos/[id]/page.tsx @@ -32,6 +32,7 @@ import { } from "yt-dlp-transcript-common/lib/platform"; import { getRegistry } from "yt-dlp-transcript-common/jobs/registry"; import { getSettings } from "yt-dlp-transcript-common/lib/settings"; +import { resolveSourceVideoQuality } from "yt-dlp-transcript-common/ytdlp/downloadFormat"; import { liveJobRows } from "../../../../jobs/active/buildActiveJobs"; import { RunningJobsList } from "../../../../jobs/components/RunningJobsList"; import { VideoPanel, type VideoFile } from "./components/VideoPanel"; @@ -309,6 +310,10 @@ export default async function VideoDetailPage({ doNotClean={doNotClean} excludedFromTruncatedCheck={excludedFromTruncatedCheck} savedVideo={savedVideo} + sourceVideoQuality={resolveSourceVideoQuality({ + channel: config.sourceVideoQuality, + global: getSettings().sourceVideoQuality, + })} clipWindows={clipWindows.map((w) => ({ file: w.file, from: w.from, diff --git a/editor/app/channels/[slug]/videos/[id]/videoActions.ts b/editor/app/channels/[slug]/videos/[id]/videoActions.ts @@ -13,7 +13,9 @@ import { AUDIO_FORMAT_VALUES } from "yt-dlp-transcript-common/lib/channelConfig" import { isDownloadFormatPreset, resolveDownloadFormatPreset, + resolveSourceVideoQuality, type DownloadFormatPreset, + type SourceVideoQuality, } from "yt-dlp-transcript-common/ytdlp/downloadFormat"; import { getPaths, type Paths } from "yt-dlp-transcript-common/lib/paths"; import { diskGate } from "yt-dlp-transcript-common/lib/diskSpace"; @@ -255,13 +257,16 @@ export async function downloadVideoPipelineAction( // media pass is otherwise skipped when a transcript or captions exist; there, // with a transcript on disk, no audio is extracted — the container is the // point. Works on a video already in the archive — downloadOneManaged has no -// archive prefilter, so it always re-downloads. +// archive prefilter, so it always re-downloads. `quality` is the control's +// per-persist choice ("original" / "video_720"); anything else, or nothing, +// inherits the channel's, else the global, source-video quality. export async function redownloadToArchiveAction( slug: string, videoId: string, queueKey?: string, + quality?: string, ): Promise<StreamActionResult> { - return archiveSourceVideo(slug, videoId, queueKey); + return archiveSourceVideo(slug, videoId, queueKey, undefined, quality); } // The body of both "Persist source video" (the button) and the full-source @@ -274,6 +279,9 @@ async function archiveSourceVideo( videoId: string, queueKey: string | undefined, persistOrigin?: SavedVideoOrigin, + // The per-persist quality override, unvalidated: resolveSourceVideoQuality + // drops anything that is not a quality, falling through to channel/global. + qualityOverride?: unknown, ): Promise<StreamActionResult> { const r = await loadConfigOrError(slug); if (!r.ok) return r; @@ -289,6 +297,11 @@ async function archiveSourceVideo( const err = await lowDiskError(paths, slug); if (err) return err; const settings = getSettings(); + const persistFormatPreset = resolveSourceVideoQuality({ + override: qualityOverride, + channel: r.config.sourceVideoQuality, + global: settings.sourceVideoQuality, + }); return runManagedFunction({ kind: "redownload-archive", queueKey: videoQueueKey(r.config, queueKey), @@ -305,7 +318,9 @@ async function archiveSourceVideo( if (persistOrigin) { onLog(`${requesterLine(persistOrigin)}\n`); } - onLog(`Re-downloading ${videoId} to archive its source video…\n`); + onLog( + `Re-downloading ${videoId} to archive its source video (quality ${persistFormatPreset})…\n`, + ); const record = await downloadOneManaged({ channelSlug: slug, channelConfig: r.config, @@ -324,6 +339,7 @@ async function archiveSourceVideo( // got two subtitle passes and no file (release 10 slice N). forceMedia: true, persistOrigin, + persistFormatPreset, }); safeRevalidate([ `/channels/${slug}/videos/${videoId}`, @@ -1153,6 +1169,8 @@ export async function fetchFullSourceAction(req: { videoId: string; provenance: FetchWindowProvenance; queueKey?: string; + // Per-fetch quality; absent = the channel's, else the global setting. + quality?: SourceVideoQuality; }): Promise<FetchMediaOutcome> { const { slug, videoId } = req; const r = await loadConfigOrError(slug); @@ -1209,7 +1227,13 @@ export async function fetchFullSourceAction(req: { ...(req.provenance.reason ? { reason: req.provenance.reason } : {}), requestedAt: req.provenance.requestedAt ?? new Date().toISOString(), }; - const res = await archiveSourceVideo(slug, videoId, req.queueKey, origin); + const res = await archiveSourceVideo( + slug, + videoId, + req.queueKey, + origin, + req.quality, + ); if (!res.ok) { // archiveSourceVideo's own low-disk refusal is the only 507-shaped one it // returns; everything else is a lookup failure or an unreachable drive. diff --git a/editor/app/channels/[slug]/videos/page.tsx b/editor/app/channels/[slug]/videos/page.tsx @@ -21,6 +21,8 @@ import { TRANSCRIPTION_QUEUE, } from "yt-dlp-transcript-common/lib/platform"; import { getRegistry } from "yt-dlp-transcript-common/jobs/registry"; +import { getSettings } from "yt-dlp-transcript-common/lib/settings"; +import { resolveSourceVideoQuality } from "yt-dlp-transcript-common/ytdlp/downloadFormat"; import { VideoPanel, type VideoFile } from "../videos/[id]/components/VideoPanel"; import { VideoWorkspace } from "./components/VideoWorkspace"; import { readChannelConfigCached } from "../lib/channelConfigCache"; @@ -255,6 +257,10 @@ export default async function ChannelVideosPage({ availabilityHistory={availabilityRecord?.history ?? []} channelAudioFormat={config.audioFormat} excludedFromTruncatedCheck={excludedTrunc} + sourceVideoQuality={resolveSourceVideoQuality({ + channel: config.sourceVideoQuality, + global: getSettings().sourceVideoQuality, + })} coverage={coverage} // The standalone /videos/[id] page has no list to walk, which is why // the embedded pane keeps prev/next and the position readout. diff --git a/editor/app/channels/components/ChannelForm.tsx b/editor/app/channels/components/ChannelForm.tsx @@ -12,6 +12,8 @@ import { import { DOWNLOAD_FORMAT_LABELS, DOWNLOAD_FORMAT_PRESETS, + SOURCE_VIDEO_QUALITIES, + SOURCE_VIDEO_QUALITY_LABELS, } from "yt-dlp-transcript-common/ytdlp/downloadFormat"; import { AUDIO_CHECK_COPY_TIMEOUT_DEFAULT_SECONDS, @@ -641,6 +643,28 @@ export function ChannelForm({ audio in their original file (e.g. Odysee). </span> </label> + <label className="flex flex-col gap-1 text-sm"> + <span className="font-medium">Source video quality</span> + <SeededSelect + state={state} + name="sourceVideoQuality" + initial={c?.sourceVideoQuality ?? ""} + className="rounded border border-border bg-card px-2 py-1 text-sm" + > + <option value="">Inherit (global default)</option> + {SOURCE_VIDEO_QUALITIES.map((q) => ( + <option key={q} value={q}> + {SOURCE_VIDEO_QUALITY_LABELS[q]} + </option> + ))} + </SeededSelect> + <span className="text-xs text-muted-foreground"> + What a full persist of this channel's videos keeps + (“Persist source video”, a whole-recording fetch, + “Persist kept now”). <strong>720p</strong> saves a + smaller H.264 file that is quick to cut for clips and editing. + </span> + </label> <label className="flex items-start gap-2 text-sm"> <input type="checkbox" diff --git a/editor/app/channels/components/channelConfigToForm.ts b/editor/app/channels/components/channelConfigToForm.ts @@ -37,6 +37,7 @@ export const CHANNEL_FORM_VALUES = [ "socialHandle", "audioFormat", "downloadFormat", + "sourceVideoQuality", "keepLatest", "extractionMode", "savedVideosDir", @@ -87,6 +88,7 @@ export function channelConfigToFormData(config: ChannelConfig): FormData { put(fd, "socialHandle", config.socialHandle); put(fd, "audioFormat", config.audioFormat); put(fd, "downloadFormat", config.downloadFormat); + put(fd, "sourceVideoQuality", config.sourceVideoQuality); if (config.keepSourceVideo) fd.set("keepSourceVideo", "on"); putNum(fd, "keepLatest", config.keepLatest); put(fd, "extractionMode", config.extractionMode); diff --git a/editor/app/channels/components/parseChannelForm.ts b/editor/app/channels/components/parseChannelForm.ts @@ -20,7 +20,10 @@ import { } from "yt-dlp-transcript-common/lib/platform"; import { handleFromAccountUrl } from "yt-dlp-transcript-common/social/fetchers"; import { isCookieMode } from "yt-dlp-transcript-common/lib/cookiePolicy"; -import { isDownloadFormatPreset } from "yt-dlp-transcript-common/ytdlp/downloadFormat"; +import { + isDownloadFormatPreset, + isSourceVideoQuality, +} from "yt-dlp-transcript-common/ytdlp/downloadFormat"; import { downloadFilterPatternProblem } from "yt-dlp-transcript-common/lib/downloadFilters"; import { isRejectedLivestreamMode } from "yt-dlp-transcript-common/lib/channelConfig"; @@ -42,6 +45,7 @@ export const CHANNEL_FORM_FIELDS = [ "url", "audioFormat", "downloadFormat", + "sourceVideoQuality", "keepSourceVideo", "keepLatest", "extractionMode", @@ -112,6 +116,10 @@ export function parseChannelForm(formData: FormData): ParsedChannelForm { const downloadFormatRaw = String(formData.get("downloadFormat") ?? ""); const downloadFormat: ChannelConfig["downloadFormat"] | undefined = isDownloadFormatPreset(downloadFormatRaw) ? downloadFormatRaw : undefined; + // Blank = inherit the global sourceVideoQuality (omit the key). + const sourceVideoQualityRaw = String(formData.get("sourceVideoQuality") ?? ""); + const sourceVideoQuality: ChannelConfig["sourceVideoQuality"] | undefined = + isSourceVideoQuality(sourceVideoQualityRaw) ? sourceVideoQualityRaw : undefined; const keepSourceVideo = formData.get("keepSourceVideo") != null; // Keep-latest retention/persistence window. Blank = inherit-off (omit), so a @@ -342,6 +350,7 @@ export function parseChannelForm(formData: FormData): ParsedChannelForm { if (url) config.url = url; if (audioFormat) config.audioFormat = audioFormat; if (downloadFormat) config.downloadFormat = downloadFormat; + if (sourceVideoQuality) config.sourceVideoQuality = sourceVideoQuality; if (keepSourceVideo) config.keepSourceVideo = true; if (keepLatest != null) config.keepLatest = keepLatest; if (extractionMode) config.extractionMode = extractionMode; diff --git a/editor/app/settings/actions.ts b/editor/app/settings/actions.ts @@ -25,7 +25,11 @@ import { DEFAULT_COOKIE_MODE, isCookieMode, } from "yt-dlp-transcript-common/lib/cookiePolicy"; -import { isDownloadFormatPreset } from "yt-dlp-transcript-common/ytdlp/downloadFormat"; +import { + DEFAULT_SOURCE_VIDEO_QUALITY, + isDownloadFormatPreset, + isSourceVideoQuality, +} from "yt-dlp-transcript-common/ytdlp/downloadFormat"; import { formValues, type FormState } from "../lib/formState"; // THE SHARED SAVE RESULT of /settings and the four operation-settings forms. A @@ -80,6 +84,12 @@ export async function saveSettingsAction( const downloadFormat = isDownloadFormatPreset(downloadFormatRaw) ? downloadFormatRaw : "auto"; + const sourceVideoQualityRaw = String( + formData.get("sourceVideoQuality") ?? "", + ).trim(); + const sourceVideoQuality = isSourceVideoQuality(sourceVideoQualityRaw) + ? sourceVideoQualityRaw + : DEFAULT_SOURCE_VIDEO_QUALITY; if (!adminTitle) { return { ok: false, error: "Admin title is required", values }; @@ -230,6 +240,7 @@ export async function saveSettingsAction( cookieMode, sleepBetweenDownloadsSeconds: sleepParsed, downloadFormat, + sourceVideoQuality, minFreeDiskGB: minFreeDiskParsed, resumeMarginGB: resumeMarginParsed, // Vestigial (the worker list is the parallelism; kept only for rollback), diff --git a/editor/app/settings/components/SettingsForm.tsx b/editor/app/settings/components/SettingsForm.tsx @@ -10,6 +10,8 @@ import type { SiteSettings } from "yt-dlp-transcript-common/lib/settings"; import { DOWNLOAD_FORMAT_LABELS, DOWNLOAD_FORMAT_PRESETS, + SOURCE_VIDEO_QUALITIES, + SOURCE_VIDEO_QUALITY_LABELS, } from "yt-dlp-transcript-common/ytdlp/downloadFormat"; import { Field, @@ -130,6 +132,30 @@ export function SettingsForm({ initial }: Props) { Advanced settings, and a single re-download can override it too. </span> </label> + <label className="flex flex-col gap-1 text-sm"> + <span className="font-medium">Source video quality</span> + <SeededSelect + state={state} + name="sourceVideoQuality" + initial={initial.sourceVideoQuality} + className="rounded border border-border bg-card px-2 py-1 text-sm" + > + {SOURCE_VIDEO_QUALITIES.map((q) => ( + <option key={q} value={q}> + {SOURCE_VIDEO_QUALITY_LABELS[q]} + </option> + ))} + </SeededSelect> + <span className="text-xs text-muted-foreground"> + What a full persist keeps: “Persist source video”, a + whole-recording fetch and “Persist kept now”.{" "} + <strong>Original</strong> takes the best video and audio;{" "} + <strong>720p</strong> takes an H.264 file at most 720p tall (480p + when there is none), smaller and quick to cut for clips and editing. + Each channel can override this in its Advanced settings, and a single + persist can override it too. + </span> + </label> <Field label="Minimum free disk space (GB)" name="minFreeDiskGB" diff --git a/plans/FACTS.md b/plans/FACTS.md @@ -8588,3 +8588,21 @@ phase deletes from the destination. - The e2e app server has no media root (`UMTOOL_MEDIA_DIR=` in `playwright.config.ts`); specs give their CLI one (`<fixture>-media`), so `deliverables.spec.ts` shows an app cutting through a link it did not make and refusing a new batch under `"media"`. + +## A persisted container's format is not in `metadata.info.json` (2026-10-04) + +- **The info json on disk describes the PREFETCH's format, not the download's.** The metadata + prefetch writes it with `--skip-download` (yt-dlp's default selector), and every later pass reads + it back with `--load-info-json` and does not rewrite it. Its top-level `format_id`/`height`/`vcodec` + can name a 1080p format for a file downloaded at 720p. +- **`requested_downloads` is never in it.** yt-dlp sets that key in `process_video_result` after + `process_info` has written the file (`YoutubeDL.py`, `info_dict['requested_downloads'] = …` just + before `run_all_pps('after_video', …)`), so only an `after_video` print can see it. +- So a persist pass prints `DLOM_FORMAT %(requested_downloads.0.height)s …vcodec…format_id` (the + `FORMAT_MARKER` in `common/ytdlp/runOneYtdlp.ts`), and the saved-video pointer's `format` is read + from that line (`savedVideoFormatFromLine`). A field yt-dlp lacks prints as `NA`. +- **Source-video quality resolves through `resolveSourceVideoQuality`** (`common/ytdlp/downloadFormat.ts`): + per-persist override > `ChannelConfig.sourceVideoQuality` > settings `sourceVideoQuality` > + `"original"`. `downloadOneManaged`'s `persistFormatPreset` takes the result; unset is the historical + `bestvideo*+bestaudio/best`. The download lane's keep-latest persists pass nothing, so they stay + `"original"`. diff --git a/settings.json.example b/settings.json.example @@ -16,6 +16,7 @@ "holdProbeMinutes": 60 }, "downloadFormat": "auto", + "sourceVideoQuality": "original", "minFreeDiskGB": 5, "resumeMarginGB": 2, "parallelTranscriptions": 2,