commit a2c6ef055e083b2a39d4833f86b117e2a5fd63d7 parent 09ad64e928b303f70bf0ead35cb5a70a232a2439 Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st> Date: Wed, 22 Jul 2026 00:22:46 -0400 Configurable cookies-from-browser: cookie modes, per-channel overrides, Needs-cookies bucket The single global cookiesFromBrowser value was hard-wired to one behavior (passed only on the download auth-retry attempt). It is now governed by a cookie MODE, resolved channel-over-global by the new client-safe common/lib/cookiePolicy.ts: - always: every yt-dlp invocation carries --cookies-from-browser (enumeration, metadata prefetch, availability probes, downloads). - when-required (default; historical behavior): cookies only to retry an auth/age failure — now ALSO cookie-retrying a failed metadata prefetch (new "metadata-prefetch-auth-retry" attempt kind), so a gated video's real download starts with cookies and reports ok-with-cookies. - defer: never in normal runs — known needs_auth videos are excluded from playlist/sync batches and from undownloadedIds, and wait in the new snapshot bucket needsCookies for a manual cookie run. The per-channel form's Advanced section overrides both the value and the mode (blank/Inherit = global). Every channel's Download stage gains a "Needs cookies (N)" card (bucket = undownloaded videos whose effective availability is in AUTH_RETRY_CLASSES, populated in every mode) whose "Download with cookies" button runs retry-bucket with a new forceCookies flag that overrides the mode to always (bookmarkable; warns when no cookie value is configured). Availability probes attach cookies in always mode only, so needs_auth keeps being observed — defer depends on that. "ok-with-cookies" keeps meaning cookies were NEEDED; always-mode prophylactic cookies on a clean success stay "ok". Back-compat: a settings.json without cookieMode behaves exactly as before. e2e: new cookies-mode.spec.ts + fake-ytdlp "cookiegated" sentinel (fails without --cookies-from-browser, succeeds with; invocation lines log cookies=<val>). Unit: cookiePolicy.test.ts, jobSpec.test.ts. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Diffstat:
30 files changed, 1131 insertions(+), 52 deletions(-)
diff --git a/common/controller/autoRunner.ts b/common/controller/autoRunner.ts @@ -32,6 +32,7 @@ import { pruneExpired, } from "../jobs/platformBackoff"; import { type DownloadFailureClass } from "../lib/availability"; +import { resolveCookiePolicy } from "../lib/cookiePolicy"; import { type DownloadOutcomeStatus } from "../lib/downloadOutcome"; import { downloadQueueKey } from "../lib/queueKeys"; import { readChannelConfig } from "./channels"; @@ -622,7 +623,7 @@ async function launchUnit(args: LaunchArgs): Promise<UnitResult> { // The child job's own abort signal: registry.cancel(childJobId) aborts // it (→ kills yt-dlp) when the runner is hard-cancelled. signal, - globalCookiesFromBrowser: settings.cookiesFromBrowser || undefined, + cookiePolicy: resolveCookiePolicy(settings, config), inlineTranscribeOnFallback: settings.inlineTranscribeOnFallback, globalSkipLiveDownloads: settings.skipLiveDownloads, downloadFormatPreset: resolveDownloadFormatPreset({ diff --git a/common/controller/channelSnapshot.ts b/common/controller/channelSnapshot.ts @@ -11,10 +11,13 @@ import { type VideoFiles, } from "../lib/videoStatus"; import { + AUTH_RETRY_CLASSES, AVAILABILITY_VALUES, EXCLUDED_FROM_DOWNLOAD, type Availability, } from "../lib/availability"; +import { resolveCookiePolicy } from "../lib/cookiePolicy"; +import { getSettings } from "../lib/settings"; import { loadAvailability, resolveEffectiveAvailability, @@ -106,6 +109,14 @@ export type ChannelSnapshot = { // user can re-download with a different format (e.g. Original). Optional: // older snapshots lack it; readers must default to []. shortAudio: string[]; + // Undownloaded playlist videos whose effective availability says browser + // cookies could recover them (AUTH_RETRY_CLASSES: needs_auth, members_only, + // private). Populated in EVERY cookie mode — members_only/private are + // batch-excluded regardless, and a subscribed/owning account's cookies can + // still fetch them — and drives the "Download with cookies" bucket button + // (retry-bucket with forceCookies). Optional: older snapshots lack it; + // readers must default to []. + needsCookies: string[]; }; undownloadedIds: string[]; excludedFromDownload?: ExcludedFromDownload; @@ -339,7 +350,11 @@ export async function generateChannelSnapshot( // "Missing metadata" or "Failed transcriptions" creates noise the user // cannot resolve. const excludedById = new Map<string, Availability>(); + const effectiveById = new Map<string, Availability>(); for (const v of perVideo) { + if (v.effectiveAvailability) { + effectiveById.set(v.id, v.effectiveAvailability); + } if ( v.effectiveAvailability && (EXCLUDED_FROM_DOWNLOAD as ReadonlyArray<Availability>).includes( @@ -535,7 +550,13 @@ export async function generateChannelSnapshot( excludedFromDownload.deleted.sort(); excludedFromDownload.private.sort(); + // The channel's resolved cookie mode. In "defer" mode, needs_auth videos are + // ALSO dropped from undownloadedIds (which the auto-download runner + // consumes), so they aren't re-attempted every run — they wait in the + // needsCookies bucket for the manual cookie run instead. + const cookiePolicy = resolveCookiePolicy(getSettings(), config ?? undefined); const undownloadedIds: string[] = []; + const needsCookies: string[] = []; for (const url of urls) { // Post-reconcile a video's dir is its canonical id, so the URL's canonical // id is the dir name directly. @@ -543,7 +564,12 @@ export async function generateChannelSnapshot( if (!dirId) continue; const f = filesById.get(dirId); if (f && videoHasAnyArtifact(f)) continue; + const effective = effectiveById.get(dirId); + if (effective && AUTH_RETRY_CLASSES.has(effective)) { + needsCookies.push(dirId); + } if (excludedById.has(dirId)) continue; + if (cookiePolicy.mode === "defer" && effective === "needs_auth") continue; undownloadedIds.push(dirId); } @@ -589,6 +615,7 @@ export async function generateChannelSnapshot( skippedByFilter: skippedByFilter.sort(), incompleteTranscript: incompleteTranscript.sort(), shortAudio: shortAudio.sort(), + needsCookies: needsCookies.sort(), }, undownloadedIds, excludedFromDownload, diff --git a/common/controller/checkAvailability.ts b/common/controller/checkAvailability.ts @@ -14,6 +14,12 @@ import { recordAvailability, resolveEffectiveAvailability, } from "../lib/availability-server"; +import { + alwaysCookies, + cookieArgs, + resolveCookiePolicy, +} from "../lib/cookiePolicy"; +import { getSettings } from "../lib/settings"; import { readChannelConfig } from "./channels"; import { resolveShardItems } from "./shard"; import type { Paths } from "../lib/paths"; @@ -115,6 +121,14 @@ export async function runAvailabilityCheck({ const dataDir = path.join(channelDir, "data"); const config = await readChannelConfig(paths, channelSlug); const extraArgs = config?.ytdlpExtraArgs ?? []; + // Probe cookies in "always" mode ONLY. when-required/defer probes stay + // cookie-free deliberately, so auth gating keeps being OBSERVED as + // needs_auth — defer mode's exclusion + Needs-cookies bucket depend on that + // signal. (Always-mode probes may report a cookie-recoverable video as + // public; that's the trade-off of prophylactic cookies.) + const probeCookies = alwaysCookies( + resolveCookiePolicy(getSettings(), config ?? undefined), + ); const limit = pLimit(concurrency && concurrency > 0 ? Math.floor(concurrency) : 1); @@ -201,6 +215,7 @@ export async function runAvailabilityCheck({ "--dump-json", "--skip-download", "--no-warnings", + ...cookieArgs(probeCookies), ...extraArgs, "--", url, diff --git a/common/controller/persistKept.ts b/common/controller/persistKept.ts @@ -2,6 +2,7 @@ import path from "node:path"; import type { Paths } from "../lib/paths"; import type { ChannelConfig } from "../lib/channelConfig"; import { getSettings } from "../lib/settings"; +import { resolveCookiePolicy } from "../lib/cookiePolicy"; import { isSavedVideo } from "../lib/savedVideo-server"; import { computeKeptVideoIds } from "./keptVideos"; import { findVideoSourceUrl } from "./undownloadedVideos"; @@ -99,7 +100,7 @@ export async function persistKept({ videoUrl: url, onLog: downloadLog, signal: downloadSignal, - globalCookiesFromBrowser: settings.cookiesFromBrowser || undefined, + cookiePolicy: resolveCookiePolicy(settings, channelConfig), inlineTranscribeOnFallback: settings.inlineTranscribeOnFallback, globalSkipLiveDownloads: settings.skipLiveDownloads, appendArchive: true, diff --git a/common/jobs/jobSpec.test.ts b/common/jobs/jobSpec.test.ts @@ -0,0 +1,50 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { parseJobSpec } from "./jobSpec"; + +// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test common/jobs/jobSpec.test.ts + +test("parses a minimal spec and rejects malformed input", () => { + assert.deepEqual(parseJobSpec({ kind: "sync", slug: "chan" }), { + kind: "sync", + slug: "chan", + }); + assert.equal(parseJobSpec(null), null); + assert.equal(parseJobSpec("sync"), null); + assert.equal(parseJobSpec({ kind: "", slug: "chan" }), null); + assert.equal(parseJobSpec({ kind: "sync" }), null); +}); + +test("accepts every replay bucket, including needsCookies", () => { + for (const bucket of [ + "partialDownloads", + "noTranscript", + "downloadedNoTranscript", + "incompleteTranscript", + "shortAudio", + "needsCookies", + ]) { + const spec = parseJobSpec({ kind: "retry-bucket", slug: "chan", bucket }); + assert.equal(spec?.bucket, bucket); + } + // Unknown buckets invalidate the whole spec. + assert.equal( + parseJobSpec({ kind: "retry-bucket", slug: "chan", bucket: "nope" }), + null, + ); +}); + +test("carries flag params verbatim (e.g. forceCookies)", () => { + const spec = parseJobSpec({ + kind: "retry-bucket", + slug: "chan", + bucket: "needsCookies", + params: { queueKey: "youtube", forceCookies: true }, + }); + assert.deepEqual(spec?.params, { queueKey: "youtube", forceCookies: true }); + // Array params are rejected (params must be a plain object). + assert.equal( + parseJobSpec({ kind: "sync", slug: "chan", params: [] })?.params, + undefined, + ); +}); diff --git a/common/jobs/jobSpec.ts b/common/jobs/jobSpec.ts @@ -20,7 +20,8 @@ export type ReplayBucket = | "noTranscript" | "downloadedNoTranscript" | "incompleteTranscript" - | "shortAudio"; + | "shortAudio" + | "needsCookies"; export type JobSpec = { kind: string; @@ -40,6 +41,7 @@ const REPLAY_BUCKETS: ReadonlySet<string> = new Set<ReplayBucket>([ "downloadedNoTranscript", "incompleteTranscript", "shortAudio", + "needsCookies", ]); // Defensive parse for a spec read back from JSON (a sidecar or the bookmarks diff --git a/common/lib/availability.ts b/common/lib/availability.ts @@ -31,6 +31,18 @@ export const EXCLUDED_FROM_DOWNLOAD: ReadonlyArray<Availability> = [ "private", ]; +// Availability classes a cookie (auth) retry can potentially recover — the +// gate for the managed downloader's auth-retry attempts and the membership +// rule for the per-channel "Needs cookies" snapshot bucket. Note the overlap +// with EXCLUDED_FROM_DOWNLOAD: members_only/private are batch-excluded, but +// cookies from a subscribed/owning account can still fetch them via the +// bucket's manual cookie run. +export const AUTH_RETRY_CLASSES: ReadonlySet<Availability> = new Set<Availability>([ + "needs_auth", + "members_only", + "private", +]); + export type AvailabilityHistorySource = "check" | "backfill" | "download"; export type AvailabilityHistoryEntry = { diff --git a/common/lib/channelConfig.ts b/common/lib/channelConfig.ts @@ -3,6 +3,7 @@ import { isDownloadFormatPreset, type DownloadFormatPreset, } from "../ytdlp/downloadFormat"; +import { isCookieMode, type CookieMode } from "./cookiePolicy"; export type ChannelHandling = "youtube" | "transcribe"; @@ -96,6 +97,14 @@ export type ChannelConfig = { // omitted, the global SiteSettings value is used. Set false to allow this // channel to download currently-live/upcoming videos. skipLiveDownloads?: boolean; + // Per-channel override of the global cookies-from-browser browser spec + // (SiteSettings.cookiesFromBrowser). Omitted/empty = inherit the global + // value. See common/lib/cookiePolicy.ts. + cookiesFromBrowser?: string; + // Per-channel override of the global cookie mode + // (SiteSettings.cookieMode). Omitted = inherit. See + // common/lib/cookiePolicy.ts for the mode semantics. + cookieMode?: CookieMode; // Opt-in audio-integrity checking for sources that intermittently serve // corrupt audio mid-download (e.g. Odysee "original" format). When // enabled, the managed downloader periodically validates the in-progress @@ -233,6 +242,12 @@ export function parseChannelConfig(raw: unknown): ChannelConfig | null { if (typeof r.skipLiveDownloads === "boolean") { config.skipLiveDownloads = r.skipLiveDownloads; } + if (typeof r.cookiesFromBrowser === "string" && r.cookiesFromBrowser.trim()) { + config.cookiesFromBrowser = r.cookiesFromBrowser.trim(); + } + if (isCookieMode(r.cookieMode)) { + config.cookieMode = r.cookieMode; + } if ( typeof r.sleepBetweenDownloadsSeconds === "number" && Number.isFinite(r.sleepBetweenDownloadsSeconds) && diff --git a/common/lib/cookiePolicy.test.ts b/common/lib/cookiePolicy.test.ts @@ -0,0 +1,109 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + DEFAULT_COOKIE_MODE, + alwaysCookies, + authRetryCookies, + cookieArgs, + isCookieMode, + resolveCookiePolicy, + type ResolvedCookiePolicy, +} from "./cookiePolicy"; + +// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test common/lib/cookiePolicy.test.ts +// (or `node_modules/.bin/tsx --test common/lib/cookiePolicy.test.ts` from the repo root) + +test("isCookieMode accepts exactly the three modes", () => { + assert.equal(isCookieMode("always"), true); + assert.equal(isCookieMode("when-required"), true); + assert.equal(isCookieMode("defer"), true); + assert.equal(isCookieMode(""), false); + assert.equal(isCookieMode("never"), false); + assert.equal(isCookieMode(undefined), false); + assert.equal(isCookieMode(null), false); + assert.equal(isCookieMode(42), false); +}); + +test("value resolution: channel > global > undefined; empty/whitespace = inherit", () => { + assert.equal( + resolveCookiePolicy({ cookiesFromBrowser: "firefox" }).cookies, + "firefox", + ); + assert.equal( + resolveCookiePolicy( + { cookiesFromBrowser: "firefox" }, + { cookiesFromBrowser: "chrome:Default" }, + ).cookies, + "chrome:Default", + ); + // Empty / whitespace channel value inherits the global. + assert.equal( + resolveCookiePolicy( + { cookiesFromBrowser: "firefox" }, + { cookiesFromBrowser: "" }, + ).cookies, + "firefox", + ); + assert.equal( + resolveCookiePolicy( + { cookiesFromBrowser: "firefox" }, + { cookiesFromBrowser: " " }, + ).cookies, + "firefox", + ); + // Nothing configured anywhere. + assert.equal(resolveCookiePolicy({}).cookies, undefined); + assert.equal(resolveCookiePolicy({ cookiesFromBrowser: " " }).cookies, undefined); + // Values are trimmed. + assert.equal( + resolveCookiePolicy({ cookiesFromBrowser: " firefox " }).cookies, + "firefox", + ); +}); + +test("mode resolution: channel > global > default", () => { + assert.equal(resolveCookiePolicy({}).mode, DEFAULT_COOKIE_MODE); + assert.equal(resolveCookiePolicy({ cookieMode: "always" }).mode, "always"); + assert.equal( + resolveCookiePolicy({ cookieMode: "always" }, { cookieMode: "defer" }).mode, + "defer", + ); + // Absent channel mode inherits the global. + assert.equal( + resolveCookiePolicy({ cookieMode: "defer" }, {}).mode, + "defer", + ); + assert.equal(resolveCookiePolicy({ cookieMode: "defer" }, null).mode, "defer"); + // Junk mode (e.g. hand-edited file that bypassed parsing) falls to default. + assert.equal( + resolveCookiePolicy({ cookieMode: "sometimes" as never }).mode, + DEFAULT_COOKIE_MODE, + ); +}); + +test("accessors encode the mode semantics", () => { + const p = (mode: ResolvedCookiePolicy["mode"], cookies?: string) => + ({ cookies, mode }) as ResolvedCookiePolicy; + + // always: cookies on every invocation, and on retries. + assert.equal(alwaysCookies(p("always", "firefox")), "firefox"); + assert.equal(authRetryCookies(p("always", "firefox")), "firefox"); + // when-required: never prophylactically, yes on auth retries. + assert.equal(alwaysCookies(p("when-required", "firefox")), undefined); + assert.equal(authRetryCookies(p("when-required", "firefox")), "firefox"); + // defer: never. + assert.equal(alwaysCookies(p("defer", "firefox")), undefined); + assert.equal(authRetryCookies(p("defer", "firefox")), undefined); + + // No value configured: every accessor degrades to undefined in every mode. + for (const mode of ["always", "when-required", "defer"] as const) { + assert.equal(alwaysCookies(p(mode)), undefined); + assert.equal(authRetryCookies(p(mode)), undefined); + } +}); + +test("cookieArgs emits the flag pair or nothing", () => { + assert.deepEqual(cookieArgs("firefox"), ["--cookies-from-browser", "firefox"]); + assert.deepEqual(cookieArgs(undefined), []); + assert.deepEqual(cookieArgs(""), []); +}); diff --git a/common/lib/cookiePolicy.ts b/common/lib/cookiePolicy.ts @@ -0,0 +1,86 @@ +// Client-safe cookie-policy resolution. No node-only imports — the editor's +// settings/channel forms are "use client" files that pull the mode constants +// in. Mirrors availability.ts in that respect. +// +// One browser-cookie spec (yt-dlp `--cookies-from-browser`) and one MODE decide +// how every yt-dlp spawn uses cookies: +// +// "always" — pass cookies on every invocation (enumeration, metadata +// prefetch, availability probes, downloads). +// "when-required" — (default; the historical behavior) cookies only to retry +// an attempt that failed with an auth/age error. Covers the +// download auth-retry AND the metadata-prefetch retry. +// "defer" — never use cookies in normal runs. Auth-gated videos are +// recorded (needs_auth), excluded from subsequent batch +// runs, and collected into the per-channel "Needs cookies" +// bucket whose button re-runs them with cookies forced on. +// +// The VALUE resolves channel-over-global (non-empty channel spec wins; empty = +// inherit). The MODE resolves the same way (absent channel mode = inherit). +// With no value configured, "always"/"when-required" degrade to the no-cookie +// behavior; "defer" still defers (the bucket run then warns that no cookie +// value is configured). + +export type CookieMode = "always" | "when-required" | "defer"; + +export const COOKIE_MODE_VALUES: ReadonlyArray<CookieMode> = [ + "always", + "when-required", + "defer", +]; + +export const DEFAULT_COOKIE_MODE: CookieMode = "when-required"; + +export function isCookieMode(v: unknown): v is CookieMode { + return v === "always" || v === "when-required" || v === "defer"; +} + +export type ResolvedCookiePolicy = { + // The resolved browser spec, or undefined when neither the channel nor the + // global settings carry a non-empty one. + cookies: string | undefined; + mode: CookieMode; +}; + +// The subset of SiteSettings / ChannelConfig this module reads. Structural, so +// neither settings.ts nor channelConfig.ts needs to be imported here (both +// import CookieMode from this file). +export type CookiePolicyInputs = { + cookiesFromBrowser?: string; + cookieMode?: CookieMode; +}; + +export function resolveCookiePolicy( + settings: CookiePolicyInputs, + channelConfig?: CookiePolicyInputs | null, +): ResolvedCookiePolicy { + const channelValue = channelConfig?.cookiesFromBrowser?.trim() ?? ""; + const globalValue = settings.cookiesFromBrowser?.trim() ?? ""; + const cookies = channelValue || globalValue || undefined; + const mode = isCookieMode(channelConfig?.cookieMode) + ? channelConfig!.cookieMode! + : isCookieMode(settings.cookieMode) + ? settings.cookieMode! + : DEFAULT_COOKIE_MODE; + return { cookies, mode }; +} + +// Cookies for an ordinary (non-retry) invocation: only "always" mode passes +// them prophylactically. Undefined in every other mode — including "defer", +// whose whole point is that normal runs stay cookie-free. +export function alwaysCookies(p: ResolvedCookiePolicy): string | undefined { + return p.mode === "always" ? p.cookies : undefined; +} + +// Cookies for retrying an attempt that failed with an auth/age error: +// available in "always" and "when-required", never in "defer" (the failure is +// recorded instead, feeding the Needs-cookies bucket). +export function authRetryCookies(p: ResolvedCookiePolicy): string | undefined { + return p.mode === "defer" ? undefined : p.cookies; +} + +// The argv fragment for a resolved cookie spec. Empty when there is none, so +// spawn sites can unconditionally spread it. +export function cookieArgs(cookies: string | undefined): string[] { + return cookies ? ["--cookies-from-browser", cookies] : []; +} diff --git a/common/lib/downloadOutcome.ts b/common/lib/downloadOutcome.ts @@ -48,7 +48,11 @@ export type DownloadAttemptKind = | "audio-checked-primary" // The metadata-only pass that runs before the real download so app-level // filters can decide, and so the download can reuse it via --load-info-json. - | "metadata-prefetch"; + | "metadata-prefetch" + // A cookie re-run of a metadata prefetch that failed with an auth/age error + // (cookie mode "always"/"when-required" with a cookie value configured). + // Recorded with n: 0 alongside the failed prefetch it retries. + | "metadata-prefetch-auth-retry"; export type AudioCheckProbeVerdict = "clean" | "partial" | "malformed"; diff --git a/common/lib/settings.ts b/common/lib/settings.ts @@ -24,6 +24,11 @@ import { defaultAutoQueue, sanitizeAutoQueue, } from "../jobs/autoQueuePolicy"; +import { + DEFAULT_COOKIE_MODE, + isCookieMode, + type CookieMode, +} from "./cookiePolicy"; export type { Worker } from "./workers"; export type { AutoQueueSettings } from "../jobs/autoQueuePolicy"; @@ -63,9 +68,18 @@ export type SiteSettings = { // app (see defaultWorkersFromApps). See common/lib/workers.ts. workers: Worker[]; // Browser spec (e.g. "firefox", "chrome:Default") passed to - // `yt-dlp --cookies-from-browser` ONLY on the auth-retry attempt of the - // managed per-video download flow. Empty string = disabled. + // `yt-dlp --cookies-from-browser`. WHEN it is passed is governed by + // `cookieMode` below. Empty string = no cookies configured. Per-channel + // override available (ChannelConfig.cookiesFromBrowser). cookiesFromBrowser: string; + // How yt-dlp invocations use the configured cookies (see + // common/lib/cookiePolicy.ts): "always" passes them on every invocation, + // "when-required" (default; the historical behavior) only to retry an + // auth/age failure, "defer" never in normal runs — auth-gated videos are + // excluded from batches and collected into the per-channel "Needs cookies" + // bucket for a manual cookie run. Per-channel override available + // (ChannelConfig.cookieMode). + cookieMode: CookieMode; // Pause (seconds) inserted between per-video yt-dlp invocations in // managed batch downloads. yt-dlp's own `-t sleep` only paces requests // within one invocation, so without this the managed loop hammers the @@ -455,6 +469,7 @@ function defaults(): SiteSettings { transcriptionApps: {}, workers: [], cookiesFromBrowser: "", + cookieMode: DEFAULT_COOKIE_MODE, sleepBetweenDownloadsSeconds: SLEEP_BETWEEN_DOWNLOADS_DEFAULT_SECONDS, downloadFormat: "auto", minFreeDiskGB: MIN_FREE_DISK_GB_DEFAULT, @@ -635,6 +650,11 @@ export function getSettings(): SiteSettings { if (typeof merged.cookiesFromBrowser !== "string") { merged.cookiesFromBrowser = ""; } + // A settings.json predating cookieMode (or carrying junk) gets the default, + // which preserves the historical retry-only behavior. + if (!isCookieMode(merged.cookieMode)) { + merged.cookieMode = DEFAULT_COOKIE_MODE; + } merged.sleepBetweenDownloadsSeconds = clampSleepBetweenDownloadsSeconds( merged.sleepBetweenDownloadsSeconds, ); @@ -826,6 +846,9 @@ export async function writeSettings(next: SiteSettings): Promise<void> { typeof next.cookiesFromBrowser === "string" ? next.cookiesFromBrowser.trim() : "", + cookieMode: isCookieMode(next.cookieMode) + ? next.cookieMode + : DEFAULT_COOKIE_MODE, sleepBetweenDownloadsSeconds: clampSleepBetweenDownloadsSeconds( next.sleepBetweenDownloadsSeconds, ), diff --git a/common/ytdlp/downloadOneManaged.ts b/common/ytdlp/downloadOneManaged.ts @@ -3,11 +3,17 @@ import { appendFile, mkdir, readdir, readFile, rm } from "node:fs/promises"; import { createWriteStream, type WriteStream } from "node:fs"; import { execa } from "execa"; import { + AUTH_RETRY_CLASSES, classifyDownloadFailure, parseUnavailableFromStderr, - type Availability, } from "../lib/availability"; import { + DEFAULT_COOKIE_MODE, + alwaysCookies, + authRetryCookies, + type ResolvedCookiePolicy, +} from "../lib/cookiePolicy"; +import { type AudioFormat, type ChannelConfig, } from "../lib/channelConfig"; @@ -82,9 +88,12 @@ export type ManagedDownloadOpts = { videoUrl: string; onLog: (s: string) => void; signal: AbortSignal; - // Resolved global setting; only used when the primary attempt fails with an - // auth/age error AND the channel doesn't already have its own cookies set. - globalCookiesFromBrowser?: string; + // Resolved cookie policy (value + mode), already collapsed channel-over- + // global by the caller (resolveCookiePolicy). Governs which attempts pass + // --cookies-from-browser: "always" on every invocation, "when-required" + // (the default when omitted) only on auth-retry passes, "defer" never — + // failures are recorded for the Needs-cookies bucket instead. + cookiePolicy?: ResolvedCookiePolicy; // When false, suppress appending to the channel archive file. Mirrors // `ignoreArchive` from the playlist-level callers. appendArchive?: boolean; @@ -349,12 +358,6 @@ function attemptSucceeded(exitCode: number | null): boolean { return exitCode === 0 || exitCode === 101; } -const AUTH_RETRY_CLASSES: ReadonlySet<Availability> = new Set<Availability>([ - "needs_auth", - "members_only", - "private", -]); - async function hasAnyTranscriptOnDisk(videoDir: string): Promise<boolean> { const entries = await readdir(videoDir).catch(() => [] as string[]); return entries.some((e) => { @@ -467,6 +470,17 @@ async function runManagedDownload( const attempts: DownloadAttempt[] = []; let status: DownloadOutcomeStatus = "failed"; let fellBackToTranscribe = false; + // Cookie policy for every yt-dlp spawn in this download. An omitted policy + // behaves like the historical default: no prophylactic cookies, retry-only + // (and with no value configured the retries degrade to no-ops). + const cookiePolicy: ResolvedCookiePolicy = opts.cookiePolicy ?? { + cookies: undefined, + mode: DEFAULT_COOKIE_MODE, + }; + // Set when a failed metadata prefetch was recovered by its cookie retry: the + // real download then gets cookies immediately (skipping a doomed cookie-less + // attempt) and a success is reported as "ok-with-cookies". + let prefetchNeededCookies = false; // Set when the download-time duration guard trips: the measured shortfall, // recorded on the outcome so the UI can explain it without re-probing. let shortAudioInfo: NonNullable<DownloadOutcomeRecord["shortAudio"]> | undefined; @@ -498,7 +512,10 @@ async function runManagedDownload( opts.channelConfig.platform ?? detectPlatform(opts.videoUrl); if (canonicalId && !opts.signal.aborted) { const videoDir = path.join(channelDir, "data", canonicalId); - const prefetchArgs = [ + // In "always" mode the prefetch (like every other invocation) carries the + // configured cookies up front. + const prefetchCookies = alwaysCookies(cookiePolicy); + const buildPrefetchArgs = (cookies: string | undefined) => [ "--ignore-config", "--restrict-filenames", ...outputArgsForUrl(opts.videoUrl), @@ -506,11 +523,15 @@ async function runManagedDownload( "--skip-download", "--no-write-subs", "--no-write-auto-subs", - ...channelConfigArgs(opts.channelConfig), + ...channelConfigArgs(opts.channelConfig, cookies), "--", opts.videoUrl, ]; - const prefetchRes = await runOneYtdlp(opts, channelDir, prefetchArgs); + const prefetchRes = await runOneYtdlp( + opts, + channelDir, + buildPrefetchArgs(prefetchCookies), + ); lastFullTail = prefetchRes.stderrTail; const prefetchAvail = attemptSucceeded(prefetchRes.exitCode) ? undefined @@ -519,7 +540,7 @@ async function runManagedDownload( n: 0, kind: "metadata-prefetch", handling: opts.channelConfig.handling, - usedCookies: false, + usedCookies: Boolean(prefetchCookies), ytdlpExitCode: prefetchRes.exitCode, availabilityClass: prefetchAvail, error: attemptSucceeded(prefetchRes.exitCode) @@ -527,6 +548,49 @@ async function runManagedDownload( : trimError(prefetchRes.stderrTail), }); + // Prefetch auth retry: a prefetch that failed with an auth/age error is + // re-run once with cookies (when the mode allows it and the failed pass + // didn't already use them). A recovery here lets the real download start + // with cookies immediately instead of burning a doomed cookie-less + // attempt first. Defer mode lands here with no retry cookies, so the + // failure stands and feeds the Needs-cookies bucket. + const prefetchRetryCookies = authRetryCookies(cookiePolicy); + if ( + !attemptSucceeded(prefetchRes.exitCode) && + prefetchAvail !== undefined && + AUTH_RETRY_CLASSES.has(prefetchAvail) && + prefetchRetryCookies !== undefined && + !prefetchCookies && + !opts.signal.aborted + ) { + opts.onLog( + `Metadata prefetch auth-required (${prefetchAvail}); retrying with --cookies-from-browser ${prefetchRetryCookies}\n`, + ); + const retryRes = await runOneYtdlp( + opts, + channelDir, + buildPrefetchArgs(prefetchRetryCookies), + ); + lastFullTail = retryRes.stderrTail; + const retryAvail = attemptSucceeded(retryRes.exitCode) + ? undefined + : parseUnavailableFromStderr(retryRes.stderrTail); + attempts.push({ + n: 0, + kind: "metadata-prefetch-auth-retry", + handling: opts.channelConfig.handling, + usedCookies: true, + ytdlpExitCode: retryRes.exitCode, + availabilityClass: retryAvail, + error: attemptSucceeded(retryRes.exitCode) + ? undefined + : trimError(retryRes.stderrTail), + }); + if (attemptSucceeded(retryRes.exitCode)) { + prefetchNeededCookies = true; + } + } + const metaPath = path.join(videoDir, "metadata.info.json"); const metadata = await loadRawMetadata(metaPath); // Only wire --load-info-json into the real attempts when we actually have @@ -629,6 +693,13 @@ async function runManagedDownload( } } + // Cookies for the real download attempts: always-mode passes them on every + // invocation; otherwise a cookie-recovered prefetch means this video needs + // them, so don't burn a doomed cookie-less attempt first. + const primaryCookies = + alwaysCookies(cookiePolicy) ?? + (prefetchNeededCookies ? cookiePolicy.cookies : undefined); + let primaryRes: AttemptOutcome; let audioCheckStats: AudioCheckAttemptStats | undefined; let audioCheckCorruptSource = false; @@ -652,7 +723,7 @@ async function runManagedDownload( ), "--print", `after_video:${ARCHIVE_MARKER} %(extractor)s %(id)s`, - ...channelConfigArgs(opts.channelConfig), + ...channelConfigArgs(opts.channelConfig, primaryCookies), "--", opts.videoUrl, ]; @@ -696,7 +767,7 @@ async function runManagedDownload( ...mediaArgs, "--print", `after_video:${ARCHIVE_MARKER} %(extractor)s %(id)s`, - ...channelConfigArgs(opts.channelConfig), + ...channelConfigArgs(opts.channelConfig, primaryCookies), ...sourceArgs(opts.videoUrl, infoJsonPath), ]; primaryRes = await runOneYtdlp(opts, channelDir, primaryArgs); @@ -710,7 +781,7 @@ async function runManagedDownload( n: 1, kind: audioCheckEnabled ? "audio-checked-primary" : "primary", handling: opts.channelConfig.handling, - usedCookies: false, + usedCookies: Boolean(primaryCookies), ytdlpExitCode: primaryRes.exitCode, availabilityClass: primaryAvail, error: attemptSucceeded(primaryRes.exitCode) @@ -725,7 +796,14 @@ async function runManagedDownload( !audioCheckCorruptFullSource && attemptSucceeded(primaryRes.exitCode); if (lastSucceeded) { - status = audioCheckEnabled ? "ok-audio-checked" : "ok"; + // "ok-with-cookies" keeps meaning "cookies were NEEDED" (the prefetch only + // succeeded with them) — always-mode prophylactic cookies on a clean + // success stay a plain "ok". + status = audioCheckEnabled + ? "ok-audio-checked" + : prefetchNeededCookies + ? "ok-with-cookies" + : "ok"; } else if (audioCheckCorruptSource) { status = "failed-corrupt-source"; } else if (audioCheckCorruptFullSource) { @@ -735,15 +813,20 @@ async function runManagedDownload( } // ---------- Attempt 2: auth retry ---------- + // Off in defer mode (authRetryCookies -> undefined), so the failure is + // recorded as-is and feeds the Needs-cookies bucket. Also skipped when the + // failed attempt already used cookies — retrying identically is futile. + const downloadRetryCookies = authRetryCookies(cookiePolicy); const shouldAuthRetry = !lastSucceeded && primaryAvail !== undefined && AUTH_RETRY_CLASSES.has(primaryAvail) && - Boolean(opts.globalCookiesFromBrowser); + downloadRetryCookies !== undefined && + !primaryCookies; if (shouldAuthRetry && !opts.signal.aborted) { opts.onLog( - `Auth-required (${primaryAvail}); retrying with --cookies-from-browser ${opts.globalCookiesFromBrowser}\n`, + `Auth-required (${primaryAvail}); retrying with --cookies-from-browser ${downloadRetryCookies}\n`, ); const retryArgs = [ "--ignore-config", @@ -752,7 +835,7 @@ async function runManagedDownload( ...mediaArgs, "--print", `after_video:${ARCHIVE_MARKER} %(extractor)s %(id)s`, - ...channelConfigArgs(opts.channelConfig, opts.globalCookiesFromBrowser), + ...channelConfigArgs(opts.channelConfig, downloadRetryCookies), ...sourceArgs(opts.videoUrl, infoJsonPath), ]; const retryRes = await runOneYtdlp(opts, channelDir, retryArgs); @@ -862,11 +945,12 @@ async function runManagedDownload( ...opts.channelConfig, handling: "transcribe", }; - // Use cookies if the auth retry succeeded with them; otherwise channel-level. + // Cookies for the fallback: always-mode passes them like every other + // invocation; otherwise only when this video demonstrably needed them + // (an attempt succeeded with cookies -> "ok-with-cookies"). const fallbackCookieOverride = - status === "ok-with-cookies" - ? opts.globalCookiesFromBrowser - : undefined; + alwaysCookies(cookiePolicy) ?? + (status === "ok-with-cookies" ? cookiePolicy.cookies : undefined); // Feed yt-dlp the metadata it already wrote during the primary // attempt instead of re-querying the extractor — saves a network // round-trip per video, which adds up across batch runs and helps diff --git a/common/ytdlp/runYtdlp.ts b/common/ytdlp/runYtdlp.ts @@ -20,6 +20,12 @@ import { classifyDownloadFailure, type Availability, } from "../lib/availability"; +import { + alwaysCookies, + cookieArgs, + resolveCookiePolicy, + type ResolvedCookiePolicy, +} from "../lib/cookiePolicy"; import { resolveEffectiveAvailability } from "../lib/availability-server"; import { backfillAvailabilityFromMetadata } from "../controller/backfillAvailability"; import { resolveShardItems } from "../controller/shard"; @@ -96,6 +102,11 @@ export type RunYtdlpOpts = { // saved playlist by extracted ID). IDs not present in the playlist are // reported and skipped. bucketIds?: ReadonlyArray<string>; + // retry-bucket only (the "Needs cookies" bucket button): force cookie mode + // "always" for this run, so every invocation carries the configured cookie + // value and the defer-mode batch exclusion is bypassed. Warns (and behaves + // like a plain run) when no cookie value is configured. + forceCookies?: boolean; // retry-bucket only: override channelConfig.handling for this run without // mutating the channel config on disk. Useful for retrying old "youtube" // videos as "transcribe". @@ -170,12 +181,46 @@ function abortableSleep(ms: number, signal: AbortSignal): Promise<void> { }); } -function configArgs(config: ChannelConfig): string[] { +function configArgs(config: ChannelConfig, cookies?: string): string[] { const args: string[] = []; + args.push(...cookieArgs(cookies)); if (config.ytdlpExtraArgs?.length) args.push(...config.ytdlpExtraArgs); return args; } +// The run-level cookie policy: settings + channel overrides, with the +// retry-bucket forceCookies flag overriding the mode to "always" (that's the +// "Download with cookies" button). Logs a warning when cookies are forced but +// no value is configured anywhere — the run then proceeds cookie-less. +function resolveRunCookiePolicy( + opts: RunYtdlpOpts, + channelConfig: ChannelConfig, +): ResolvedCookiePolicy { + const policy = resolveCookiePolicy(getSettings(), channelConfig); + if (!opts.forceCookies) return policy; + if (!policy.cookies) { + opts.onLog( + "Force-cookies run requested, but no cookies-from-browser value is configured (global or channel). Proceeding without cookies.\n", + ); + } + return { ...policy, mode: "always" }; +} + +// Defer-mode batch exclusion: a video whose effective availability is +// needs_auth is skipped by batch runs when the resolved cookie mode is +// "defer" (it lands in the snapshot's Needs-cookies bucket instead of being +// re-attempted on every sync/download-missing). members_only/private are +// already covered by EXCLUDED_FROM_DOWNLOAD. forceCookies runs resolve to +// mode "always" and therefore bypass this. +export async function isDeferredAuthExcluded( + videoDir: string, + policy: ResolvedCookiePolicy, +): Promise<boolean> { + if (policy.mode !== "defer") return false; + const cls = await resolveEffectiveAvailability(videoDir); + return cls === "needs_auth"; +} + const OUTPUT_ARGS: string[] = [ "-o", "data/%(id)s/audio.%(ext)s", @@ -247,7 +292,15 @@ async function enumeratePlaylistUrls( if (range) { args.push("--lazy-playlist", "-I", `${range.start}:${range.end}`); } - args.push(...configArgs(opts.channelConfig), opts.channelConfig.url!); + // Cookie mode "always" covers enumeration too (a members-only or otherwise + // gated channel may not even list without cookies). + args.push( + ...configArgs( + opts.channelConfig, + alwaysCookies(resolveRunCookiePolicy(opts, opts.channelConfig)), + ), + opts.channelConfig.url!, + ); opts.onLog(`$ ${opts.paths.ytdlpBin} ${args.join(" ")}\n`); const child = execa(opts.paths.ytdlpBin, args, { @@ -436,8 +489,14 @@ async function downloadPlaylistManaged( // Exclude videos whose effective availability marks them as permanently // unavailable (members_only, deleted, private). Applies to both prefilter // modes: failed download attempts don't write to the archive, so the - // archive-prefilter path would also keep retrying them. + // archive-prefilter path would also keep retrying them. Under cookie mode + // "defer", needs_auth videos are excluded too — they wait in the snapshot's + // Needs-cookies bucket for a manual cookie run instead of being re-attempted + // (and re-failing) on every batch. A forceCookies run resolves to mode + // "always" and so bypasses the defer exclusion. + const runCookiePolicy = resolveRunCookiePolicy(opts, effectiveChannelConfig); const excludedCounts = { members_only: 0, deleted: 0, private: 0 }; + let deferredAuthCount = 0; const filteredTofetch: string[] = []; for (const url of tofetch) { const dirId = extractVideoId(url); @@ -452,6 +511,10 @@ async function downloadPlaylistManaged( excludedCounts[cls as keyof typeof excludedCounts]++; continue; } + if (runCookiePolicy.mode === "defer" && cls === "needs_auth") { + deferredAuthCount++; + continue; + } } filteredTofetch.push(url); } @@ -464,6 +527,11 @@ async function downloadPlaylistManaged( `Excluded ${totalExcluded} from this run (members_only=${excludedCounts.members_only}, deleted=${excludedCounts.deleted}, private=${excludedCounts.private}). Clear via Diagnostics > Recheck if a video became public again.\n`, ); } + if (deferredAuthCount > 0) { + opts.onLog( + `Cookie mode is "defer": needs_auth deferred=${deferredAuthCount} — run the "Needs cookies" bucket to download them with cookies.\n`, + ); + } tofetch.length = 0; tofetch.push(...filteredTofetch); @@ -539,7 +607,12 @@ async function downloadPlaylistManaged( } const { okCount, failedCount, skippedCount, processedCount, firstFailure } = - await runManagedDownloads(opts, items, effectiveChannelConfig); + await runManagedDownloads( + opts, + items, + effectiveChannelConfig, + runCookiePolicy, + ); if (firstFailure && !opts.signal.aborted) { throw firstFailure; } @@ -575,9 +648,15 @@ async function runManagedDownloads( opts: RunYtdlpOpts, urls: ReadonlyArray<string>, effectiveChannelConfig: ChannelConfig, + // Cookie policy for every managed download in this run. Callers that + // already resolved it (for their own prefilter) pass it through so the + // forceCookies-without-value warning isn't logged twice. + cookiePolicyOverride?: ResolvedCookiePolicy, ): Promise<ManagedRunResult> { const settings = getSettings(); - const globalCookies = settings.cookiesFromBrowser; + const cookiePolicy = + cookiePolicyOverride ?? + resolveRunCookiePolicy(opts, effectiveChannelConfig); const inlineTranscribeOnFallback = settings.inlineTranscribeOnFallback; const globalSkipLiveDownloads = settings.skipLiveDownloads; // Resolve the download format once per run (override > channel > global), @@ -644,7 +723,7 @@ async function runManagedDownloads( videoUrl: url, onLog: task ? task.onLog : opts.onLog, signal: opts.signal, - globalCookiesFromBrowser: globalCookies || undefined, + cookiePolicy, appendArchive: !opts.ignoreArchive, inlineTranscribeOnFallback, globalSkipLiveDownloads, @@ -782,7 +861,10 @@ async function downloadSubsForUrl( "--skip-download", "--no-write-info-json", "--no-overwrites", - ...configArgs(opts.channelConfig), + ...configArgs( + opts.channelConfig, + alwaysCookies(resolveRunCookiePolicy(opts, opts.channelConfig)), + ), "--", url, ]; @@ -951,7 +1033,10 @@ async function downloadOneAudio(opts: RunYtdlpOpts): Promise<void> { "--audio-format", fmt, ...(opts.channelConfig.keepSourceVideo ? ["-k"] : []), - ...configArgs(opts.channelConfig), + ...configArgs( + opts.channelConfig, + alwaysCookies(resolveRunCookiePolicy(opts, opts.channelConfig)), + ), ...(opts.extraYtdlpArgs ?? []), "--", opts.singleVideoUrl, @@ -1013,6 +1098,9 @@ async function sync(opts: RunYtdlpOpts): Promise<void> { let totalFailed = 0; let totalSkipped = 0; let firstFailure: Error | null = null; + // Resolved once for the whole sync: drives both the defer-mode needs_auth + // exclusion below and the managed downloads themselves. + const runCookiePolicy = resolveRunCookiePolicy(opts, opts.channelConfig); for ( let page = 0; @@ -1026,20 +1114,44 @@ async function sync(opts: RunYtdlpOpts): Promise<void> { const newUrls: string[] = []; let archivedHits = 0; + let deferredAuthCount = 0; for (const url of pageUrls) { const archiveId = await archiveIdForUrl(url, dataDir); if (archiveId && archive.ids.has(archiveId)) { archivedHits++; continue; } + // Cookie mode "defer": don't re-attempt known auth-gated videos on + // every sync — they wait in the Needs-cookies bucket instead. + const dirId = extractVideoId(url); + if ( + dirId && + (await isDeferredAuthExcluded( + path.join(dataDir, dirId), + runCookiePolicy, + )) + ) { + deferredAuthCount++; + continue; + } newUrls.push(url); } opts.onLog( `Sync page ${page + 1}: ${pageUrls.length} entries, ${newUrls.length} new, ${archivedHits} already archived.\n`, ); + if (deferredAuthCount > 0) { + opts.onLog( + `Cookie mode is "defer": needs_auth deferred=${deferredAuthCount} — run the "Needs cookies" bucket to download them with cookies.\n`, + ); + } if (newUrls.length > 0) { - const res = await runManagedDownloads(opts, newUrls, opts.channelConfig); + const res = await runManagedDownloads( + opts, + newUrls, + opts.channelConfig, + runCookiePolicy, + ); totalNew += res.okCount; totalFailed += res.failedCount; totalSkipped += res.skippedCount; diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## [Unreleased] +- **Cookies-from-browser is now configurable: three cookie modes, per-channel overrides, and a "Needs cookies" bucket.** The single global cookies value used to be hard-wired to one behavior (passed only on the download auth-retry attempt). Settings now carry a **Cookie mode** next to the value — **When required** (default; the old behavior, now also cookie-retrying a failed *metadata prefetch*, so an age-gated video succeeds on its first real attempt and reports `ok-with-cookies` with a `metadata-prefetch-auth-retry` attempt in `download-outcome.json`), **Always** (every yt-dlp invocation carries `--cookies-from-browser`: enumeration, prefetch, availability probes, downloads — for channels whose listing itself needs auth; a clean success still reports plain `ok`), and **Defer** (normal runs never use cookies: known `needs_auth` videos are excluded from sync/download-missing batches — previously they were re-attempted and re-failed on *every* run — and wait for a manual cookie run). Both the value and the mode can be overridden per channel in the channel form's Advanced section (blank/Inherit = use global). Every channel's Download stage gains a **Needs cookies (N)** card listing undownloaded videos whose recorded availability is cookie-recoverable (`needs_auth`, and also `members_only`/`private`, which batches always excluded but a subscribed/owning account's cookies can fetch) with a **Download with cookies** button that re-runs them with cookies forced on every invocation (bookmarkable, like the other bucket jobs; it warns if no cookie value is configured anywhere). Caveats: in defer mode a *manual single-video* download intentionally gets no cookies and no auth retry — the bucket button is the explicit cookie path; availability probes only attach cookies in Always mode, so needs_auth keeps being observed (defer depends on that signal). Back-compat: an existing `settings.json` without `cookieMode` behaves exactly as before (`when-required`). See `common/lib/cookiePolicy.ts`, `common/ytdlp/{downloadOneManaged,runYtdlp}.ts`, `common/controller/{channelSnapshot,checkAvailability}.ts`, and `editor/e2e/cookies-mode.spec.ts`. - **Channel forms now edit site membership directly — pick sites, groups, and create groups inline.** A channel's site membership used to be editable only from the Site form (`/sites/<id>`), and creating a channel silently appended it to the active site (a hidden `activeSite` field) with no group choice and no visibility. Both the **New channel** and channel **Configure** forms now carry a **Sites** section: every configured site listed with a membership checkbox and a compact group dropdown — `(default)`, any existing group, or **"+ New group…"**, which reveals a name input and creates the group on that site (id slugified from the name, reused if it already exists) as part of the save. On create, the active site (`?site=`) is pre-checked, reproducing the old behavior but visibly and overridably; on edit, current memberships pre-check with their groups, and unchecking removes the membership. The checked sites serialize into one hidden `siteMembershipsJson` field (the SiteForm hidden-JSON precedent); the server plans all writes up front — validating group ids, preserving other channels' entries and this channel's `order`, erroring clearly on a since-deleted group, skipping since-deleted sites, and rewriting only sites that actually changed. See the new `editor/app/channels/{lib/siteMemberships.ts,components/SiteMembershipsSection.tsx}`, `editor/app/channels/{actions.ts,components/{ChannelForm,ChannelFormClient}.tsx,new/page.tsx,[slug]/page.tsx}`, and `editor/e2e/channel-site-membership.spec.ts`. - **The monitor widget can now start a sync and shows sync freshness + scheduler health — each behind its own flag.** The widget was start-a-sync-less and said nothing about how fresh your channels were. Five new opt-in URL flags, all toggleable in the builder and the in-widget gear (all default off, so existing links are unchanged): **(1) a channel-aware Sync button** (`sync=1`) — pinned to one channel (`channel=X`) it runs that channel's streaming sync; otherwise it sweeps all channels (`syncAllChannelsAction`), briefly showing `Queued X · skipped Y`. **(2) A last-sync readout** (`lastsync=1`) — `Last full sync: …` from a new persisted `lastSyncAllAt` marker written at the end of each Sync-all sweep, plus a `Last channel sync: …` line when an individual channel synced more recently. **(3) A scheduler-status strip** (`sched=1`) — `Auto-sync on/off · next … · last run …`, derived from the same `buildScheduleView` the `/scheduler` page uses. **(4) An absolute-time toggle** (`abstime=1`) — readouts show locale timestamps instead of relative "5m ago". **(5) A Sync-all confirm** (`syncask=1`) — a `window.confirm` before a full sweep. The two readouts poll a new lightweight `/api/widget/sync` route (a few scalars, 15s floor) and, with the Sync button/controls, keep the widget from collapsing to "Idle" — sync health is exactly what you check when nothing's running. See `editor/app/widget/lib/config.ts`, `editor/app/widget/components/{MonitorWidget,WidgetControls,WidgetConfigForm}.tsx`, the new `editor/app/api/widget/sync/route.ts`, `common/jobs/syncSchedulerState.ts` (`lastSyncAllAt`), `editor/app/channels/actions.ts`, and `editor/e2e/widget.spec.ts`. - **The audio-integrity check now tightens its probe interval when a source starts serving corruption, then relaxes as it stabilises.** Previously the integrity probe ran on a fixed cadence (default 60s) for the whole download, so up to ~60s of bytes were downloaded — and discarded — between a corruption event and the checkpoint that caught it. The interval is now adaptive (AIMD, like TCP congestion control, inverted): each **malformed** checkpoint **halves** the live interval (60→30→15→10s, floored at the existing `AUDIO_CHECK_INTERVAL_MIN_SECONDS` of 10s), so a misbehaving source gets probed more aggressively and wastes fewer bytes per rollback; a run of clean checkpoints then **steps it back up** additively (+15s after every 2 clean probes) toward the configured interval. The reduced cadence persists across yt-dlp relaunches for the rest of the download run. Fully backward compatible — a clean download never leaves the configured interval. Tunable via constants in `common/lib/channelConfig.ts` (`AUDIO_CHECK_INTERVAL_BACKOFF_FACTOR_DEFAULT`, `AUDIO_CHECK_INTERVAL_RECOVER_STEP_SECONDS`, `AUDIO_CHECK_INTERVAL_RECOVER_AFTER_CLEAN`) plus test-only env overrides. See `common/ytdlp/audioCheckCadence.ts` (pure AIMD math + `audioCheckCadence.test.ts`) and `common/ytdlp/audioCheckedDownload.ts` (`resolveKnobs`, the watcher loop, and the advance/malformed checkpoint branches). diff --git a/editor/app/channels/[slug]/components/RetryBucketControl.tsx b/editor/app/channels/[slug]/components/RetryBucketControl.tsx @@ -17,6 +17,9 @@ type Props = { // The snapshot bucket these ids represent. When set, the launched retry job // is bookmarkable and re-runs against the current bucket members. bucketKey?: ReplayBucket; + // Needs-cookies bucket: force cookie mode "always" for the run and label the + // button "Download with cookies" so the manual cookie path is explicit. + forceCookies?: boolean; }; export function RetryBucketControl({ @@ -26,6 +29,7 @@ export function RetryBucketControl({ defaultQueueKey, existingQueues, bucketKey, + forceCookies, }: Props) { const [queue, setQueue] = useState(defaultQueueKey); const [handlingOverride, setHandlingOverride] = useState(""); @@ -47,10 +51,15 @@ export function RetryBucketControl({ abortOnError, handlingOverride || undefined, bucketKey, + forceCookies, ) } cancelAction={cancelJobAction} - buttonLabel={`Retry (${ids.length})`} + buttonLabel={ + forceCookies + ? `Download with cookies (${ids.length})` + : `Retry (${ids.length})` + } runningLabel="Retrying…" label={`Retry ${actionLabel}`} extraControls={ diff --git a/editor/app/channels/[slug]/components/stages/DownloadStage.tsx b/editor/app/channels/[slug]/components/stages/DownloadStage.tsx @@ -28,6 +28,7 @@ type Props = { excludedFromDownload: ExcludedFromDownload; noTranscriptIds: string[]; partialDownloadIds: string[]; + needsCookiesIds: string[]; missingShard: ShardConfigSummary | null; }; @@ -40,6 +41,7 @@ export function DownloadStage({ excludedFromDownload, noTranscriptIds, partialDownloadIds, + needsCookiesIds, missingShard, }: Props) { const [downloadQueue, setDownloadQueue] = useState(defaultQueueKey); @@ -217,6 +219,12 @@ export function DownloadStage({ defaultQueueKey={defaultQueueKey} existingQueues={existingQueues} /> + <NeedsCookiesList + slug={slug} + ids={needsCookiesIds} + defaultQueueKey={defaultQueueKey} + existingQueues={existingQueues} + /> </div> ); } @@ -416,6 +424,54 @@ function PartialDownloadsList({ ); } +function NeedsCookiesList({ + slug, + ids, + defaultQueueKey, + existingQueues, +}: { + slug: string; + ids: string[]; + defaultQueueKey: string; + existingQueues: string[]; +}) { + if (ids.length === 0) return null; + return ( + <div className="flex flex-col gap-2 rounded border border-border p-3"> + <div> + <h4 className="text-sm font-semibold"> + Needs cookies ({ids.length}) + </h4> + <p className="text-xs text-muted-foreground"> + Undownloaded videos that yt-dlp reported as auth-gated + (age-restricted, members-only, or private) — browser cookies from a + signed-in account may recover them. The button below re-runs them + with <code>--cookies-from-browser</code> forced on every invocation, + using the cookie value from Settings (or this channel's + override). + </p> + </div> + <VideoIdList + slug={slug} + ids={ids} + ariaLabel="needs cookies list" + emptyAriaLabel="needs cookies empty" + emptyMessage="None" + itemAriaLabel={(id) => `needs cookies ${id}`} + /> + <RetryBucketControl + slug={slug} + ids={ids} + actionLabel="needs cookies" + defaultQueueKey={defaultQueueKey} + existingQueues={existingQueues} + bucketKey="needsCookies" + forceCookies + /> + </div> + ); +} + function Heading({ title, desc }: { title: string; desc: string }) { return ( <div> diff --git a/editor/app/channels/[slug]/lib/stageStatus.ts b/editor/app/channels/[slug]/lib/stageStatus.ts @@ -30,6 +30,7 @@ export function normalizeBuckets( skippedByFilter: raw?.skippedByFilter ?? [], incompleteTranscript: raw?.incompleteTranscript ?? [], shortAudio: raw?.shortAudio ?? [], + needsCookies: raw?.needsCookies ?? [], }; } diff --git a/editor/app/channels/[slug]/page.tsx b/editor/app/channels/[slug]/page.tsx @@ -277,6 +277,7 @@ export default async function ChannelDetailPage({ excludedFromDownload={excludedFromDownload} noTranscriptIds={actionableNoTranscriptIds} partialDownloadIds={buckets.partialDownloads} + needsCookiesIds={buckets.needsCookies} missingShard={downloadMissingShard} /> ), diff --git a/editor/app/channels/[slug]/pipelineActions.ts b/editor/app/channels/[slug]/pipelineActions.ts @@ -24,6 +24,7 @@ import { import { extractVideoId, runYtdlp } from "yt-dlp-transcript-common/ytdlp/runYtdlp"; import { downloadOneManaged } from "yt-dlp-transcript-common/ytdlp/downloadOneManaged"; import { getSettings } from "yt-dlp-transcript-common/lib/settings"; +import { resolveCookiePolicy } from "yt-dlp-transcript-common/lib/cookiePolicy"; import { checkDiskSpace } from "yt-dlp-transcript-common/lib/diskSpace"; import { formatBytes } from "yt-dlp-transcript-common/lib/format"; import { @@ -70,6 +71,10 @@ async function runPipelineAction( shardIndex?: number; bucketIds?: ReadonlyArray<string>; handlingOverride?: ChannelHandling; + // retry-bucket only (the "Needs cookies" bucket): force cookie mode + // "always" for this run so every yt-dlp invocation carries the configured + // cookies and the defer-mode exclusion is bypassed. + forceCookies?: boolean; // Per-run persistence overrides (Phase 2), forwarded to runYtdlp → // downloadOneManaged for every managed download in this run. keepSourceVideoOverride?: boolean; @@ -166,6 +171,7 @@ async function runPipelineAction( shardIndex: options?.shardIndex, bucketIds: options?.bucketIds, handlingOverride: options?.handlingOverride, + forceCookies: options?.forceCookies, keepSourceVideoOverride: options?.keepSourceVideoOverride, extractImmediately: options?.extractImmediately, audioFormatOverride: options?.audioFormatOverride, @@ -302,6 +308,8 @@ export async function retryBucketAction( // against the CURRENT bucket; omitted by ad-hoc checkbox selections, which are // therefore not bookmarkable. bucketKey?: ReplayBucket, + // Needs-cookies bucket only: run with cookie mode forced to "always". + forceCookies?: boolean, ): Promise<StreamActionResult> { if (!Array.isArray(bucketIds) || bucketIds.length === 0) { return { ok: false, error: "No video IDs supplied for retry." }; @@ -321,7 +329,7 @@ export async function retryBucketAction( kind: "retry-bucket", slug, bucket: bucketKey, - params: { queueKey, abortOnError, handlingOverride }, + params: { queueKey, abortOnError, handlingOverride, forceCookies }, } : undefined; return runPipelineAction( @@ -329,7 +337,13 @@ export async function retryBucketAction( "retry-bucket", "retry-bucket", queueKey, - { bucketIds, handlingOverride: handling, abortOnError, spec }, + { + bucketIds, + handlingOverride: handling, + abortOnError, + forceCookies, + spec, + }, ); } @@ -380,7 +394,7 @@ export async function importVideoAction( videoUrl, onLog: task.onLog, signal, - globalCookiesFromBrowser: settings.cookiesFromBrowser || undefined, + cookiePolicy: resolveCookiePolicy(settings, channelConfig), inlineTranscribeOnFallback: settings.inlineTranscribeOnFallback, globalSkipLiveDownloads: settings.skipLiveDownloads, appendArchive: true, diff --git a/editor/app/channels/[slug]/videos/[id]/videoActions.ts b/editor/app/channels/[slug]/videos/[id]/videoActions.ts @@ -36,6 +36,7 @@ import { transcribeWithWorker } from "yt-dlp-transcript-common/controller/transc import { findVideoSourceUrl } from "yt-dlp-transcript-common/controller/undownloadedVideos"; import { unpersistSavedVideo } from "yt-dlp-transcript-common/lib/savedVideo-server"; import { getSettings } from "yt-dlp-transcript-common/lib/settings"; +import { resolveCookiePolicy } from "yt-dlp-transcript-common/lib/cookiePolicy"; import { downloadOneManaged } from "yt-dlp-transcript-common/ytdlp/downloadOneManaged"; import { runYtdlp } from "yt-dlp-transcript-common/ytdlp/runYtdlp"; import { @@ -172,7 +173,7 @@ export async function downloadVideoPipelineAction( videoUrl: url, onLog: task.onLog, signal, - globalCookiesFromBrowser: settings.cookiesFromBrowser || undefined, + cookiePolicy: resolveCookiePolicy(settings, r.config), inlineTranscribeOnFallback: settings.inlineTranscribeOnFallback, globalSkipLiveDownloads: settings.skipLiveDownloads, downloadFormatPreset: resolveDownloadFormatPreset({ @@ -237,7 +238,7 @@ export async function redownloadToArchiveAction( videoUrl: url, onLog: task.onLog, signal, - globalCookiesFromBrowser: settings.cookiesFromBrowser || undefined, + cookiePolicy: resolveCookiePolicy(settings, r.config), inlineTranscribeOnFallback: settings.inlineTranscribeOnFallback, globalSkipLiveDownloads: settings.skipLiveDownloads, appendArchive: true, diff --git a/editor/app/channels/components/ChannelForm.tsx b/editor/app/channels/components/ChannelForm.tsx @@ -251,6 +251,39 @@ export function ChannelForm({ className="rounded border border-border bg-card px-2 py-1 text-sm font-mono" /> </label> + <Field + label="Cookies from browser" + name="cookiesFromBrowser" + defaultValue={c?.cookiesFromBrowser ?? ""} + placeholder="(inherit global)" + hint="Per-channel override of the global yt-dlp --cookies-from-browser browser spec (e.g. firefox, chrome:Default). Blank inherits the global value from /settings." + /> + <label className="flex flex-col gap-1 text-sm"> + <span className="font-medium">Cookie mode</span> + <select + name="cookieMode" + defaultValue={c?.cookieMode ?? ""} + aria-label="cookie mode" + className="rounded border border-border bg-card px-2 py-1 text-sm" + > + <option value="">Inherit global</option> + <option value="when-required"> + When required — retry auth/age failures with cookies + </option> + <option value="always"> + Always — pass cookies on every yt-dlp invocation + </option> + <option value="defer"> + Defer — never in normal runs; collect into "Needs cookies" + </option> + </select> + <span className="text-xs text-muted-foreground"> + When this channel's downloads use browser cookies. Leave on + Inherit to use the global mode from /settings. Defer excludes + auth-gated videos from batch runs and gathers them into the + “Needs cookies” bucket on the Download stage. + </span> + </label> <label className="flex flex-col gap-1 text-sm"> <span className="font-medium"> Sleep between managed downloads diff --git a/editor/app/channels/components/parseChannelForm.ts b/editor/app/channels/components/parseChannelForm.ts @@ -17,6 +17,7 @@ import { detectPlatform, type Platform, } from "yt-dlp-transcript-common/lib/platform"; +import { isCookieMode } from "yt-dlp-transcript-common/lib/cookiePolicy"; import { isDownloadFormatPreset } from "yt-dlp-transcript-common/ytdlp/downloadFormat"; export type ParsedChannelForm = { @@ -41,6 +42,8 @@ export const CHANNEL_FORM_FIELDS = [ "ytdlpExtraArgs", "syncIntervalMinutes", "sleepBetweenDownloadsSeconds", + "cookiesFromBrowser", + "cookieMode", "audioCheck", ] as const satisfies ReadonlyArray<keyof ChannelConfig>; @@ -143,6 +146,12 @@ export function parseChannelForm(formData: FormData): ParsedChannelForm { syncIntervalMinutes = n; } + // Cookie overrides. Blank value / "Inherit global" mode = omit, so a cleared + // input actually clears the stored override (see CHANNEL_FORM_FIELDS). + const cookiesFromBrowser = stringOrUndef(formData, "cookiesFromBrowser"); + const cookieModeRaw = String(formData.get("cookieMode") ?? "").trim(); + const cookieMode = isCookieMode(cookieModeRaw) ? cookieModeRaw : undefined; + const audioCheckEnabled = formData.get("audioCheckEnabled") != null; let audioCheck: AudioCheckConfig | undefined; if (audioCheckEnabled) { @@ -221,6 +230,8 @@ export function parseChannelForm(formData: FormData): ParsedChannelForm { if (sleepBetweenDownloadsSeconds != null) { config.sleepBetweenDownloadsSeconds = sleepBetweenDownloadsSeconds; } + if (cookiesFromBrowser) config.cookiesFromBrowser = cookiesFromBrowser; + if (cookieMode) config.cookieMode = cookieMode; if (audioCheck) config.audioCheck = audioCheck; return { name, slug, config }; diff --git a/editor/app/jobs/jobReplayRegistry.ts b/editor/app/jobs/jobReplayRegistry.ts @@ -125,6 +125,7 @@ export const JOB_REPLAY_HANDLERS: Record<string, ReplayHandler> = { bool(p.abortOnError), str(p.handlingOverride), spec.bucket, + Boolean(p.forceCookies), ); }, "download-from-playlist": (spec) => { diff --git a/editor/app/settings/actions.ts b/editor/app/settings/actions.ts @@ -21,6 +21,10 @@ import { type SocialLink, } from "yt-dlp-transcript-common/lib/settings"; import { DEFAULT_TRANSCRIPTION_APP_ID } from "yt-dlp-transcript-common/lib/transcriptionApps"; +import { + DEFAULT_COOKIE_MODE, + isCookieMode, +} from "yt-dlp-transcript-common/lib/cookiePolicy"; import { isDownloadFormatPreset } from "yt-dlp-transcript-common/ytdlp/downloadFormat"; import { sanitizeWorkers, @@ -40,6 +44,10 @@ export async function saveSettingsAction( const cookiesFromBrowser = String( formData.get("cookiesFromBrowser") ?? "", ).trim(); + const cookieModeRaw = String(formData.get("cookieMode") ?? "").trim(); + const cookieMode = isCookieMode(cookieModeRaw) + ? cookieModeRaw + : DEFAULT_COOKIE_MODE; const sleepRaw = String( formData.get("sleepBetweenDownloadsSeconds") ?? "", ).trim(); @@ -213,6 +221,7 @@ export async function saveSettingsAction( transcriptionApps: {}, workers, cookiesFromBrowser, + cookieMode, sleepBetweenDownloadsSeconds: sleepParsed, downloadFormat, minFreeDiskGB: minFreeDiskParsed, diff --git a/editor/app/settings/components/SettingsForm.tsx b/editor/app/settings/components/SettingsForm.tsx @@ -74,11 +74,40 @@ export function SettingsForm({ initial, apps }: Props) { <WorkersField initial={initial.workers} apps={apps} name="workersJson" /> </fieldset> <Field - label="Cookies from browser (retry only)" + label="Cookies from browser" name="cookiesFromBrowser" defaultValue={initial.cookiesFromBrowser} - hint="Browser spec passed to yt-dlp --cookies-from-browser only when the primary download attempt fails with an auth/age error and the channel doesn't have its own cookies set. e.g. firefox, chrome:Default. Leave blank to disable." + hint="Browser spec passed to yt-dlp --cookies-from-browser (e.g. firefox, chrome:Default) for age-restricted, members-only, and private videos. When it is used is set by the cookie mode below. Leave blank for no cookies. Each channel can override both in its Advanced settings." /> + <label className="flex flex-col gap-1 text-sm"> + <span className="font-medium">Cookie mode</span> + <select + name="cookieMode" + defaultValue={initial.cookieMode} + className="rounded border border-border bg-card px-2 py-1 text-sm" + > + <option value="when-required"> + When required — retry auth/age failures with cookies + </option> + <option value="always"> + Always — pass cookies on every yt-dlp invocation + </option> + <option value="defer"> + Defer — never in normal runs; collect into "Needs cookies" + </option> + </select> + <span className="text-xs text-muted-foreground"> + <strong>When required</strong> (default) runs cookie-free and retries + only the attempts that fail with an auth/age error.{" "} + <strong>Always</strong> sends cookies with every invocation + (enumeration, metadata prefetch, availability probes, downloads) — + for channels whose listing itself needs auth.{" "} + <strong>Defer</strong> never uses cookies in normal runs: auth-gated + videos are excluded from batches and gathered into each + channel's “Needs cookies” bucket, downloaded on + demand with its “Download with cookies” button. + </span> + </label> <Field label="Sleep between managed downloads (seconds)" name="sleepBetweenDownloadsSeconds" diff --git a/editor/e2e/cookies-mode.spec.ts b/editor/e2e/cookies-mode.spec.ts @@ -0,0 +1,331 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; +import { test, expect } from "@playwright/test"; +import { + pathExists, + readJson, + resetData, + resolvePath, + writeSettings, +} from "./helpers"; + +// Configurable cookies-from-browser: global + per-channel value, and a cookie +// MODE (always / when-required / defer). The fake yt-dlp's `cookiegated` +// sentinel fails any metadata/download invocation without +// --cookies-from-browser (yt-dlp's age-gate error) and succeeds with it, and +// every relevant invocation line logs `cookies=<value>` so flag presence can +// be asserted per spawn. + +const CHANNEL = "test-cookies"; +const FIXTURE = "cookies-mode-channel"; +const ROOT = `test-transcripts/channels/${CHANNEL}`; +const COOKIES = "firefox:test"; +const here = path.dirname(fileURLToPath(import.meta.url)); + +type DownloadOutcome = { + status: string; + attempts: Array<{ kind: string; usedCookies: boolean }>; +}; + +type Snapshot = { + buckets?: { needsCookies?: string[] }; +}; + +async function defaultSettings(): Promise<Record<string, unknown>> { + const raw = await readFile( + path.join(here, "fixtures", "test-settings.default.json"), + "utf8", + ); + return JSON.parse(raw) as Record<string, unknown>; +} + +async function readInvocations(): Promise<string> { + try { + return await readFile( + path.join(here, "..", ROOT, "fake-ytdlp.invocations"), + "utf8", + ); + } catch { + return ""; + } +} + +// The download job arms a debounced (~1s) snapshot regen; poll snapshot.json +// until the needsCookies bucket reaches the expected size before reloading. +async function waitForNeedsCookiesBucket(size: number): Promise<void> { + await expect + .poll( + async () => { + const snap = await readJson<Snapshot>(`${ROOT}/snapshot.json`).catch( + () => null, + ); + return snap?.buckets?.needsCookies?.length ?? -1; + }, + { timeout: 15_000 }, + ) + .toBe(size); +} + +test("settings: cookie value + mode round-trip through the form", async ({ + page, +}) => { + await resetData(FIXTURE); + await page.goto("/settings"); + await page.getByLabel(/^cookies from browser/i).fill(COOKIES); + // The value field's hint mentions "cookie mode", so a label lookup is + // ambiguous — target the select by name. + const modeSelect = page.locator('select[name="cookieMode"]'); + await modeSelect.selectOption("always"); + await page.getByRole("button", { name: /save settings/i }).click(); + await expect( + page.getByRole("status").filter({ hasText: "Saved" }), + ).toBeVisible(); + + const saved = await readJson<{ + cookiesFromBrowser?: string; + cookieMode?: string; + }>("test-settings.json"); + expect(saved.cookiesFromBrowser).toBe(COOKIES); + expect(saved.cookieMode).toBe("always"); + + await page.reload(); + await expect(page.getByLabel(/^cookies from browser/i)).toHaveValue(COOKIES); + await expect(modeSelect).toHaveValue("always"); +}); + +test("channel form: overrides persist and clear back to inherit", async ({ + page, +}) => { + await resetData(FIXTURE); + await page.goto(`/channels/${CHANNEL}`); + await page.locator("summary").filter({ hasText: "Advanced" }).click(); + await page.getByLabel(/^cookies from browser/i).fill("chrome:Profile 1"); + await page.getByLabel("cookie mode").selectOption("defer"); + await page.getByRole("button", { name: /save changes/i }).click(); + + await expect + .poll(async () => + readJson<{ cookiesFromBrowser?: string; cookieMode?: string }>( + `${ROOT}/config.json`, + ), + ) + .toMatchObject({ + cookiesFromBrowser: "chrome:Profile 1", + cookieMode: "defer", + }); + + // Clear both back to inherit. CHANNEL_FORM_FIELDS layering must actually + // remove the stored values, not re-layer the stale ones. + await page.reload(); + await page.locator("summary").filter({ hasText: "Advanced" }).click(); + const cookiesField = page.getByLabel(/^cookies from browser/i); + await expect(cookiesField).toHaveValue("chrome:Profile 1"); + await cookiesField.fill(""); + await page.getByLabel("cookie mode").selectOption(""); + await page.getByRole("button", { name: /save changes/i }).click(); + + await expect + .poll(async () => { + const config = await readJson<Record<string, unknown>>( + `${ROOT}/config.json`, + ); + return { + cookiesFromBrowser: config.cookiesFromBrowser, + cookieMode: config.cookieMode, + }; + }) + .toEqual({ cookiesFromBrowser: undefined, cookieMode: undefined }); +}); + +test("always mode: every yt-dlp invocation carries the cookie value", async ({ + page, +}) => { + test.setTimeout(120_000); + await resetData(FIXTURE); + await writeSettings({ + ...(await defaultSettings()), + cookiesFromBrowser: COOKIES, + cookieMode: "always", + }); + await page.goto(`/channels/${CHANNEL}`); + await page.getByRole("button", { name: "Download videos" }).click(); + const log = page.getByLabel("Download videos output"); + await expect(log).toContainText( + "Managed download complete: 2 succeeded, 0 failed", + { timeout: 60_000 }, + ); + + const invocations = await readInvocations(); + expect(invocations).toMatch( + new RegExp(`prefetch:.*vidcookiegated1 cookies=${COOKIES}`), + ); + expect(invocations).toMatch( + new RegExp(`prefetch:.*normalvideo1 cookies=${COOKIES}`), + ); + expect(invocations).toMatch( + new RegExp(`youtube-single:.*vidcookiegated1 cookies=${COOKIES}`), + ); + expect(invocations).toMatch( + new RegExp(`youtube-single:.*normalvideo1 cookies=${COOKIES}`), + ); + // No cookie-less invocation lines at all in always mode. + expect(invocations).not.toMatch(/cookies=$/m); + + // Prophylactic cookies on a clean success stay "ok" — "ok-with-cookies" + // keeps meaning "cookies were needed". + const outcome = await readJson<DownloadOutcome>( + `${ROOT}/data/vidcookiegated1/download-outcome.json`, + ); + expect(outcome.status).toBe("ok"); + expect(outcome.attempts.every((a) => a.usedCookies)).toBe(true); +}); + +test("when-required: failed prefetch is retried with cookies -> ok-with-cookies", async ({ + page, +}) => { + test.setTimeout(120_000); + await resetData(FIXTURE); + await writeSettings({ + ...(await defaultSettings()), + cookiesFromBrowser: COOKIES, + // cookieMode omitted -> back-compat default "when-required" + }); + await page.goto(`/channels/${CHANNEL}`); + await page.getByRole("button", { name: "Download videos" }).click(); + const log = page.getByLabel("Download videos output"); + await expect(log).toContainText( + `Metadata prefetch auth-required (needs_auth); retrying with --cookies-from-browser ${COOKIES}`, + { timeout: 60_000 }, + ); + await expect(log).toContainText( + "Managed download complete: 2 succeeded, 0 failed", + { timeout: 60_000 }, + ); + + // Two prefetch lines for the gated video: cookie-less first, cookies second. + const invocations = await readInvocations(); + const gatedPrefetches = invocations + .split("\n") + .filter((l) => l.startsWith("prefetch:") && l.includes("vidcookiegated1")); + expect(gatedPrefetches).toHaveLength(2); + expect(gatedPrefetches[0]).toMatch(/cookies=$/); + expect(gatedPrefetches[1]).toContain(`cookies=${COOKIES}`); + // The recovered prefetch hands cookies straight to the real download. + expect(invocations).toMatch( + new RegExp(`youtube-single:.*vidcookiegated1 cookies=${COOKIES}`), + ); + // The normal video never sees cookies in when-required mode. + const normalLines = invocations + .split("\n") + .filter((l) => l.includes("normalvideo1") && l.includes("cookies=")); + expect(normalLines.length).toBeGreaterThan(0); + expect(normalLines.every((l) => l.endsWith("cookies="))).toBe(true); + + const outcome = await readJson<DownloadOutcome>( + `${ROOT}/data/vidcookiegated1/download-outcome.json`, + ); + expect(outcome.status).toBe("ok-with-cookies"); + expect( + outcome.attempts.some( + (a) => a.kind === "metadata-prefetch-auth-retry" && a.usedCookies, + ), + ).toBe(true); +}); + +test("defer: excluded from batches, surfaced in Needs cookies, downloads via the bucket button", async ({ + page, +}) => { + test.setTimeout(180_000); + await resetData(FIXTURE); + await writeSettings({ + ...(await defaultSettings()), + cookiesFromBrowser: COOKIES, + cookieMode: "defer", + }); + + // Run 1: the gated video fails cookie-less (no prefetch retry, no auth + // retry — defer never uses cookies in normal runs). + await page.goto(`/channels/${CHANNEL}`); + await page.getByRole("button", { name: "Download videos" }).click(); + const log = page.getByLabel("Download videos output"); + await expect(log).toContainText( + "Managed download complete: 1 succeeded, 1 failed", + { timeout: 60_000 }, + ); + let invocations = await readInvocations(); + expect(invocations).not.toContain(`cookies=${COOKIES}`); + expect( + invocations + .split("\n") + .filter((l) => l.startsWith("prefetch:") && l.includes("vidcookiegated1")), + ).toHaveLength(1); + + // The failure lands the video in the snapshot's needsCookies bucket. + await waitForNeedsCookiesBucket(1); + + // Run 2: the recorded needs_auth video is excluded, not re-attempted. + await page.reload(); + await page.getByRole("button", { name: "Download videos" }).click(); + const log2 = page.getByLabel("Download videos output"); + await expect(log2).toContainText("needs_auth deferred=1", { + timeout: 60_000, + }); + await expect(log2).toContainText("Nothing to fetch.", { timeout: 60_000 }); + + // The Needs-cookies card offers the manual cookie run. + const bucket = page.getByLabel("retry needs cookies bucket"); + await expect(bucket).toBeVisible(); + await bucket + .getByRole("button", { name: /^Download with cookies \(1\)$/ }) + .click(); + const retryLog = page.getByLabel("Retry needs cookies output"); + await expect(retryLog).toContainText( + "Managed download complete: 1 succeeded, 0 failed", + { timeout: 60_000 }, + ); + + expect( + await pathExists(`${ROOT}/data/vidcookiegated1/transcript.en.vtt`), + ).toBe(true); + invocations = await readInvocations(); + expect(invocations).toMatch( + new RegExp(`youtube-single:.*vidcookiegated1 cookies=${COOKIES}`), + ); + + // The bucket empties once the snapshot regenerates. + await waitForNeedsCookiesBucket(0); + await page.reload(); + await expect(page.getByLabel("retry needs cookies bucket")).toHaveCount(0); +}); + +test("members_only videos surface in Needs cookies regardless of mode", async ({ + page, +}) => { + await resetData(FIXTURE); + // Default settings: when-required mode. A members-only playlist video with + // no downloaded artifact must still land in the bucket — cookies from a + // subscribed account could recover it even though batches exclude it. + await writeFile( + resolvePath(`${ROOT}/playlist`), + "https://www.youtube.com/watch?v=vidmembers1\n", + ); + const videoDir = resolvePath(`${ROOT}/data/vidmembers1`); + await mkdir(videoDir, { recursive: true }); + await writeFile( + path.join(videoDir, "availability.json"), + JSON.stringify({ + checkedAt: new Date().toISOString(), + availability: "members_only", + webpageUrl: "https://www.youtube.com/watch?v=vidmembers1", + }), + ); + + await page.goto(`/channels/${CHANNEL}`); + const bucket = page.getByLabel("retry needs cookies bucket"); + await expect(bucket).toBeVisible(); + await expect( + bucket.getByRole("button", { name: /^Download with cookies \(1\)$/ }), + ).toBeVisible(); + await expect(page.getByLabel("needs cookies vidmembers1")).toBeVisible(); +}); diff --git a/editor/e2e/fixtures/bin/fake-ytdlp.mjs b/editor/e2e/fixtures/bin/fake-ytdlp.mjs @@ -46,6 +46,27 @@ function lastNonFlag() { return undefined; } +// The `cookiegated` sentinel simulates an age-gated video: any metadata/ +// download invocation for it fails with yt-dlp's age-gate error UNLESS +// --cookies-from-browser was passed. (Deliberately distinct from the +// `needsauth` sentinel, which only affects --dump-json probes and still +// downloads fine — retry-bucket.spec depends on that.) +function cookieArg() { + return arg("--cookies-from-browser") ?? ""; +} + +function cookieGateBlocked(url) { + const lower = (url ?? "").toLowerCase(); + return lower.includes("cookiegated") && !cookieArg(); +} + +function failCookieGate(url) { + process.stderr.write( + `ERROR: [youtube] ${url}: Sign in to confirm your age. This video may be inappropriate for some users.\n`, + ); + process.exit(1); +} + function urlIdYouTube(u) { try { const parsed = new URL(u); @@ -214,7 +235,11 @@ async function modeDownloadFromFile(file, opts = {}) { async function modeDownloadOneUrl(url, opts) { process.stdout.write(`[fake-ytdlp] single-url download ${url}\n`); - await appendFile("fake-ytdlp.invocations", `download-one:${url}\n`); + await appendFile( + "fake-ytdlp.invocations", + `download-one:${url} cookies=${cookieArg()}\n`, + ); + if (cookieGateBlocked(url)) failCookieGate(url); // Record the resolved -f selector so format-selection tests can assert it // (e.g. Odysee "auto" -> original/bestaudio/worst). await appendFile("fake-ytdlp.invocations", `format:${arg("-f") ?? ""}\n`); @@ -409,8 +434,9 @@ async function modeYoutubeSingleUrlManaged(url) { await ensureDir(videoDir); await appendFile( "fake-ytdlp.invocations", - `youtube-single:${url}\n`, + `youtube-single:${url} cookies=${cookieArg()}\n`, ); + if (cookieGateBlocked(url)) failCookieGate(url); process.stdout.write(`[fake-ytdlp] managed single-URL ${id}\n`); // URL sentinels for no-subs-fallback tests. The sentinels live in the @@ -475,7 +501,11 @@ async function main() { if (has("--dump-json") && has("--skip-download")) { const url = lastNonFlag() ?? ""; const lower = url.toLowerCase(); - await appendFile("fake-ytdlp.invocations", `dump-json:${url}\n`); + await appendFile( + "fake-ytdlp.invocations", + `dump-json:${url} cookies=${cookieArg()}\n`, + ); + if (cookieGateBlocked(url)) failCookieGate(url); if (lower.includes("deleted")) { process.stderr.write(`ERROR: [youtube] ${url}: Video unavailable\n`); process.exit(1); @@ -587,7 +617,11 @@ async function main() { const id = urlIdYouTube(url); const videoDir = path.join("data", id); await ensureDir(videoDir); - await appendFile("fake-ytdlp.invocations", `prefetch:${url}\n`); + await appendFile( + "fake-ytdlp.invocations", + `prefetch:${url} cookies=${cookieArg()}\n`, + ); + if (cookieGateBlocked(url)) failCookieGate(url); await writeMetadata(videoDir, id, urlSentinels(url)); process.stdout.write(`[fake-ytdlp] metadata prefetch ${id}\n`); return; diff --git a/editor/e2e/fixtures/test-transcripts/cookies-mode-channel/channels/test-cookies/config.json b/editor/e2e/fixtures/test-transcripts/cookies-mode-channel/channels/test-cookies/config.json @@ -0,0 +1,5 @@ +{ + "handling": "youtube", + "name": "Test Cookies", + "url": "https://www.youtube.com/@example/videos" +} diff --git a/editor/e2e/fixtures/test-transcripts/cookies-mode-channel/channels/test-cookies/playlist b/editor/e2e/fixtures/test-transcripts/cookies-mode-channel/channels/test-cookies/playlist @@ -0,0 +1,2 @@ +https://www.youtube.com/watch?v=vidcookiegated1 +https://www.youtube.com/watch?v=normalvideo1