// Client-safe types and constants. No node-only imports — the editor's // Diagnostics stage component is a "use client" file that pulls this in. // Server-only I/O lives in availability-server.ts. export type Availability = | "public" | "unlisted" | "private" | "members_only" | "needs_auth" | "deleted" | "error"; export const AVAILABILITY_VALUES: Availability[] = [ "public", "unlisted", "private", "members_only", "needs_auth", "deleted", "error", ]; // Availability classes treated as permanently blocking: videos in these // states are filtered out of undownloadedIds and skipped by the managed // download flows. needs_auth is intentionally excluded (auth-retry may // recover it); error is excluded too (often transient: rate limits, network). export const EXCLUDED_FROM_DOWNLOAD: ReadonlyArray = [ "members_only", "deleted", "private", ]; // The single home for "this video is gone from the source for good". Used by // the keep-latest deletion sweep (checkKeptDeleted) and by the pre-clean // availability gate (verifyBeforeClean) to decide what to pin as do-not-clean. // A null availability means "we have never resolved this video" — NOT gone. // Narrows on the way through, so a caller that pins the video can pass the // availability straight into the marker note without re-asserting the type. export function isPermanentlyGone( a: Availability | null | undefined, ): a is Availability { return a != null && EXCLUDED_FROM_DOWNLOAD.includes(a); } // Availability classes a cookie (auth) retry can potentially recover — the // gate for the managed downloader's auth-retry attempts and the membership // rule for the per-channel "Needs cookies" snapshot bucket. Note the overlap // with EXCLUDED_FROM_DOWNLOAD: members_only/private are batch-excluded, but // cookies from a subscribed/owning account can still fetch them via the // bucket's manual cookie run. export const AUTH_RETRY_CLASSES: ReadonlySet = new Set([ "needs_auth", "members_only", "private", ]); // What the viewer shows for a video's presence on its source platform. Distinct // from `Availability`, which is what a per-video probe *reported*: this adds // `maybe_missing` (fell out of the channel listing, not yet individually // probed) and folds needs_auth/error back into `available`, since neither is // evidence the video is gone. export type VideoState = | "available" | "maybe_missing" | "unlisted" | "private" | "members_only" | "deleted"; // Ordered for UI: the umbrella's leaves in the order they're offered. export const MISSING_STATES: ReadonlyArray = [ "maybe_missing", "deleted", "private", "members_only", "unlisted", ]; export const VIDEO_STATES: ReadonlyArray = [ "available", ...MISSING_STATES, ]; // Every state except `available` means the video left its channel's listing. export function isMissingState(s: VideoState): boolean { return s !== "available"; } export function isVideoState(v: unknown): v is VideoState { return typeof v === "string" && (VIDEO_STATES as string[]).includes(v); } // What a per-video probe result means for the viewer. `needs_auth` and `error` // map to `available`: an age-gated video or a transient fetch failure is not // evidence the video left its channel. export function stateFromAvailability( a: Availability | null | undefined, ): VideoState { switch (a) { case "deleted": return "deleted"; case "private": return "private"; case "members_only": return "members_only"; case "unlisted": return "unlisted"; default: return "available"; } } // Read a summary's state, falling back to the legacy booleans. The fallback is // not dead code: in hub mode the MCP and the federated viewer read summaries // pages straight off member origins, which may have been built before `state` // existed. Structurally typed so DisplaySummary (which pulls in node-only // helpers) needn't be imported into client code. export function summaryState(t: { state?: VideoState; isDeleted?: boolean; isUnlisted?: boolean; }): VideoState { if (t.state) return t.state; if (t.isDeleted) return "deleted"; if (t.isUnlisted) return "unlisted"; return "available"; } export const VIDEO_STATE_LABELS: Record = { available: "Available", maybe_missing: "Unconfirmed", deleted: "Deleted", private: "Private", members_only: "Members-only", unlisted: "Unlisted", }; export type AvailabilityHistorySource = "check" | "backfill" | "download"; export type AvailabilityHistoryEntry = { availability: Availability; observedAt: string; // ISO source: AvailabilityHistorySource; }; export type AvailabilityRecord = { checkedAt: string; availability: Availability; error?: string; webpageUrl?: string; history?: AvailabilityHistoryEntry[]; // append-on-change; optional → backward compatible }; export const AVAILABILITY_FILENAME = "availability.json"; // YouTube's SOFT BLOCK. YouTube answers a session it is rate-limiting with the // playability reason "This content isn't available, try again later." — worded // like a removed video, and it is not one: the same video plays for anyone else // and for this session an hour later. yt-dlp (since #12958, 2025-04) re-words // it to say so: // // ERROR: [youtube] : This content isn't available, try again later. The // current session has been rate-limited by YouTube for up to an hour. It is // recommended to use `-t sleep` to add a delay between video requests to // avoid exceeding the rate limit. For more information, refer to // https://github.com/yt-dlp/yt-dlp/wiki/Extractors#this-content-isnt-available-try-again-later // // ("Your account has been rate-limited …" when cookies were passed). Until // release 10 its "content isn't available" matched the `deleted` group below, so // the soft block read as a removed video: per_video, no platform cooldown, the // batch kept going, and the video was excluded from download as gone. It is a // rate limit, and it is classified as one: `parseUnavailableFromStderr` says // "error" (transient) and `classifyDownloadFailure` says `rate_limit`. // // "try again later" is what marks it: a bare "Video unavailable" or "This // content isn't available." with no retry advice is still read as removed. The // apostrophe may be a right single quote, so match either. export function isSoftBlock(stderr: string): boolean { return /isn['’]?t available,? try again later/i.test(stderr); } // Map yt-dlp's stderr text to an Availability. yt-dlp does not expose a // stable machine-readable reason on failure, so we pattern-match the human // messages it emits across YouTube, Rumble, and a few other extractors. export function parseUnavailableFromStderr(stderr: string): Availability { // Checked FIRST: the soft block's wording would otherwise match `deleted` // below. It says nothing about the video, so it is an error, not a class. if (isSoftBlock(stderr)) return "error"; const s = stderr.toLowerCase(); if ( /private video/.test(s) || /this video is private/.test(s) ) { return "private"; } if ( /members[- ]only/.test(s) || /join this channel/.test(s) || /join channel/.test(s) ) { return "members_only"; } if ( /age[- ]restricted/.test(s) || /sign in to confirm your age/.test(s) || /confirm your age/.test(s) ) { return "needs_auth"; } if ( /video unavailable/.test(s) || /has been removed/.test(s) || /removed by the uploader/.test(s) || /account .* terminated/.test(s) || /account .* has been terminated/.test(s) || /this video has been removed/.test(s) || /content isn't available/.test(s) || /no longer available/.test(s) || // HTTP 410 Gone is the server saying the video was removed for good — // Rumble answers a taken-down video with it (`HTTP Error 410: Gone`). /http error 410/.test(s) ) { return "deleted"; } return "error"; } // Classify a download failure for the managed batch loop's "abort vs skip" // decision. Per-video permanent failures (private, members-only, etc.) are // `per_video` — the rest of the batch can continue. Rate-limit and network // errors are batch-level signals: continuing would just hammer the source or // waste cycles. Everything else is `unknown` and treated as fatal to be safe. // // `subs_rate_limit` (release 17, slice RL) is the one class that is not a // failure of the download: the SUBTITLE fetch answered 429 and nothing else // failed. YouTube's timedtext endpoint refuses per video while the webpage, // player and media requests in the same spawn succeed (measured 2026-09-25 and // 2026-10-01), so it says nothing about the platform: the downloader goes on to // the media, the record is a success carrying this class, and only the video's // subtitles are deferred (jobs/platformBackoff.ts). export type DownloadFailureClass = | "per_video" | "rate_limit" | "subs_rate_limit" | "network" | "unknown"; // yt-dlp's subtitle-fetch failure, as an ERROR (the default) or as a WARNING // (under --ignore-errors, which is how the managed downloader asks it to carry // on to the media): `Unable to download video subtitles for 'en': HTTP Error // 429: Too Many Requests`. const SUBTITLE_RATE_LIMIT_RE = /unable to download video subtitles for [^\n]*?(?:http error 429|too many requests|rate[- ]?limit)/i; export function hasSubtitleRateLimit(stderr: string): boolean { return SUBTITLE_RATE_LIMIT_RE.test(stderr); } const SUBTITLE_FAILURE_RE = /unable to download video subtitles/i; const RATE_LIMIT_WORDS_RE = /http error 429|too many requests|rate[- ]?limit/i; // A subtitle failure that is NOT a rate limit — a 403/404/5xx on the subtitle // URL, a failed live_chat replay, an OSError writing the file. Under // --ignore-errors yt-dlp reports EVERY subtitle failure as a WARNING and exits // 0, so the managed downloader asks this to keep such a failure a failure: // only a 429 takes the "download the media anyway" path (review H1). export function hasNonRateLimitSubtitleFailure(stderr: string): boolean { return stderr .split("\n") .some((l) => SUBTITLE_FAILURE_RE.test(l) && !RATE_LIMIT_WORDS_RE.test(l)); } // True when a subtitle 429 is the ONLY failure in the tail: every ERROR line // is a subtitle-download line (yt-dlp's own "The info failed to download … // trying with URL" retry is a WARNING and is allowed), every subtitle failure // is a rate limit, every 429 / too-many-requests line is a subtitle line (a // retried webpage 429 logged as a WARNING is a platform signal), and neither // the soft block nor the bot check — both platform signals — is anywhere in it. export function isSubtitleRateLimitOnly(stderr: string): boolean { if (!hasSubtitleRateLimit(stderr)) return false; if (isSoftBlock(stderr) || isBotCheck(stderr)) return false; if (hasNonRateLimitSubtitleFailure(stderr)) return false; for (const line of stderr.split("\n")) { const subtitleLine = SUBTITLE_FAILURE_RE.test(line); if (/^\s*ERROR:/i.test(line) && !subtitleLine) return false; if (RATE_LIMIT_WORDS_RE.test(line) && !subtitleLine) return false; } return true; } const PER_VIDEO_CLASSES: ReadonlyArray = [ "private", "members_only", "needs_auth", "deleted", ]; // YouTube's bot check. Not a 429 and not per-video: once it fires, every // subsequent request in the same batch fails the same way, so it is a // batch-level signal exactly like a rate limit — and it is what a metadata scan // over a whole channel listing trips first. The apostrophe is a right single // quote in yt-dlp's output, so match either. // // EXPORTED SEPARATELY from the rate-limit patterns, and consumed on its own by // ytdlp/metadataScan.ts, because the two call for different responses. A 429 // means "back off". A bot check means "you are not signed in" — cookies answer // it, and a cooldown recorded before they have even been tried costs the // operator an hour of waiting for a problem a retry would have solved in // seconds. export function isBotCheck(stderr: string): boolean { return /confirm you['\u2019]?re not a bot/i.test(stderr); } export function classifyDownloadFailure( stderrTail: string, availabilityClass: Availability | undefined, ): DownloadFailureClass { // The soft block wins over a per-video class, whichever parser produced it: // a caller holding a `deleted` read from before release 10 (or from a tail // that also names a removed video) must still back off. Erring this way costs // one cooldown; erring the other way keeps requesting into the block. if (isSoftBlock(stderrTail)) return "rate_limit"; // The subtitle fetch alone was refused: the media is unaffected, and the // platform is not to back off for it (release 17, slice RL). if (isSubtitleRateLimitOnly(stderrTail)) return "subs_rate_limit"; if ( availabilityClass !== undefined && PER_VIDEO_CLASSES.includes(availabilityClass) ) { return "per_video"; } // A subtitle 429 beside another subtitle failure is still not a platform // signal (release 17 re-review R1): its lines are dropped before the // generic rate-limit test, so the class is the OTHER failure's. const s = stderrTail .split("\n") .filter((l) => !(SUBTITLE_FAILURE_RE.test(l) && RATE_LIMIT_WORDS_RE.test(l))) .join("\n") .toLowerCase(); if ( /http error 429/.test(s) || /too many requests/.test(s) || /rate[- ]?limit/.test(s) || /throttl/.test(s) || isBotCheck(stderrTail) ) { return "rate_limit"; } if ( // A bare 403 (Cloudflare's fingerprint block on Rumble) backs the platform // off like any other transport failure. Before this it matched only when // the traceback happened to contain "ssl". /http error 403/.test(s) || /econnrefused/.test(s) || /etimedout/.test(s) || /enetunreach/.test(s) || /getaddrinfo/.test(s) || /network is unreachable/.test(s) || /temporary failure in name resolution/.test(s) || /connection reset/.test(s) || /tls handshake/.test(s) || /ssl/.test(s) ) { return "network"; } return "unknown"; } // Map the `availability` field that yt-dlp puts in --dump-json output to our // enum. yt-dlp uses: public, unlisted, needs_auth, premium_only, subscriber_only, // public_unlisted (rare), private, etc. export function availabilityFromJsonField(value: unknown): Availability { if (typeof value !== "string") return "public"; const v = value.toLowerCase(); if (v === "public") return "public"; if (v === "unlisted" || v === "public_unlisted") return "unlisted"; if (v === "private") return "private"; if (v === "subscriber_only" || v === "premium_only") return "members_only"; if (v === "needs_auth") return "needs_auth"; return "public"; }