import { PLATFORM_VALUES, type Platform } from "./platform"; import { isDownloadFormatPreset, isSourceVideoQuality, type DownloadFormatPreset, type SourceVideoQuality, } from "../ytdlp/downloadFormat"; import { isCookieMode, type CookieMode } from "./cookiePolicy"; import type { FieldDocs } from "./fieldDocs"; import { coerceRecordedDateRule, type RecordedDateRule } from "./recordedDate"; export type ChannelHandling = "youtube" | "transcribe"; // What KIND of source this channel is. Deliberately a separate axis from // `handling` rather than a third handling value: every existing // `handling === "transcribe" ? … : …` branch stays binary and so can never // misroute a social channel down a video path. // "video" (default) — yt-dlp + whisper, the original pipeline. // "social" — a social account fetched into the parallel posts corpus // (see common/lib/posts.ts). Skipped by the video scan. export type ChannelSourceKind = "video" | "social"; export const SOURCE_KIND_VALUES: ReadonlyArray = [ "video", "social", ]; export function isSocialChannel( config: Pick | null | undefined, ): boolean { return config?.sourceKind === "social"; } export type AudioFormat = "m4a" | "mp3" | "opus"; // Who owns audio extraction for a transcribe-handling download: // "ytdlp" (default) -> yt-dlp's own `-x --audio-format` postprocessor, exactly // as before. No source container is kept. // "app" -> yt-dlp downloads the source container (no `-x`) and the // app runs ffmpeg (transcodeAudio) to produce audio.. // Required whenever the source video must be kept/inspected // (the keep-latest persistence rule forces this). export type ExtractionMode = "ytdlp" | "app"; export const EXTRACTION_MODE_VALUES: ReadonlyArray = [ "ytdlp", "app", ]; // Each field is documented in AUDIO_CHECK_FIELD_DOCS below. export type AudioCheckConfig = { enabled: boolean; intervalSeconds?: number; maxRollbacks?: number; copyTimeoutSeconds?: number; resumeDuringProbe?: boolean; }; export const AUDIO_CHECK_FIELD_DOCS: FieldDocs = { enabled: "Turn the check on. Required: an `audioCheck` object without a boolean `enabled` is dropped whole.", intervalSeconds: "Seconds between integrity probes of the in-progress `.part` file; clamped to [10, 600]. Absent = 60. The live cadence adapts (AIMD): a malformed checkpoint halves it toward the 10 s floor, clean ones step it back up.", maxRollbacks: "Rollbacks to the last known-good snapshot before the download is given up; clamped to [1, 20]. Absent = 5.", copyTimeoutSeconds: "Seconds allowed for the snapshot copy; clamped to [5, 120]. Absent = 30.", resumeDuringProbe: "When false (default), yt-dlp stays SIGSTOPped across each probe, so it never downloads bytes a malformed verdict would discard and force a re-fetch — minimising HTTP 429 risk. True = the legacy behaviour: resume right after the snapshot copy and probe while the download keeps running.", }; // Per-channel download filter patterns. Regex SOURCES, always compiled with // the "i" flag; an unparseable pattern makes the filter inert rather than // failing a download (the editor form refuses to save one). Each field is // documented in DOWNLOAD_FILTER_FIELD_DOCS below. export type DownloadFilterConfig = { include?: string; exclude?: string; includeLivestreams?: boolean; rejectedLivestreams?: RejectedLivestreamMode; }; export const DOWNLOAD_FILTER_FIELD_DOCS: FieldDocs = { include: "Case-insensitive regex SOURCE (no delimiters, no flags) a video's `title + \"\\n\" + description` must match to be downloaded. Trimmed; blank = no include rule.", exclude: "Case-insensitive regex source that rejects a matching video. Wins over `include`. Trimmed; blank = no exclude rule.", includeLivestreams: "Opt every livestream VOD in, whatever it is called. A SECOND positive selector beside `include`, not a modifier of it — so `{ includeLivestreams: true }` alone rejects plain uploads and passes livestreams. Stored only when `true`.", rejectedLivestreams: "What to do with a livestream the filter REJECTED: `\"skip\"` (default, and what every channel predating the field did) or `\"chat-only\"` — the video is not downloaded, its live chat is, and it joins the corpus as a chat track with no captions. Stored only when not `\"skip\"`, and only beside a real filter: with nothing to reject it names a decision that can never be taken.", }; // Absent behaves as "skip". Spelled as a union rather than a boolean because a // third answer (keeping the audio at a lower quality, say) is a plausible next // one and a boolean would have to be migrated to make room for it. export type RejectedLivestreamMode = "skip" | "chat-only"; export function isRejectedLivestreamMode( v: unknown, ): v is RejectedLivestreamMode { return v === "skip" || v === "chat-only"; } // A channel's `config.json`. Each field is documented in // CHANNEL_CONFIG_FIELD_DOCS below (rendered into CHANNEL.md); the keys that are // SYNC STATE rather than configuration are CHANNEL_SYNC_STATE_KEYS. // // `excludeFromSync` IS GONE. It said "stop syncing, keep everything else", // which is exactly what `channelPriority`'s per-operation override map says — // `{tier:, overrides:{sync:"paused"}}` — and the model can say it for // every operation, not just this one. The parser drops the key rather than // carrying it, so an un-migrated config.json still parses: the migration // (common/bin/migrate-channel-priority.ts) reads it from the RAW file, not // through this parser, precisely because this parser no longer knows the word. export type ChannelConfig = { handling: ChannelHandling; sourceKind?: ChannelSourceKind; postFetcher?: string; socialHandle?: string; postPagePauseSeconds?: number; platform?: Platform; name?: string; url?: string; audioFormat?: AudioFormat; downloadFormat?: DownloadFormatPreset; sourceVideoQuality?: SourceVideoQuality; keepSourceVideo?: boolean; keepLatest?: number; extractionMode?: ExtractionMode; savedVideosDir?: string; dataDir?: string; mediaDir?: string; ytdlpExtraArgs?: string[]; subLangs?: string; lastSyncedAt?: string; lastFullDownloadAt?: string; lastFullSweepAt?: string; excludeFromBuild?: boolean; excludeFromCleanup?: boolean; syncIntervalMinutes?: number; fullSweepIntervalMinutes?: number; skipLiveDownloads?: boolean; downloadFilter?: DownloadFilterConfig; cookiesFromBrowser?: string; cookieMode?: CookieMode; sleepBetweenDownloadsSeconds?: number; audioCheck?: AudioCheckConfig; recordedDate?: RecordedDateRule; }; // In the order parseChannelConfig emits (and CHANNEL.md lists) them. export const CHANNEL_CONFIG_FIELD_DOCS: FieldDocs = { handling: 'REQUIRED. `"youtube"` (fetch the platform\'s captions) or `"transcribe"` (download audio and transcribe it locally). A file without a valid `handling` is not a channel: it reads as null.', sourceKind: 'What KIND of source this is: `"video"` (default — yt-dlp + transcription) or `"social"` (an account fetched into the posts corpus, skipped by the video scan). A separate axis from `handling`, so every binary handling branch stays binary.', postFetcher: 'Social channels only: which social fetcher drives ingest (e.g. `"bluesky-atproto"`, `"x-gallery-dl"`). Absent = resolve by URL detection. Trimmed.', socialHandle: 'Social channels only: the bare account handle (a leading "@" is stripped). Derived from `url` at creation but stored, so a later URL-format change upstream cannot silently re-point ingest at a different account.', postPagePauseSeconds: "Social channels that are read page by page (a forum thread) only: the pause between two page loads, in seconds; each pause is jittered to 0.85–1.65× of it. Absent = the fetcher's own (12 s, so 10–20 s); floored at 5, capped at 600.", platform: "The source platform: youtube, rumble, odysee, twitch, kick, archiveorg (archive.org items — imported, never listed; see README.md, \"archive.org items\"), bitchute (BitChute videos and channels; see README.md, \"BitChute\"), twitter, bluesky or xenforo (a forum thread). An unknown value is dropped.", name: "Display name.", url: "The channel / playlist / account URL syncs enumerate. Absent = the channel is never auto-synced.", audioFormat: '`"m4a"`, `"mp3"` or `"opus"`: the audio a transcribe-handling download keeps.', downloadFormat: "Per-channel override for the yt-dlp `-f` download format preset. Absent = inherit the global `downloadFormat`, which itself falls back to the per-source \"auto\" selector. Lets a channel whose source serves full-length audio only in its `original` format (e.g. Odysee) force it.", sourceVideoQuality: 'Per-channel override for the quality of the source container a full persist keeps ("Persist source video", the whole-recording fetch, "Persist kept now"). `"original"` (best video + audio) or `"video_720"` (≤720p H.264, for clips/editing). Absent = inherit the global `sourceVideoQuality`.', keepSourceVideo: "Keep the downloaded source video beside the audio.", keepLatest: "Keep-latest window: the newest N videos (by upload date) are protected from the Clean-audio sweep AND have their source video persisted to the saved-video store. 0 or absent = disabled; positives clamp to [1, 100000]. A kept video later found deleted at the source is pinned permanently via the do-not-clean marker.", extractionMode: '`"ytdlp"` (default — yt-dlp\'s own `-x --audio-format` postprocessor, no source container kept) or `"app"` (yt-dlp downloads the source container and the app runs ffmpeg). The keep-latest persistence rule forces `"app"` for the videos it persists.', savedVideosDir: "Per-channel override for the saved-video store root: this channel's persisted source videos live under `///`. Trimmed; blank = the global store.", dataDir: "RETIRED (release 17). The whole-directory layout's record: the absolute path `channels//data` was a symlink to. Still parsed for one release so a write never erases it: a channel that carries it — or whose `data/` is a link — is `legacy`, and every media job, lane and build holds it until `archilyzer storage migrate-tier ` moves its text back and its media into `mediaDir`. Written only by the re-point of a storage location (which keeps it `//data` on the new root); removed by that migration.", mediaDir: "Where this channel's big files live when relocated: `channels//media` is a symlink to it, `//media`. Absent = in place. Written only by relocate / re-point / the tier migration — a record of what is on disk, never free text, because a value that disagrees with the link is an \"inconsistent\" channel every media guard refuses. The text (`data/`) never moves.", ytdlpExtraArgs: "Extra yt-dlp arguments, appended verbatim. Must be an array of strings or it is dropped.", subLangs: "yt-dlp `--sub-langs` value for caption downloads.", lastSyncedAt: "SYNC STATE. When the channel last synced (ISO time). Stamped by every sync, and by a social fetch; read by the scheduler's cadence gate.", lastFullDownloadAt: "SYNC STATE. When a full download pass last completed (ISO time).", lastFullSweepAt: "SYNC STATE. When this channel last paid for a sync FULL SWEEP — the deep pass that re-enumerates the whole listing to refresh `playlist` and flag videos that have left it. Stamped by the sweep; read by the cadence gate to decide whether the next sync sweeps or stays on the cheap newest-first paged walk.", excludeFromBuild: "Leave this channel out of every site build.", excludeFromCleanup: "Leave this channel's reclaimable bytes out of the aggregate \"cleanable data\" total on /cleanup and its badge. The per-channel cleanup sweeps stay available; only the running total changes.", syncIntervalMinutes: "Auto-sync cadence: the scheduler syncs this channel when `now - lastSyncedAt >= syncIntervalMinutes`. Absent = inherit the global default; 0 = auto-sync off (still manually syncable); positives clamp to [1, 44640] (~31 days). A missing `url`, or a `sync` tier of paused in the channel-priority document, also disables auto-sync.", fullSweepIntervalMinutes: "Full-sweep cadence: a sync upgrades itself to a full sweep when `now - lastFullSweepAt >= fullSweepIntervalMinutes`. Absent = inherit `syncScheduler.fullSweepIntervalMinutes`; 0 = never sweep (every sync is a paged walk); positives clamp to [1, 44640].", skipLiveDownloads: "Per-channel override for the global `skipLiveDownloads`. Absent = inherit; false = allow downloading currently-live / upcoming videos.", downloadFilter: "Per-channel title/description download filter, matched against the per-video metadata prefetch. A declined video is SETTLED by a terminal download-outcome keyed on the filter's signature, not by an archive line, so changing either pattern re-evaluates every settled video on the next run. An object with no real rule is dropped (the filter is inert).", cookiesFromBrowser: "Per-channel override of the global cookies-from-browser spec. Trimmed; blank = inherit.", cookieMode: "Per-channel override of the global cookie mode. Absent = inherit.", sleepBetweenDownloadsSeconds: "Per-channel override for the global pause between downloads. Absent = inherit; 0 = no sleep; floored and capped at 600.", audioCheck: "Opt-in audio-integrity checking for sources that intermittently serve corrupt audio mid-download (e.g. Odysee \"original\"): the managed downloader periodically validates the in-progress `.part` file and rolls back to the last known-good snapshot on corruption. transcribe-handling only.", recordedDate: "For a channel that MIRRORS another's streams (a VOD archive): how to read the date a video was RECORDED from its title, since its `upload_date` is the date of the copy. The index stores the result as the record's `recordedDate` (`YYYYMMDD`), which coverage reads before the upload date. A title it does not match, a date that is not a real day, or one after the upload date gives none. Changing the rule re-derives the channel's records on the next index build. An object without a usable `titlePattern` is dropped." }; // Every key a config.json may carry, in emission order. The unknown-key oracle: // a key not listed here is dropped by every read and every write. export const CHANNEL_CONFIG_KEYS = Object.keys( CHANNEL_CONFIG_FIELD_DOCS, ) as ReadonlyArray; // The keys that are MUTABLE SYNC STATE rather than configuration: stamped by // the sync / sweep / download passes, never by the channel form. They live in // the same file on purpose (the file is not split); a form save preserves them // because it unsets only the form's own fields. export const CHANNEL_SYNC_STATE_KEYS = [ "lastSyncedAt", "lastFullDownloadAt", "lastFullSweepAt", ] as const satisfies ReadonlyArray; export type ChannelSyncStateKey = (typeof CHANNEL_SYNC_STATE_KEYS)[number]; export const CHANNEL_SLEEP_BETWEEN_DOWNLOADS_MAX_SECONDS = 600; // Bounds for a channel's auto-sync interval. A nonzero value is clamped into // this window; 0 is preserved as "disabled". Max is ~31 days. export const SYNC_INTERVAL_MIN_MINUTES = 1; export const SYNC_INTERVAL_MAX_MINUTES = 44640; // Upper bound for a channel's keep-latest window. 0/undefined disables it. export const KEEP_LATEST_MAX = 100000; export const AUDIO_CHECK_INTERVAL_DEFAULT_SECONDS = 60; export const AUDIO_CHECK_INTERVAL_MIN_SECONDS = 10; export const AUDIO_CHECK_INTERVAL_MAX_SECONDS = 600; // Adaptive (AIMD) probe cadence. Each malformed checkpoint multiplies the live // interval by this factor (fast tightening) down to AUDIO_CHECK_INTERVAL_MIN_SECONDS // as the floor; each run of clean checkpoints steps it back up additively toward // the configured interval (slow relaxation). See runAudioCheckedYtdlp. export const AUDIO_CHECK_INTERVAL_BACKOFF_FACTOR_DEFAULT = 0.5; export const AUDIO_CHECK_INTERVAL_RECOVER_STEP_SECONDS = 15; export const AUDIO_CHECK_INTERVAL_RECOVER_AFTER_CLEAN = 2; export const AUDIO_CHECK_MAX_ROLLBACKS_DEFAULT = 5; export const AUDIO_CHECK_MAX_ROLLBACKS_MIN = 1; export const AUDIO_CHECK_MAX_ROLLBACKS_MAX = 20; export const AUDIO_CHECK_COPY_TIMEOUT_DEFAULT_SECONDS = 30; export const AUDIO_CHECK_COPY_TIMEOUT_MIN_SECONDS = 5; export const AUDIO_CHECK_COPY_TIMEOUT_MAX_SECONDS = 120; function clampInt( value: number, min: number, max: number, ): number { return Math.min(Math.max(Math.floor(value), min), max); } export const HANDLING_VALUES: ReadonlyArray = [ "youtube", "transcribe", ]; export const AUDIO_FORMAT_VALUES: ReadonlyArray = [ "m4a", "mp3", "opus", ]; // Only an object whose `handling` is valid is a channel. Everything else — // `null`, an array, a number, `{}` — reads as "not a channel". export function isChannelShape(raw: unknown): raw is Record { if (!raw || typeof raw !== "object") return false; const h = (raw as Record).handling; return h === "youtube" || h === "transcribe"; } const isFiniteNumber = (v: unknown): v is number => typeof v === "number" && Number.isFinite(v); const trimmedNonBlank = (v: unknown): string | undefined => typeof v === "string" && v.trim() ? v.trim() : undefined; const bool = (v: unknown): boolean | undefined => typeof v === "boolean" ? v : undefined; const str = (v: unknown): string | undefined => typeof v === "string" ? v : undefined; // 0 is a sentinel ("disabled") preserved as-is; any other non-negative value // is clamped into [min, max]. const zeroOrClamped = (min: number, max: number) => (v: unknown): number | undefined => isFiniteNumber(v) && v >= 0 ? (v === 0 ? 0 : clampInt(v, min, max)) : undefined; export function coerceDownloadFilter(v: unknown): DownloadFilterConfig | undefined { // Blank/whitespace patterns are dropped rather than stored, so an all-blank // object leaves the key absent and the filter inert — the same "empty means // inherit-off" rule the cookie overrides use. if (!v || typeof v !== "object") return undefined; const df = v as Record; const include = typeof df.include === "string" ? df.include.trim() : ""; const exclude = typeof df.exclude === "string" ? df.exclude.trim() : ""; const includeLivestreams = df.includeLivestreams === true; // ONLY ALONGSIDE A REAL FILTER, and only when it is not the default. The // mode says what to do with a REJECTED livestream, so with nothing to reject // it names a decision that can never be taken — storing it would put a // setting on the Configure form that does nothing and explains nothing. const rejectedLivestreams = isRejectedLivestreamMode(df.rejectedLivestreams) ? df.rejectedLivestreams : "skip"; if (!include && !exclude && !includeLivestreams) return undefined; return { ...(include ? { include } : {}), ...(exclude ? { exclude } : {}), ...(includeLivestreams ? { includeLivestreams: true } : {}), ...(rejectedLivestreams !== "skip" ? { rejectedLivestreams } : {}), }; } export function coerceAudioCheck(v: unknown): AudioCheckConfig | undefined { if (!v || typeof v !== "object") return undefined; const a = v as Record; if (typeof a.enabled !== "boolean") return undefined; const ac: AudioCheckConfig = { enabled: a.enabled }; if (isFiniteNumber(a.intervalSeconds)) { ac.intervalSeconds = clampInt( a.intervalSeconds, AUDIO_CHECK_INTERVAL_MIN_SECONDS, AUDIO_CHECK_INTERVAL_MAX_SECONDS, ); } if (isFiniteNumber(a.maxRollbacks)) { ac.maxRollbacks = clampInt( a.maxRollbacks, AUDIO_CHECK_MAX_ROLLBACKS_MIN, AUDIO_CHECK_MAX_ROLLBACKS_MAX, ); } if (isFiniteNumber(a.copyTimeoutSeconds)) { ac.copyTimeoutSeconds = clampInt( a.copyTimeoutSeconds, AUDIO_CHECK_COPY_TIMEOUT_MIN_SECONDS, AUDIO_CHECK_COPY_TIMEOUT_MAX_SECONDS, ); } if (typeof a.resumeDuringProbe === "boolean") { ac.resumeDuringProbe = a.resumeDuringProbe; } return ac; } // THE ONE SET OF COERCIONS for a config.json field: raw value in, the legal // value — or `undefined`, meaning "omit the key" — out. Each is total over // `unknown`. parseChannelConfig (below, client-safe) and channelConfigSchema // (lib/channelConfigSchema.ts, server-only zod) both compose exactly these, so // the two cannot disagree. Keyed and ordered like CHANNEL_CONFIG_FIELD_DOCS. export const CHANNEL_CONFIG_COERCIONS: { readonly [K in keyof ChannelConfig]-?: (v: unknown) => ChannelConfig[K] | undefined; } = { handling: (v) => (v === "youtube" || v === "transcribe" ? v : undefined), sourceKind: (v) => (v === "video" || v === "social" ? v : undefined), postFetcher: trimmedNonBlank, socialHandle: (v) => typeof v === "string" && v.trim() ? v.trim().replace(/^@/, "") : undefined, postPagePauseSeconds: (v) => isFiniteNumber(v) && v > 0 ? Math.min(Math.max(Math.floor(v), 5), 600) : undefined, platform: (v) => typeof v === "string" && PLATFORM_VALUES.includes(v as Platform) ? (v as Platform) : undefined, name: str, url: str, audioFormat: (v) => (v === "m4a" || v === "mp3" || v === "opus" ? v : undefined), downloadFormat: (v) => (isDownloadFormatPreset(v) ? v : undefined), sourceVideoQuality: (v) => (isSourceVideoQuality(v) ? v : undefined), keepSourceVideo: bool, keepLatest: zeroOrClamped(1, KEEP_LATEST_MAX), extractionMode: (v) => (v === "ytdlp" || v === "app" ? v : undefined), savedVideosDir: trimmedNonBlank, dataDir: trimmedNonBlank, mediaDir: trimmedNonBlank, ytdlpExtraArgs: (v) => Array.isArray(v) && v.every((x) => typeof x === "string") ? (v as string[]) : undefined, subLangs: str, lastSyncedAt: str, lastFullDownloadAt: str, lastFullSweepAt: str, excludeFromBuild: bool, excludeFromCleanup: bool, syncIntervalMinutes: zeroOrClamped(SYNC_INTERVAL_MIN_MINUTES, SYNC_INTERVAL_MAX_MINUTES), fullSweepIntervalMinutes: zeroOrClamped( SYNC_INTERVAL_MIN_MINUTES, SYNC_INTERVAL_MAX_MINUTES, ), skipLiveDownloads: bool, downloadFilter: coerceDownloadFilter, cookiesFromBrowser: trimmedNonBlank, cookieMode: (v) => (isCookieMode(v) ? v : undefined), sleepBetweenDownloadsSeconds: (v) => isFiniteNumber(v) && v >= 0 ? Math.min(Math.floor(v), CHANNEL_SLEEP_BETWEEN_DOWNLOADS_MAX_SECONDS) : undefined, audioCheck: coerceAudioCheck, recordedDate: coerceRecordedDateRule, }; // A channel's config.json as a ChannelConfig, or null when it is not a channel // (see isChannelShape). An ill-typed or out-of-range OPTIONAL key is OMITTED — // never defaulted — so `"k" in config` means "the file set it", which the // inherit-the-global rule for every override depends on. Unknown keys (and the // retired `excludeFromSync`) are dropped. // // Client-safe (no zod): the channel form and five other `"use client"` modules // import this file. The server reads and writes through // lib/channelConfigSchema.ts, which composes the same coercions. export function parseChannelConfig(raw: unknown): ChannelConfig | null { if (!isChannelShape(raw)) return null; const out: Partial> = {}; for (const key of CHANNEL_CONFIG_KEYS) { const value = CHANNEL_CONFIG_COERCIONS[key](raw[key]); if (value !== undefined) out[key] = value; } return out as ChannelConfig; }