Archilyzer · Source

archilyzer

Archilyzer
git clone https://archilyzer.pages.dev/source/archilyzer.git
Log | Files | Refs | README | LICENSE

commit 96798480c38e2a1cabc124ab080ca220c3071db1
parent 7e3c150d847135d55e7c16172b3aa8823f46c6e3
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date:   Thu, 24 Sep 2026 13:23:50 -0400

lib: channelConfigSchema — one set of coercions for config.json

channelConfig.ts (pure, client-safe — six "use client" value importers)
now holds the ONE set of per-key coercions, CHANNEL_CONFIG_COERCIONS,
extracted verbatim from the hand parser (clamps with 0 preserved, trims,
downloadFilter normalisation, enums; excludeFromSync still dropped), and
parseChannelConfig composes them without zod, keeping its name, its null
for a non-channel (isChannelShape) and its OMIT rule for an invalid key.
Also: CHANNEL_CONFIG_FIELD_DOCS (the type's field comments moved into it),
AUDIO_CHECK_FIELD_DOCS, DOWNLOAD_FILTER_FIELD_DOCS, CHANNEL_CONFIG_KEYS
(derived from the docs record — the unknown-key oracle) and
CHANNEL_SYNC_STATE_KEYS = lastSyncedAt, lastFullDownloadAt, lastFullSweepAt.

channelConfigSchema.ts (server-only, zod): the same 29 coercions as
settingsField()s, .describe()d, then stripUndefined — the server reader and
writer use it from the next commit. The test pins that it and
parseChannelConfig agree over generated inputs.

Parity with main: 20,000 random inputs parse deepStrictEqual and
key-order-equal to main's parseChannelConfig.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Diffstat:
Mcommon/lib/channelConfig.ts | 560++++++++++++++++++++++++++++++++++++-------------------------------------------
Acommon/lib/channelConfigSchema.test.ts | 158+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Acommon/lib/channelConfigSchema.ts | 90+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 503 insertions(+), 305 deletions(-)

diff --git a/common/lib/channelConfig.ts b/common/lib/channelConfig.ts @@ -4,6 +4,7 @@ import { type DownloadFormatPreset, } from "../ytdlp/downloadFormat"; import { isCookieMode, type CookieMode } from "./cookiePolicy"; +import type { FieldDocs } from "./fieldDocs"; export type ChannelHandling = "youtube" | "transcribe"; @@ -43,44 +44,51 @@ export const EXTRACTION_MODE_VALUES: ReadonlyArray<ExtractionMode> = [ "app", ]; + +// Each field is documented in AUDIO_CHECK_FIELD_DOCS below. export type AudioCheckConfig = { enabled: boolean; intervalSeconds?: number; maxRollbacks?: number; copyTimeoutSeconds?: number; - // When false (default), yt-dlp stays SIGSTOPped across each integrity - // probe, so it never downloads bytes that a malformed verdict would - // discard and force a re-fetch — minimising HTTP 429 risk. Set true for - // the legacy behavior: resume immediately after the snapshot copy and - // probe concurrently while the download keeps running. resumeDuringProbe?: boolean; }; +export const AUDIO_CHECK_FIELD_DOCS: FieldDocs<AudioCheckConfig> = { + enabled: + "Turn the check on. Required: an `audioCheck` object without a boolean `enabled` is dropped whole.", + intervalSeconds: + "Seconds between integrity probes of the in-progress `.part` file; clamped to [10, 600]. Absent = 60. The live cadence adapts (AIMD): a malformed checkpoint halves it toward the 10 s floor, clean ones step it back up.", + maxRollbacks: + "Rollbacks to the last known-good snapshot before the download is given up; clamped to [1, 20]. Absent = 5.", + copyTimeoutSeconds: + "Seconds allowed for the snapshot copy; clamped to [5, 120]. Absent = 30.", + resumeDuringProbe: + "When false (default), yt-dlp stays SIGSTOPped across each probe, so it never downloads bytes a malformed verdict would discard and force a re-fetch — minimising HTTP 429 risk. True = the legacy behaviour: resume right after the snapshot copy and probe while the download keeps running.", +}; + // Per-channel download filter patterns. Regex SOURCES, always compiled with // the "i" flag; an unparseable pattern makes the filter inert rather than -// failing a download (the editor form refuses to save one). +// failing a download (the editor form refuses to save one). Each field is +// documented in DOWNLOAD_FILTER_FIELD_DOCS below. export type DownloadFilterConfig = { include?: string; exclude?: string; - // Opt every livestream VOD in, regardless of what it is called. A SECOND - // positive selector beside `include`, not a modifier of it — which is why - // `{ includeLivestreams: true }` alone rejects plain uploads and passes - // livestreams. See titleFilterRejects. includeLivestreams?: boolean; - // WHAT TO DO WITH A LIVESTREAM THE FILTER REJECTED. Absent = "skip", which - // is every channel that predates this field and is byte-for-byte what they - // did before it existed. - // - // "chat-only" is the middle answer that did not exist: a multi-hour stream - // whose TITLE says nothing about the subject is usually not worth its audio, - // but its live chat is text, it is small, and it is the only record of what - // the room said. So the video is not downloaded, its chat is, and it joins - // the corpus as a chat track with no captions — NOT as a downloaded video. - // See classifyAgainstFilter for why this is a third VERDICT rather than a - // flag read at download time. rejectedLivestreams?: RejectedLivestreamMode; }; +export const DOWNLOAD_FILTER_FIELD_DOCS: FieldDocs<DownloadFilterConfig> = { + include: + "Case-insensitive regex SOURCE (no delimiters, no flags) a video's `title + \"\\n\" + description` must match to be downloaded. Trimmed; blank = no include rule.", + exclude: + "Case-insensitive regex source that rejects a matching video. Wins over `include`. Trimmed; blank = no exclude rule.", + includeLivestreams: + "Opt every livestream VOD in, whatever it is called. A SECOND positive selector beside `include`, not a modifier of it — so `{ includeLivestreams: true }` alone rejects plain uploads and passes livestreams. Stored only when `true`.", + rejectedLivestreams: + "What to do with a livestream the filter REJECTED: `\"skip\"` (default, and what every channel predating the field did) or `\"chat-only\"` — the video is not downloaded, its live chat is, and it joins the corpus as a chat track with no captions. Stored only when not `\"skip\"`, and only beside a real filter: with nothing to reject it names a decision that can never be taken.", +}; + // Absent behaves as "skip". Spelled as a union rather than a boolean because a // third answer (keeping the audio at a lower quality, say) is a plausible next // one and a boolean would have to be migrated to make room for it. @@ -92,138 +100,119 @@ export function isRejectedLivestreamMode( return v === "skip" || v === "chat-only"; } +// A channel's `config.json`. Each field is documented in +// CHANNEL_CONFIG_FIELD_DOCS below (rendered into CHANNEL.md); the keys that are +// SYNC STATE rather than configuration are CHANNEL_SYNC_STATE_KEYS. +// +// `excludeFromSync` IS GONE. It said "stop syncing, keep everything else", +// which is exactly what `channelPriority`'s per-operation override map says — +// `{tier:<base>, overrides:{sync:"paused"}}` — and the model can say it for +// every operation, not just this one. The parser drops the key rather than +// carrying it, so an un-migrated config.json still parses: the migration +// (common/bin/migrate-channel-priority.ts) reads it from the RAW file, not +// through this parser, precisely because this parser no longer knows the word. export type ChannelConfig = { handling: ChannelHandling; - // Omitted = "video" (every channel that predates the posts corpus). sourceKind?: ChannelSourceKind; - // Social channels only: which SocialFetcher drives ingest (see - // common/social/fetchers.ts), e.g. "bluesky-atproto" / "x-gallery-dl". - // Omitted = resolve by URL detection. postFetcher?: string; - // Social channels only: the bare account handle, without "@" or URL wrapper. - // Derived from `url` at creation time but stored so a later URL-format change - // upstream can't silently re-point ingest at a different account. socialHandle?: string; platform?: Platform; name?: string; url?: string; audioFormat?: AudioFormat; - // Per-channel override for the yt-dlp `-f` download format (see - // common/ytdlp/downloadFormat.ts). Omitted -> inherit the global default - // (SiteSettings.downloadFormat), which itself falls back to the per-source - // "auto" selector. Lets a channel whose source only serves full-length audio - // in its `original` format (e.g. Odysee) force it regardless of the global. downloadFormat?: DownloadFormatPreset; keepSourceVideo?: boolean; - // Keep-latest retention/persistence window. The newest N videos (by upload - // date) are protected from the Clean-audio sweep AND have their source video - // persisted to the saved-video store (see common/controller/keptVideos.ts and - // the per-download persistence rule in downloadOneManaged.ts). Semantics: - // undefined / 0 -> disabled - // > 0 -> keep newest N (clamped to KEEP_LATEST_MAX) - // A kept video later found deleted-from-source is pinned permanently via the - // do-not-clean marker so it survives even after it rolls out of the window. keepLatest?: number; - // Audio-extraction strategy for transcribe-handling downloads (see - // ExtractionMode above). Omitted/unknown -> "ytdlp" (legacy behavior). The - // keep-latest persistence rule forces "app" for the videos it persists, - // regardless of this setting. extractionMode?: ExtractionMode; - // Per-channel override for the saved-video store root (global default is - // paths.savedVideosDir / SAVED_VIDEOS_DIR). When set, this channel's persisted - // source videos live under <savedVideosDir>/<slug>/<videoId>/. Lets a single - // channel's large videos land on a different disk than the rest. Resolved by - // savedVideoDir() in common/lib/savedVideo.ts. Empty/whitespace = use global. savedVideosDir?: string; - // Where this channel's downloaded media ACTUALLY lives, when it has been - // relocated to another drive: the absolute path `channels/<slug>/data` is a - // symlink to. Blank/absent = in place. Written ONLY by the relocate job on - // success (common/controller/relocateChannelMedia.ts) — it is a record of - // what is on disk, never a free-text field, because a value that disagrees - // with the link is an "inconsistent" channel that every guard refuses. See - // common/lib/channelMedia.ts. dataDir?: string; ytdlpExtraArgs?: string[]; subLangs?: string; lastSyncedAt?: string; lastFullDownloadAt?: string; - // When this channel last paid for a sync FULL SWEEP — the deep pass that - // re-enumerates the whole listing to refresh `playlist` and flag videos that - // have left it. Stamped by the sweep itself; read by the cadence gate in - // common/jobs/deepSync.ts to decide whether the next sync sweeps or stays on - // the cheap newest-first paged walk. lastFullSweepAt?: string; excludeFromBuild?: boolean; - // `excludeFromSync` IS GONE. It said "stop syncing, keep everything else", - // which is exactly what `channelPriority`'s per-operation override map says - // — `{tier:<base>, overrides:{sync:"paused"}}` — and the model can say it - // for every operation, not just this one. The sanitizer below drops the key - // rather than carrying it, so an un-migrated config.json still parses: the - // migration (common/bin/migrate-channel-priority.ts) reads it from the RAW - // file, not through this parser, precisely because this parser no longer - // knows the word. - // Opt this channel OUT of the aggregate "cleanable data" total shown on the - // /cleanup page and its sidebar badge. The per-channel cleanup sweeps remain - // fully available; this flag only removes the channel's reclaimable bytes from - // the running total (e.g. keep a finicky-to-redownload channel's audio around - // for now without it nagging in the badge). Toggled from the cleanup ledger. excludeFromCleanup?: boolean; - // Auto-sync cadence for the scheduled sync system (common/jobs/syncScheduler.ts). - // The cron-driven scheduler syncs this channel when - // `now - lastSyncedAt >= syncIntervalMinutes`. Semantics: - // undefined -> inherit the global SiteSettings default interval - // 0 -> auto-sync disabled for this channel (still manually syncable) - // > 0 -> sync this often (clamped to [SYNC_INTERVAL_MIN/MAX_MINUTES]) - // A missing `url` also disables auto-sync, as does a `sync` tier of `paused` - // in the channel-priority document (common/lib/channelPriority.ts). syncIntervalMinutes?: number; - // Full-sweep cadence for this channel (see common/jobs/deepSync.ts). A sync - // upgrades itself to a full sweep when - // `now - lastFullSweepAt >= fullSweepIntervalMinutes`. Same semantics as - // syncIntervalMinutes: - // undefined -> inherit the global syncScheduler.fullSweepIntervalMinutes - // 0 -> never sweep this channel (every sync is a paged walk) - // > 0 -> sweep this often (clamped to [SYNC_INTERVAL_MIN/MAX_MINUTES]) - // A sweep is one full enumeration, so long cadences (daily to weekly) are the - // norm — it is much more expensive than one 50-entry sync page. fullSweepIntervalMinutes?: number; - // Per-channel override for the global setting of the same name. When - // omitted, the global SiteSettings value is used. 0 disables the sleep - // for this channel. - sleepBetweenDownloadsSeconds?: number; - // Per-channel override for the global skipLiveDownloads setting. When - // omitted, the global SiteSettings value is used. Set false to allow this - // channel to download currently-live/upcoming videos. skipLiveDownloads?: boolean; - // Per-channel title/description download filter. Both patterns are - // case-insensitive regex SOURCES (no delimiters, no flags) matched against - // `title + "\n" + description` from the per-video metadata prefetch — the - // only text a download filter ever gets to see. `exclude` wins over - // `include`. A video the filter declines is SETTLED by a terminal - // download-outcome keyed on the filter's signature (see - // downloadFilterSignature), NOT by an archive line: an archive line would - // mean "downloaded" to verifyTranscripts and the missingFromArchive bucket, - // and would still leave the id in the artifact-based undownloadedIds. The - // signature is what makes the settlement self-expiring — change either - // pattern and every settled video is re-evaluated on the next run. - // Omitted/empty = no filter (the registry entry is inert). downloadFilter?: DownloadFilterConfig; - // Per-channel override of the global cookies-from-browser browser spec - // (SiteSettings.cookiesFromBrowser). Omitted/empty = inherit the global - // value. See common/lib/cookiePolicy.ts. cookiesFromBrowser?: string; - // Per-channel override of the global cookie mode - // (SiteSettings.cookieMode). Omitted = inherit. See - // common/lib/cookiePolicy.ts for the mode semantics. cookieMode?: CookieMode; - // Opt-in audio-integrity checking for sources that intermittently serve - // corrupt audio mid-download (e.g. Odysee "original" format). When - // enabled, the managed downloader periodically validates the in-progress - // .part file via ffmpeg and rolls back to the last known-good snapshot - // on corruption. transcribe-handling only. + sleepBetweenDownloadsSeconds?: number; audioCheck?: AudioCheckConfig; }; +// In the order parseChannelConfig emits (and CHANNEL.md lists) them. +export const CHANNEL_CONFIG_FIELD_DOCS: FieldDocs<ChannelConfig> = { + handling: + 'REQUIRED. `"youtube"` (fetch the platform\'s captions) or `"transcribe"` (download audio and transcribe it locally). A file without a valid `handling` is not a channel: it reads as null.', + sourceKind: + 'What KIND of source this is: `"video"` (default — yt-dlp + transcription) or `"social"` (an account fetched into the posts corpus, skipped by the video scan). A separate axis from `handling`, so every binary handling branch stays binary.', + postFetcher: + 'Social channels only: which social fetcher drives ingest (e.g. `"bluesky-atproto"`, `"x-gallery-dl"`). Absent = resolve by URL detection. Trimmed.', + socialHandle: + 'Social channels only: the bare account handle (a leading "@" is stripped). Derived from `url` at creation but stored, so a later URL-format change upstream cannot silently re-point ingest at a different account.', + platform: "The source platform (youtube, rumble, …). An unknown value is dropped.", + name: "Display name.", + url: "The channel / playlist / account URL syncs enumerate. Absent = the channel is never auto-synced.", + audioFormat: '`"m4a"`, `"mp3"` or `"opus"`: the audio a transcribe-handling download keeps.', + downloadFormat: + "Per-channel override for the yt-dlp `-f` download format preset. Absent = inherit the global `downloadFormat`, which itself falls back to the per-source \"auto\" selector. Lets a channel whose source serves full-length audio only in its `original` format (e.g. Odysee) force it.", + keepSourceVideo: "Keep the downloaded source video beside the audio.", + keepLatest: + "Keep-latest window: the newest N videos (by upload date) are protected from the Clean-audio sweep AND have their source video persisted to the saved-video store. 0 or absent = disabled; positives clamp to [1, 100000]. A kept video later found deleted at the source is pinned permanently via the do-not-clean marker.", + extractionMode: + '`"ytdlp"` (default — yt-dlp\'s own `-x --audio-format` postprocessor, no source container kept) or `"app"` (yt-dlp downloads the source container and the app runs ffmpeg). The keep-latest persistence rule forces `"app"` for the videos it persists.', + savedVideosDir: + "Per-channel override for the saved-video store root: this channel's persisted source videos live under `<savedVideosDir>/<slug>/<videoId>/`. Trimmed; blank = the global store.", + dataDir: + "Where this channel's media ACTUALLY lives when relocated to another drive: the absolute path `channels/<slug>/data` is a symlink to. Absent = in place. Written ONLY by the relocate / re-point jobs on success — a record of what is on disk, never free text, because a value that disagrees with the link is an \"inconsistent\" channel every guard refuses.", + ytdlpExtraArgs: "Extra yt-dlp arguments, appended verbatim. Must be an array of strings or it is dropped.", + subLangs: "yt-dlp `--sub-langs` value for caption downloads.", + lastSyncedAt: + "SYNC STATE. When the channel last synced (ISO time). Stamped by every sync, and by a social fetch; read by the scheduler's cadence gate.", + lastFullDownloadAt: "SYNC STATE. When a full download pass last completed (ISO time).", + lastFullSweepAt: + "SYNC STATE. When this channel last paid for a sync FULL SWEEP — the deep pass that re-enumerates the whole listing to refresh `playlist` and flag videos that have left it. Stamped by the sweep; read by the cadence gate to decide whether the next sync sweeps or stays on the cheap newest-first paged walk.", + excludeFromBuild: "Leave this channel out of every site build.", + excludeFromCleanup: + "Leave this channel's reclaimable bytes out of the aggregate \"cleanable data\" total on /cleanup and its badge. The per-channel cleanup sweeps stay available; only the running total changes.", + syncIntervalMinutes: + "Auto-sync cadence: the scheduler syncs this channel when `now - lastSyncedAt >= syncIntervalMinutes`. Absent = inherit the global default; 0 = auto-sync off (still manually syncable); positives clamp to [1, 44640] (~31 days). A missing `url`, or a `sync` tier of paused in the channel-priority document, also disables auto-sync.", + fullSweepIntervalMinutes: + "Full-sweep cadence: a sync upgrades itself to a full sweep when `now - lastFullSweepAt >= fullSweepIntervalMinutes`. Absent = inherit `syncScheduler.fullSweepIntervalMinutes`; 0 = never sweep (every sync is a paged walk); positives clamp to [1, 44640].", + skipLiveDownloads: + "Per-channel override for the global `skipLiveDownloads`. Absent = inherit; false = allow downloading currently-live / upcoming videos.", + downloadFilter: + "Per-channel title/description download filter, matched against the per-video metadata prefetch. A declined video is SETTLED by a terminal download-outcome keyed on the filter's signature, not by an archive line, so changing either pattern re-evaluates every settled video on the next run. An object with no real rule is dropped (the filter is inert).", + cookiesFromBrowser: + "Per-channel override of the global cookies-from-browser spec. Trimmed; blank = inherit.", + cookieMode: "Per-channel override of the global cookie mode. Absent = inherit.", + sleepBetweenDownloadsSeconds: + "Per-channel override for the global pause between downloads. Absent = inherit; 0 = no sleep; floored and capped at 600.", + audioCheck: + "Opt-in audio-integrity checking for sources that intermittently serve corrupt audio mid-download (e.g. Odysee \"original\"): the managed downloader periodically validates the in-progress `.part` file and rolls back to the last known-good snapshot on corruption. transcribe-handling only.", +}; + +// Every key a config.json may carry, in emission order. The unknown-key oracle: +// a key not listed here is dropped by every read and every write. +export const CHANNEL_CONFIG_KEYS = Object.keys( + CHANNEL_CONFIG_FIELD_DOCS, +) as ReadonlyArray<keyof ChannelConfig>; + +// The keys that are MUTABLE SYNC STATE rather than configuration: stamped by +// the sync / sweep / download passes, never by the channel form. They live in +// the same file on purpose (the file is not split); a form save preserves them +// because it unsets only the form's own fields. +export const CHANNEL_SYNC_STATE_KEYS = [ + "lastSyncedAt", + "lastFullDownloadAt", + "lastFullSweepAt", +] as const satisfies ReadonlyArray<keyof ChannelConfig>; + +export type ChannelSyncStateKey = (typeof CHANNEL_SYNC_STATE_KEYS)[number]; + export const CHANNEL_SLEEP_BETWEEN_DOWNLOADS_MAX_SECONDS = 600; // Bounds for a channel's auto-sync interval. A nonzero value is clamped into @@ -254,6 +243,7 @@ export const AUDIO_CHECK_COPY_TIMEOUT_DEFAULT_SECONDS = 30; export const AUDIO_CHECK_COPY_TIMEOUT_MIN_SECONDS = 5; export const AUDIO_CHECK_COPY_TIMEOUT_MAX_SECONDS = 120; + function clampInt( value: number, min: number, @@ -273,192 +263,152 @@ export const AUDIO_FORMAT_VALUES: ReadonlyArray<AudioFormat> = [ "opus", ]; -export function parseChannelConfig(raw: unknown): ChannelConfig | null { - if (!raw || typeof raw !== "object") return null; - const r = raw as Record<string, unknown>; - if (r.handling !== "youtube" && r.handling !== "transcribe") return null; - const config: ChannelConfig = { handling: r.handling }; - if (r.sourceKind === "video" || r.sourceKind === "social") { - config.sourceKind = r.sourceKind; - } - if (typeof r.postFetcher === "string" && r.postFetcher.trim()) { - config.postFetcher = r.postFetcher.trim(); - } - if (typeof r.socialHandle === "string" && r.socialHandle.trim()) { - config.socialHandle = r.socialHandle.trim().replace(/^@/, ""); - } - if ( - typeof r.platform === "string" && - PLATFORM_VALUES.includes(r.platform as Platform) - ) { - config.platform = r.platform as Platform; - } - if (typeof r.name === "string") config.name = r.name; - if (typeof r.url === "string") config.url = r.url; - if ( - r.audioFormat === "m4a" || - r.audioFormat === "mp3" || - r.audioFormat === "opus" - ) { - config.audioFormat = r.audioFormat; - } - if (isDownloadFormatPreset(r.downloadFormat)) { - config.downloadFormat = r.downloadFormat; - } - if (typeof r.keepSourceVideo === "boolean") { - config.keepSourceVideo = r.keepSourceVideo; - } - if ( - typeof r.keepLatest === "number" && - Number.isFinite(r.keepLatest) && - r.keepLatest >= 0 - ) { - // 0 is the "disabled" sentinel preserved as-is; positives clamp to the cap. - config.keepLatest = - r.keepLatest === 0 ? 0 : clampInt(r.keepLatest, 1, KEEP_LATEST_MAX); - } - if (r.extractionMode === "ytdlp" || r.extractionMode === "app") { - config.extractionMode = r.extractionMode; - } - if (typeof r.savedVideosDir === "string" && r.savedVideosDir.trim() !== "") { - config.savedVideosDir = r.savedVideosDir.trim(); - } - if (typeof r.dataDir === "string" && r.dataDir.trim() !== "") { - config.dataDir = r.dataDir.trim(); - } - if ( - Array.isArray(r.ytdlpExtraArgs) && - r.ytdlpExtraArgs.every((x) => typeof x === "string") - ) { - config.ytdlpExtraArgs = r.ytdlpExtraArgs as string[]; - } - if (typeof r.subLangs === "string") config.subLangs = r.subLangs; - if (typeof r.lastSyncedAt === "string") config.lastSyncedAt = r.lastSyncedAt; - if (typeof r.lastFullDownloadAt === "string") { - config.lastFullDownloadAt = r.lastFullDownloadAt; - } - if (typeof r.lastFullSweepAt === "string") { - config.lastFullSweepAt = r.lastFullSweepAt; - } - if (typeof r.excludeFromBuild === "boolean") { - config.excludeFromBuild = r.excludeFromBuild; - } - if (typeof r.excludeFromCleanup === "boolean") { - config.excludeFromCleanup = r.excludeFromCleanup; - } - if ( - typeof r.syncIntervalMinutes === "number" && - Number.isFinite(r.syncIntervalMinutes) && - r.syncIntervalMinutes >= 0 - ) { - // 0 is a sentinel ("auto-sync disabled") preserved as-is; any other value - // is clamped into the supported window. - config.syncIntervalMinutes = - r.syncIntervalMinutes === 0 - ? 0 - : clampInt( - r.syncIntervalMinutes, - SYNC_INTERVAL_MIN_MINUTES, - SYNC_INTERVAL_MAX_MINUTES, - ); - } - if ( - typeof r.fullSweepIntervalMinutes === "number" && - Number.isFinite(r.fullSweepIntervalMinutes) && - r.fullSweepIntervalMinutes >= 0 - ) { - // Same shape as syncIntervalMinutes above: 0 is the "never sweep" sentinel. - config.fullSweepIntervalMinutes = - r.fullSweepIntervalMinutes === 0 - ? 0 - : clampInt( - r.fullSweepIntervalMinutes, - SYNC_INTERVAL_MIN_MINUTES, - SYNC_INTERVAL_MAX_MINUTES, - ); - } - if (typeof r.skipLiveDownloads === "boolean") { - config.skipLiveDownloads = r.skipLiveDownloads; - } - // Download filter. Blank/whitespace patterns are dropped rather than stored, - // so an all-blank object leaves the key absent and the filter inert — the - // same "empty means inherit-off" rule the cookie overrides use. - if (r.downloadFilter && typeof r.downloadFilter === "object") { - const df = r.downloadFilter as Record<string, unknown>; - const include = typeof df.include === "string" ? df.include.trim() : ""; - const exclude = typeof df.exclude === "string" ? df.exclude.trim() : ""; - const includeLivestreams = df.includeLivestreams === true; - // ONLY ALONGSIDE A REAL FILTER, and only when it is not the default. The - // mode says what to do with a REJECTED livestream, so with nothing to - // reject it names a decision that can never be taken — storing it would put - // a setting on the Configure form that does nothing and explains nothing. - const rejectedLivestreams = isRejectedLivestreamMode(df.rejectedLivestreams) - ? df.rejectedLivestreams - : "skip"; - if (include || exclude || includeLivestreams) { - config.downloadFilter = { - ...(include ? { include } : {}), - ...(exclude ? { exclude } : {}), - ...(includeLivestreams ? { includeLivestreams: true } : {}), - ...(rejectedLivestreams !== "skip" ? { rejectedLivestreams } : {}), - }; - } - } - if (typeof r.cookiesFromBrowser === "string" && r.cookiesFromBrowser.trim()) { - config.cookiesFromBrowser = r.cookiesFromBrowser.trim(); +// Only an object whose `handling` is valid is a channel. Everything else — +// `null`, an array, a number, `{}` — reads as "not a channel". +export function isChannelShape(raw: unknown): raw is Record<string, unknown> { + if (!raw || typeof raw !== "object") return false; + const h = (raw as Record<string, unknown>).handling; + return h === "youtube" || h === "transcribe"; +} + +const isFiniteNumber = (v: unknown): v is number => + typeof v === "number" && Number.isFinite(v); +const trimmedNonBlank = (v: unknown): string | undefined => + typeof v === "string" && v.trim() ? v.trim() : undefined; +const bool = (v: unknown): boolean | undefined => + typeof v === "boolean" ? v : undefined; +const str = (v: unknown): string | undefined => + typeof v === "string" ? v : undefined; + +// 0 is a sentinel ("disabled") preserved as-is; any other non-negative value +// is clamped into [min, max]. +const zeroOrClamped = (min: number, max: number) => (v: unknown): number | undefined => + isFiniteNumber(v) && v >= 0 ? (v === 0 ? 0 : clampInt(v, min, max)) : undefined; + +export function coerceDownloadFilter(v: unknown): DownloadFilterConfig | undefined { + // Blank/whitespace patterns are dropped rather than stored, so an all-blank + // object leaves the key absent and the filter inert — the same "empty means + // inherit-off" rule the cookie overrides use. + if (!v || typeof v !== "object") return undefined; + const df = v as Record<string, unknown>; + const include = typeof df.include === "string" ? df.include.trim() : ""; + const exclude = typeof df.exclude === "string" ? df.exclude.trim() : ""; + const includeLivestreams = df.includeLivestreams === true; + // ONLY ALONGSIDE A REAL FILTER, and only when it is not the default. The + // mode says what to do with a REJECTED livestream, so with nothing to reject + // it names a decision that can never be taken — storing it would put a + // setting on the Configure form that does nothing and explains nothing. + const rejectedLivestreams = isRejectedLivestreamMode(df.rejectedLivestreams) + ? df.rejectedLivestreams + : "skip"; + if (!include && !exclude && !includeLivestreams) return undefined; + return { + ...(include ? { include } : {}), + ...(exclude ? { exclude } : {}), + ...(includeLivestreams ? { includeLivestreams: true } : {}), + ...(rejectedLivestreams !== "skip" ? { rejectedLivestreams } : {}), + }; +} + +export function coerceAudioCheck(v: unknown): AudioCheckConfig | undefined { + if (!v || typeof v !== "object") return undefined; + const a = v as Record<string, unknown>; + if (typeof a.enabled !== "boolean") return undefined; + const ac: AudioCheckConfig = { enabled: a.enabled }; + if (isFiniteNumber(a.intervalSeconds)) { + ac.intervalSeconds = clampInt( + a.intervalSeconds, + AUDIO_CHECK_INTERVAL_MIN_SECONDS, + AUDIO_CHECK_INTERVAL_MAX_SECONDS, + ); } - if (isCookieMode(r.cookieMode)) { - config.cookieMode = r.cookieMode; + if (isFiniteNumber(a.maxRollbacks)) { + ac.maxRollbacks = clampInt( + a.maxRollbacks, + AUDIO_CHECK_MAX_ROLLBACKS_MIN, + AUDIO_CHECK_MAX_ROLLBACKS_MAX, + ); } - if ( - typeof r.sleepBetweenDownloadsSeconds === "number" && - Number.isFinite(r.sleepBetweenDownloadsSeconds) && - r.sleepBetweenDownloadsSeconds >= 0 - ) { - config.sleepBetweenDownloadsSeconds = Math.min( - Math.floor(r.sleepBetweenDownloadsSeconds), - CHANNEL_SLEEP_BETWEEN_DOWNLOADS_MAX_SECONDS, + if (isFiniteNumber(a.copyTimeoutSeconds)) { + ac.copyTimeoutSeconds = clampInt( + a.copyTimeoutSeconds, + AUDIO_CHECK_COPY_TIMEOUT_MIN_SECONDS, + AUDIO_CHECK_COPY_TIMEOUT_MAX_SECONDS, ); } - if (r.audioCheck && typeof r.audioCheck === "object") { - const a = r.audioCheck as Record<string, unknown>; - if (typeof a.enabled === "boolean") { - const ac: AudioCheckConfig = { enabled: a.enabled }; - if ( - typeof a.intervalSeconds === "number" && - Number.isFinite(a.intervalSeconds) - ) { - ac.intervalSeconds = clampInt( - a.intervalSeconds, - AUDIO_CHECK_INTERVAL_MIN_SECONDS, - AUDIO_CHECK_INTERVAL_MAX_SECONDS, - ); - } - if ( - typeof a.maxRollbacks === "number" && - Number.isFinite(a.maxRollbacks) - ) { - ac.maxRollbacks = clampInt( - a.maxRollbacks, - AUDIO_CHECK_MAX_ROLLBACKS_MIN, - AUDIO_CHECK_MAX_ROLLBACKS_MAX, - ); - } - if ( - typeof a.copyTimeoutSeconds === "number" && - Number.isFinite(a.copyTimeoutSeconds) - ) { - ac.copyTimeoutSeconds = clampInt( - a.copyTimeoutSeconds, - AUDIO_CHECK_COPY_TIMEOUT_MIN_SECONDS, - AUDIO_CHECK_COPY_TIMEOUT_MAX_SECONDS, - ); - } - if (typeof a.resumeDuringProbe === "boolean") { - ac.resumeDuringProbe = a.resumeDuringProbe; - } - config.audioCheck = ac; - } + if (typeof a.resumeDuringProbe === "boolean") { + ac.resumeDuringProbe = a.resumeDuringProbe; + } + return ac; +} + +// THE ONE SET OF COERCIONS for a config.json field: raw value in, the legal +// value — or `undefined`, meaning "omit the key" — out. Each is total over +// `unknown`. parseChannelConfig (below, client-safe) and channelConfigSchema +// (lib/channelConfigSchema.ts, server-only zod) both compose exactly these, so +// the two cannot disagree. Keyed and ordered like CHANNEL_CONFIG_FIELD_DOCS. +export const CHANNEL_CONFIG_COERCIONS: { + readonly [K in keyof ChannelConfig]-?: (v: unknown) => ChannelConfig[K] | undefined; +} = { + handling: (v) => (v === "youtube" || v === "transcribe" ? v : undefined), + sourceKind: (v) => (v === "video" || v === "social" ? v : undefined), + postFetcher: trimmedNonBlank, + socialHandle: (v) => + typeof v === "string" && v.trim() ? v.trim().replace(/^@/, "") : undefined, + platform: (v) => + typeof v === "string" && PLATFORM_VALUES.includes(v as Platform) + ? (v as Platform) + : undefined, + name: str, + url: str, + audioFormat: (v) => (v === "m4a" || v === "mp3" || v === "opus" ? v : undefined), + downloadFormat: (v) => (isDownloadFormatPreset(v) ? v : undefined), + keepSourceVideo: bool, + keepLatest: zeroOrClamped(1, KEEP_LATEST_MAX), + extractionMode: (v) => (v === "ytdlp" || v === "app" ? v : undefined), + savedVideosDir: trimmedNonBlank, + dataDir: trimmedNonBlank, + ytdlpExtraArgs: (v) => + Array.isArray(v) && v.every((x) => typeof x === "string") + ? (v as string[]) + : undefined, + subLangs: str, + lastSyncedAt: str, + lastFullDownloadAt: str, + lastFullSweepAt: str, + excludeFromBuild: bool, + excludeFromCleanup: bool, + syncIntervalMinutes: zeroOrClamped(SYNC_INTERVAL_MIN_MINUTES, SYNC_INTERVAL_MAX_MINUTES), + fullSweepIntervalMinutes: zeroOrClamped( + SYNC_INTERVAL_MIN_MINUTES, + SYNC_INTERVAL_MAX_MINUTES, + ), + skipLiveDownloads: bool, + downloadFilter: coerceDownloadFilter, + cookiesFromBrowser: trimmedNonBlank, + cookieMode: (v) => (isCookieMode(v) ? v : undefined), + sleepBetweenDownloadsSeconds: (v) => + isFiniteNumber(v) && v >= 0 + ? Math.min(Math.floor(v), CHANNEL_SLEEP_BETWEEN_DOWNLOADS_MAX_SECONDS) + : undefined, + audioCheck: coerceAudioCheck, +}; + +// A channel's config.json as a ChannelConfig, or null when it is not a channel +// (see isChannelShape). An ill-typed or out-of-range OPTIONAL key is OMITTED — +// never defaulted — so `"k" in config` means "the file set it", which the +// inherit-the-global rule for every override depends on. Unknown keys (and the +// retired `excludeFromSync`) are dropped. +// +// Client-safe (no zod): the channel form and five other `"use client"` modules +// import this file. The server reads and writes through +// lib/channelConfigSchema.ts, which composes the same coercions. +export function parseChannelConfig(raw: unknown): ChannelConfig | null { + if (!isChannelShape(raw)) return null; + const out: Partial<Record<keyof ChannelConfig, unknown>> = {}; + for (const key of CHANNEL_CONFIG_KEYS) { + const value = CHANNEL_CONFIG_COERCIONS[key](raw[key]); + if (value !== undefined) out[key] = value; } - return config; + return out as ChannelConfig; } diff --git a/common/lib/channelConfigSchema.test.ts b/common/lib/channelConfigSchema.test.ts @@ -0,0 +1,158 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + AUDIO_CHECK_COPY_TIMEOUT_MAX_SECONDS, + AUDIO_CHECK_INTERVAL_MIN_SECONDS, + AUDIO_CHECK_MAX_ROLLBACKS_MAX, + CHANNEL_CONFIG_COERCIONS, + CHANNEL_CONFIG_FIELD_DOCS, + CHANNEL_CONFIG_KEYS, + CHANNEL_SLEEP_BETWEEN_DOWNLOADS_MAX_SECONDS, + CHANNEL_SYNC_STATE_KEYS, + KEEP_LATEST_MAX, + SYNC_INTERVAL_MAX_MINUTES, + parseChannelConfig, + type ChannelConfig, +} from "./channelConfig"; +import type { z } from "zod"; +import { + channelConfigObjectSchema, + channelConfigSchema, +} from "./channelConfigSchema"; + +const parse = (raw: unknown) => channelConfigSchema.parse(raw); + +// The zod object's output names exactly ChannelConfig's keys, and fits it — +// `handling` aside: its coercion may answer undefined, and it is isChannelShape, +// run first, that guarantees it. +type Out = z.output<typeof channelConfigObjectSchema>; +const sameKeys: [keyof Out] extends [keyof ChannelConfig] + ? [keyof ChannelConfig] extends [keyof Out] + ? true + : false + : false = true; +const fits: Omit<Out, "handling"> extends Omit<ChannelConfig, "handling"> + ? true + : false = true; + +test("one key list: docs = coercions = schema shape, sync-state keys inside it", () => { + assert.deepEqual(Object.keys(CHANNEL_CONFIG_COERCIONS), [...CHANNEL_CONFIG_KEYS]); + assert.deepEqual(Object.keys(channelConfigObjectSchema.shape), [...CHANNEL_CONFIG_KEYS]); + assert.deepEqual(Object.keys(CHANNEL_CONFIG_FIELD_DOCS), [...CHANNEL_CONFIG_KEYS]); + assert.equal(CHANNEL_CONFIG_KEYS.length, 29); + assert.equal(sameKeys, true); + assert.equal(fits, true); + for (const k of CHANNEL_SYNC_STATE_KEYS) assert.ok(CHANNEL_CONFIG_KEYS.includes(k), k); +}); + +test("not a channel → null", () => { + for (const raw of [null, undefined, 3, "youtube", [], {}, { handling: "x" }, { handling: null }]) { + assert.equal(parse(raw), null, JSON.stringify(raw)); + assert.equal(parseChannelConfig(raw), null, JSON.stringify(raw)); + } +}); + +test("an invalid optional key is OMITTED, not defaulted", () => { + const cfg = parse({ + handling: "youtube", + keepLatest: -1, + platform: "nope", + postFetcher: " ", + audioCheck: { enabled: "yes" }, + downloadFilter: { include: " ", rejectedLivestreams: "chat-only" }, + ytdlpExtraArgs: ["a", 1], + syncIntervalMinutes: "60", + })!; + assert.deepEqual(cfg, { handling: "youtube" }); + for (const k of ["keepLatest", "platform", "postFetcher", "audioCheck", "downloadFilter", "ytdlpExtraArgs", "syncIntervalMinutes"]) { + assert.equal(k in cfg, false, k); + } +}); + +test("clamps: 0 is preserved as the disabled sentinel, positives clamp, fractions floor", () => { + const cfg = parse({ + handling: "transcribe", + keepLatest: 0, + syncIntervalMinutes: 0, + fullSweepIntervalMinutes: 0, + sleepBetweenDownloadsSeconds: 0, + })!; + assert.equal(cfg.keepLatest, 0); + assert.equal(cfg.syncIntervalMinutes, 0); + assert.equal(cfg.fullSweepIntervalMinutes, 0); + assert.equal(cfg.sleepBetweenDownloadsSeconds, 0); + const big = parse({ + handling: "transcribe", + keepLatest: 1e9, + syncIntervalMinutes: 0.5, + fullSweepIntervalMinutes: 1e9, + sleepBetweenDownloadsSeconds: 1e9, + audioCheck: { enabled: true, intervalSeconds: 1, maxRollbacks: 1e9, copyTimeoutSeconds: 1e9 }, + })!; + assert.equal(big.keepLatest, KEEP_LATEST_MAX); + assert.equal(big.syncIntervalMinutes, 1); + assert.equal(big.fullSweepIntervalMinutes, SYNC_INTERVAL_MAX_MINUTES); + assert.equal(big.sleepBetweenDownloadsSeconds, CHANNEL_SLEEP_BETWEEN_DOWNLOADS_MAX_SECONDS); + assert.deepEqual(big.audioCheck, { + enabled: true, + intervalSeconds: AUDIO_CHECK_INTERVAL_MIN_SECONDS, + maxRollbacks: AUDIO_CHECK_MAX_ROLLBACKS_MAX, + copyTimeoutSeconds: AUDIO_CHECK_COPY_TIMEOUT_MAX_SECONDS, + }); + assert.equal(parse({ handling: "youtube", keepLatest: 2.9 })!.keepLatest, 2); +}); + +test("trims and normalises", () => { + const cfg = parse({ + handling: "youtube", + socialHandle: " @me ", + savedVideosDir: " /x ", + dataDir: " /d ", + cookiesFromBrowser: " firefox ", + downloadFilter: { include: " a ", exclude: "", includeLivestreams: true, rejectedLivestreams: "skip" }, + })!; + assert.equal(cfg.socialHandle, "me"); + assert.equal(cfg.savedVideosDir, "/x"); + assert.equal(cfg.dataDir, "/d"); + assert.equal(cfg.cookiesFromBrowser, "firefox"); + assert.deepEqual(cfg.downloadFilter, { include: "a", includeLivestreams: true }); +}); + +test("excludeFromSync and unknown keys are dropped; every emitted key is declared", () => { + const cfg = parse({ handling: "youtube", excludeFromSync: true, bogus: 1, name: "n" })!; + assert.deepEqual(cfg, { handling: "youtube", name: "n" }); + const full = parse({ + handling: "transcribe", + sourceKind: "social", + name: "n", + url: "u", + lastSyncedAt: "t", + audioCheck: { enabled: false }, + })!; + for (const k of Object.keys(full)) { + assert.ok((CHANNEL_CONFIG_KEYS as readonly string[]).includes(k), k); + } +}); + +test("the zod schema and the client-safe parser agree, key order included", () => { + const vals: unknown[] = [undefined, null, 0, -1, 2.5, 1e9, "", " x ", "@h", true, false, [], ["a"], {}, { enabled: true, intervalSeconds: 5 }, { include: "a" }]; + const pools: Record<string, unknown[]> = { + handling: ["youtube", "transcribe"], + platform: ["youtube", "rumble", "nope"], + cookieMode: ["always", "never", "nope"], + }; + let seed = 7; + const rand = () => ((seed = (seed * 1103515245 + 12345) % 2 ** 31) / 2 ** 31); + for (let i = 0; i < 3000; i++) { + const raw: Record<string, unknown> = {}; + for (const k of [...CHANNEL_CONFIG_KEYS, "excludeFromSync", "bogus"]) { + if (rand() < 0.5) continue; + const pool = pools[k] && rand() < 0.8 ? pools[k] : vals; + raw[k] = pool[Math.floor(rand() * pool.length)]; + } + const a = parseChannelConfig(raw); + const b = parse(raw); + assert.deepEqual(b, a, JSON.stringify(raw)); + assert.deepEqual(Object.keys(b ?? {}), Object.keys(a ?? {})); + } +}); diff --git a/common/lib/channelConfigSchema.ts b/common/lib/channelConfigSchema.ts @@ -0,0 +1,90 @@ +// THE CHANNEL config.json SCHEMA, as zod — the server's reader and writer. +// +// one-core phase 3 slice 4b, on slice 4a's pattern: every field is +// `settingsField(coerce)`, and every coerce is the entry for that key in +// CHANNEL_CONFIG_COERCIONS (lib/channelConfig.ts) — the ONE set of coercions. +// The client-safe `parseChannelConfig` in that file composes the same functions +// without zod; this module is the same list as a zod object, `.describe()`d +// from CHANNEL_CONFIG_FIELD_DOCS, for the server paths +// (controller/channels.ts: readChannelConfigFile, writeChannelConfig). +// +// WHY TWO COMPOSITIONS AND NOT ONE. channelConfig.ts is imported as VALUES by +// six `"use client"` modules; a zod import there would ship zod to the browser. +// So the coercions live there, zod-free, and this server-only module wraps +// them (slice 4b record, deviation 2). channelConfigSchema.test.ts pins that +// the two agree. +// +// THE OMIT RULE. A config's optional keys are OMITTED when invalid, never +// defaulted: `"keepLatest" in config` means the file set it. zod emits a key +// whose transform returned `undefined` when that key was PRESENT in the input, +// so `stripUndefined` runs after the object parse. The nested objects +// (`audioCheck`, `downloadFilter`) are built by their coercions with no +// undefined members, so the strip is one level deep. +// +// NOT A CHANNEL → null. The object parse runs only for a value +// `isChannelShape` accepts (an object with a valid `handling`). + +import { z } from "zod"; +import { + CHANNEL_CONFIG_COERCIONS, + CHANNEL_CONFIG_FIELD_DOCS, + isChannelShape, + type ChannelConfig, +} from "./channelConfig"; +import { settingsField } from "./settingsFieldSchemas"; + +function field<K extends keyof ChannelConfig>(key: K) { + // The cast is the correlated-union case TS cannot follow through a `-?` + // mapped type; the map's own declaration is what checks each entry. + const coerce = CHANNEL_CONFIG_COERCIONS[key] as ( + v: unknown, + ) => ChannelConfig[K] | undefined; + return settingsField(coerce).describe(CHANNEL_CONFIG_FIELD_DOCS[key]); +} + +export const channelConfigObjectSchema = z.object({ + handling: field("handling"), + sourceKind: field("sourceKind"), + postFetcher: field("postFetcher"), + socialHandle: field("socialHandle"), + platform: field("platform"), + name: field("name"), + url: field("url"), + audioFormat: field("audioFormat"), + downloadFormat: field("downloadFormat"), + keepSourceVideo: field("keepSourceVideo"), + keepLatest: field("keepLatest"), + extractionMode: field("extractionMode"), + savedVideosDir: field("savedVideosDir"), + dataDir: field("dataDir"), + ytdlpExtraArgs: field("ytdlpExtraArgs"), + subLangs: field("subLangs"), + lastSyncedAt: field("lastSyncedAt"), + lastFullDownloadAt: field("lastFullDownloadAt"), + lastFullSweepAt: field("lastFullSweepAt"), + excludeFromBuild: field("excludeFromBuild"), + excludeFromCleanup: field("excludeFromCleanup"), + syncIntervalMinutes: field("syncIntervalMinutes"), + fullSweepIntervalMinutes: field("fullSweepIntervalMinutes"), + skipLiveDownloads: field("skipLiveDownloads"), + downloadFilter: field("downloadFilter"), + cookiesFromBrowser: field("cookiesFromBrowser"), + cookieMode: field("cookieMode"), + sleepBetweenDownloadsSeconds: field("sleepBetweenDownloadsSeconds"), + audioCheck: field("audioCheck"), +}); + +// Drop every own key whose value is `undefined` (the OMIT rule, above). +export function stripUndefined<T extends object>(obj: T): T { + const out: Record<string, unknown> = {}; + for (const [k, v] of Object.entries(obj)) if (v !== undefined) out[k] = v; + return out as T; +} + +export const channelConfigSchema: z.ZodType<ChannelConfig | null, unknown> = + z.unknown().transform((raw): ChannelConfig | null => + isChannelShape(raw) + ? (stripUndefined(channelConfigObjectSchema.parse(raw)) as ChannelConfig) + : null, + ); +