// A CHANNEL'S COVERAGE: WHAT IT HOLDS, BY DATE, AND WHERE THE GAPS ARE // (release 19 A9; the MCP's `channel_coverage`, `GET /api/ops/coverage`). // // Every held video (a data// directory) is dated with `coverageDate` // (lib/recordedDate.ts, release 20 D1): its RECORDED date when the channel has // a title rule (`config.recordedDate`) and the title yields one, else its // upload date. A VOD mirror's coverage is then the streams' days, not the // copies'. The dates come off disk, not the index, so a video imported an hour // ago counts. // // CHEAP ON A BIG CHANNEL: metadata.info.json can run to megabytes (formats), so // a small file is parsed whole and a big one is read at both ends — `title` // sits near the head of yt-dlp's JSON, `upload_date` near the tail (the same // tail read recencyIndex.ts measured at 0.19 ms a file). A video with neither // is `undated`, never guessed. Nothing is written. import path from "node:path"; import { open, readFile, stat } from "node:fs/promises"; import type { Paths } from "../lib/paths"; import { assertChannelTextReadable } from "../lib/channelMedia"; import { mapConcurrent } from "../lib/concurrency"; import { compileRecordedDateRule, coverageDate, deriveRecordedDate } from "../lib/recordedDate"; import { readChannelConfig } from "./channels"; import { heldVideoIds } from "./videoCues"; const WHOLE_FILE_BYTES = 256 * 1024; const HEAD_BYTES = 16 * 1024; const TAIL_BYTES = 8 * 1024; export const DEFAULT_GAP_DAYS = 30; export type DatedVideo = { id: string; date: string; // YYYYMMDD uploadDate: string; recordedDate?: string; title?: string; }; export type ChannelCoverage = { slug: string; // The channel's recorded-date title rule, when it has one. titlePattern: string | null; held: number; dated: number; // Dated by the title rule / by the upload date. byRecordedDate: number; byUploadDate: number; undated: string[]; first: string | null; last: string | null; // Videos per year and per month ("2024", "2024-03"), only those with any. byYear: Record; byMonth: Record; // Spans longer than `gapDays` with no held video, oldest first: the last // held date before, the first after, and the days between. gapDays: number; gaps: { after: string; before: string; days: number }[]; // With `list`: every dated video in the window, oldest first. videos?: DatedVideo[]; }; const UPLOAD_RE = /"upload_date"\s*:\s*"(\d{8})"/; const TITLE_RE = /"title"\s*:\s*("(?:[^"\\]|\\.)*")/; function fromText(text: string): { uploadDate?: string; title?: string } { const up = UPLOAD_RE.exec(text)?.[1]; const rawTitle = TITLE_RE.exec(text)?.[1]; let title: string | undefined; if (rawTitle) { try { title = JSON.parse(rawTitle) as string; } catch { /* a title cut by the read boundary is no title */ } } return { ...(up ? { uploadDate: up } : {}), ...(title ? { title } : {}) }; } // The upload date and title of one video, from its metadata.info.json. export async function readDateAndTitle(videoDir: string): Promise<{ uploadDate?: string; title?: string }> { const file = path.join(videoDir, "metadata.info.json"); const st = await stat(file).catch(() => null); if (!st?.isFile()) return {}; if (st.size <= WHOLE_FILE_BYTES) { try { const j = JSON.parse(await readFile(file, "utf8")) as { upload_date?: unknown; title?: unknown }; return { ...(typeof j.upload_date === "string" && /^\d{8}$/.test(j.upload_date) ? { uploadDate: j.upload_date } : {}), ...(typeof j.title === "string" && j.title ? { title: j.title } : {}), }; } catch { return {}; } } const fh = await open(file, "r"); try { const head = Buffer.alloc(HEAD_BYTES); const tail = Buffer.alloc(TAIL_BYTES); await fh.read(head, 0, HEAD_BYTES, 0); await fh.read(tail, 0, TAIL_BYTES, st.size - TAIL_BYTES); // A multi-byte sequence split by a read boundary decodes as U+FFFD; the // regexes only take a whole quoted string, so a cut title is no title. const h = fromText(head.toString("utf8")); const t = fromText(tail.toString("utf8")); const uploadDate = t.uploadDate ?? h.uploadDate; const title = h.title ?? t.title; return { ...(uploadDate ? { uploadDate } : {}), ...(title ? { title } : {}) }; } finally { await fh.close(); } } function daysBetween(a: string, b: string): number { const toMs = (d: string) => Date.UTC(Number(d.slice(0, 4)), Number(d.slice(4, 6)) - 1, Number(d.slice(6, 8))); return Math.round((toMs(b) - toMs(a)) / 86_400_000); } // The pure half: dated videos → the summary (byYear, byMonth, gaps). export function summarizeCoverage( videos: DatedVideo[], opts: { gapDays?: number } = {}, ): Pick { const gapDays = opts.gapDays ?? DEFAULT_GAP_DAYS; const sorted = [...videos].sort((a, b) => a.date.localeCompare(b.date) || a.id.localeCompare(b.id)); const byYear: Record = {}; const byMonth: Record = {}; const gaps: ChannelCoverage["gaps"] = []; for (let i = 0; i < sorted.length; i++) { const d = sorted[i].date; byYear[d.slice(0, 4)] = (byYear[d.slice(0, 4)] ?? 0) + 1; const m = `${d.slice(0, 4)}-${d.slice(4, 6)}`; byMonth[m] = (byMonth[m] ?? 0) + 1; if (i > 0) { const prev = sorted[i - 1].date; const days = daysBetween(prev, d); if (days > gapDays) gaps.push({ after: prev, before: d, days }); } } return { first: sorted[0]?.date ?? null, last: sorted.at(-1)?.date ?? null, byYear, byMonth, gapDays, gaps, }; } export async function channelCoverage( paths: Paths, slug: string, opts: { gapDays?: number; from?: string; to?: string; list?: boolean } = {}, ): Promise { const config = await readChannelConfig(paths, slug); if (!config) throw new Error(`Channel "${slug}" not found`); // The text guard: an unmounted or moving drive is not an empty channel. await assertChannelTextReadable(paths, slug, config); const dataDir = path.join(paths.channelsDir, slug, "data"); const ids = [...(await heldVideoIds(dataDir))].sort(); const rule = config.recordedDate ? compileRecordedDateRule(config.recordedDate) : null; const read = await mapConcurrent(ids, 32, async (id) => ({ id, ...(await readDateAndTitle(path.join(dataDir, id))) })); const dated: DatedVideo[] = []; const undated: string[] = []; let byRecordedDate = 0; for (const r of read) { if (!r.uploadDate) { undated.push(r.id); continue; } const recordedDate = rule && r.title ? deriveRecordedDate(rule, r.title, r.uploadDate) : undefined; if (recordedDate) byRecordedDate++; dated.push({ id: r.id, date: coverageDate({ uploadDate: r.uploadDate, ...(recordedDate ? { recordedDate } : {}) }), uploadDate: r.uploadDate, ...(recordedDate ? { recordedDate } : {}), ...(r.title ? { title: r.title } : {}), }); } const inWindow = dated.filter((v) => (!opts.from || v.date >= opts.from) && (!opts.to || v.date <= opts.to)); const summary = summarizeCoverage(inWindow, { gapDays: opts.gapDays }); return { slug, titlePattern: config.recordedDate?.titlePattern ?? null, held: ids.length, dated: dated.length, byRecordedDate, byUploadDate: dated.length - byRecordedDate, undated, ...summary, ...(opts.list ? { videos: [...inWindow].sort((a, b) => a.date.localeCompare(b.date) || a.id.localeCompare(b.id)) } : {}), }; }