// Produce a compact, self-describing parsed copy of transcript.live_chat.json // alongside the raw file so that downstream consumers (buildIndex, archives) // can skip re-parsing potentially huge live-chat JSON-lines payloads. Shape // matches NormalizedTranscript with source: "live_chat". import path from "node:path"; import { lstat, readFile, stat } from "node:fs/promises"; import { writeJsonAtomic } from "../lib/jsonFile-server"; import { tierMediaFile } from "../lib/mediaTier-server"; import { parseLiveChat } from "../lib/liveChat"; import type { Cue } from "../lib/vtt"; import { summarize, type RawMetadata } from "../lib/transcripts-server"; import { type NormalizedLiveChat, } from "./normalizeTranscript"; import { LIVE_CHAT_CUES_FILENAME, LIVE_CHAT_FILENAME, META_FILENAME, } from "../lib/videoStatus"; export const LIVE_CHAT_FILE_VERSION = 1; export type NormalizeLiveChatOptions = { videoDir: string; channelSlug: string; configName?: string; log?: (msg: string) => void; force?: boolean; // Tier the raw replay after writing the cues (default true). The export // build's archive pass passes false: a build reads, it does not move media. tier?: boolean; }; export type NormalizeLiveChatOutcome = | { status: "wrote"; cuesPath: string } | { status: "fresh"; cuesPath: string } | { status: "skipped"; reason: "no-live-chat" | "no-metadata" }; async function mtimeMs(p: string): Promise { try { return (await stat(p)).mtimeMs; } catch { return null; } } // THE RAW REPLAY'S mtime, WITHOUT FOLLOWING A LINK (release 17). The raw file // is media: on a tiered channel `transcript.live_chat.json` is a relative link // into channels//media, possibly on another drive. The tier hook copies // the file's mtime onto the link (`lutimes`), so the link answers the same // freshness question from the corpus disk — the index build asks it per video // and must never reach the media drive to do so. async function rawMtimeMs(p: string): Promise { try { return (await lstat(p)).mtimeMs; } catch { return null; } } // TODO: live_chat files can be hundreds of MB. parseLiveChat already splits // on "\n", so a streaming readline variant is straightforward if we hit a // memory wall. For now this matches the readFile pattern used by the // transcript normalizer. export async function normalizeLiveChat( opts: NormalizeLiveChatOptions, ): Promise { const metaPath = path.join(opts.videoDir, META_FILENAME); const rawPath = path.join(opts.videoDir, LIVE_CHAT_FILENAME); const cuesPath = path.join(opts.videoDir, LIVE_CHAT_CUES_FILENAME); const [metaStatMs, rawStatMs, cuesMs] = await Promise.all([ mtimeMs(metaPath), rawMtimeMs(rawPath), mtimeMs(cuesPath), ]); if (metaStatMs === null) return { status: "skipped", reason: "no-metadata" }; if (rawStatMs === null) return { status: "skipped", reason: "no-live-chat" }; if ( !opts.force && cuesMs !== null && cuesMs >= metaStatMs && cuesMs >= rawStatMs ) { return { status: "fresh", cuesPath }; } const metaRaw = await readFile(metaPath, "utf8"); const parsedMeta = JSON.parse(metaRaw) as RawMetadata; const summary = summarize( opts.channelSlug, path.basename(opts.videoDir), parsedMeta, opts.configName, ); const rawLiveChat = await readFile(rawPath, "utf8"); let cues: Cue[]; try { cues = parseLiveChat(rawLiveChat); } catch (err) { throw new Error( `Failed to parse ${LIVE_CHAT_FILENAME} for ${opts.channelSlug}/${path.basename(opts.videoDir)}: ${(err as Error).message}`, ); } const out: NormalizedLiveChat = { version: LIVE_CHAT_FILE_VERSION, source: "live_chat", ...summary, cues, }; // Compact, no trailing newline: live_chat.cues.json's historical bytes. await writeJsonAtomic(cuesPath, out, { indent: 0, newline: false }); opts.log?.( `Normalized live chat ${opts.channelSlug}/${path.basename(opts.videoDir)} (${cues.length} cues)`, ); // THE MEDIA TIER'S HOOK (release 17): the raw replay is read once, here, and // every other reader uses the cues just written — so it moves into // channels//media now (a relative link stays). Never throws. if (opts.tier !== false) await tierMediaFile(opts.videoDir, LIVE_CHAT_FILENAME, { onLog: opts.log ? (line) => opts.log?.(line.trimEnd()) : undefined, }); return { status: "wrote", cuesPath }; } export async function readNormalizedLiveChat( cuesPath: string, ): Promise { try { const raw = await readFile(cuesPath, "utf8"); const parsed = JSON.parse(raw) as NormalizedLiveChat; if (typeof parsed.version !== "number") return null; if (parsed.source !== "live_chat") return null; return parsed; } catch { return null; } } // Sidecar is fresh iff it exists and its mtime is >= both metadata.info.json // and transcript.live_chat.json. Used by buildIndex to decide whether to // trust the cached cues without re-parsing the raw payload. export async function isLiveChatCuesFresh( videoDir: string, ): Promise<{ fresh: boolean; cuesPath: string }> { const cuesPath = path.join(videoDir, LIVE_CHAT_CUES_FILENAME); const metaPath = path.join(videoDir, META_FILENAME); const rawPath = path.join(videoDir, LIVE_CHAT_FILENAME); const [cuesMs, metaMs, rawMs] = await Promise.all([ mtimeMs(cuesPath), mtimeMs(metaPath), rawMtimeMs(rawPath), ]); if (cuesMs === null || metaMs === null || rawMs === null) { return { fresh: false, cuesPath }; } if (cuesMs < metaMs) return { fresh: false, cuesPath }; if (cuesMs < rawMs) return { fresh: false, cuesPath }; return { fresh: true, cuesPath }; }