commit 2db84f3f1182a18da57f81d1b6ed2170d85c6d6e
parent 98d701bb98444d3c6c075bf2b5dbf82e4708e62b
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 18 May 2026 20:30:49 -0400
Export live chat
Diffstat:
9 files changed, 799 insertions(+), 11 deletions(-)
diff --git a/common/controller/archiveLiveChat.ts b/common/controller/archiveLiveChat.ts
@@ -0,0 +1,374 @@
+// Build archives of every channel's parsed live_chat.cues.json files.
+// Mirrors archiveTranscripts but produces a separate archive so the
+// transcript archives stay scoped to spoken-content cues.
+//
+// archiveLiveChat() — one <slug>.live_chat.<ext> per channel
+// archiveCombinedLiveChat() — one all-live-chat.<ext> total
+//
+// Internal layout of <slug>.live_chat.<ext>:
+// <slug>/channel.json
+// <slug>/<videoId>/live_chat.cues.json
+//
+// Internal layout of all-live-chat.<ext>:
+// manifest.json
+// <slug>/channel.json
+// <slug>/<videoId>/live_chat.cues.json
+
+import path from "node:path";
+import {
+ link,
+ mkdir,
+ readdir,
+ rm,
+ writeFile,
+ stat,
+} from "node:fs/promises";
+import pLimit from "p-limit";
+import { listChannels, type ChannelStat } from "./channels";
+import { normalizeLiveChat } from "./normalizeLiveChat";
+import {
+ archiveExtension,
+ compressArchive,
+ resolveBuild,
+ writeTransformedCues,
+} from "./archiveTranscripts";
+import { LIVE_CHAT_CUES_FILENAME } from "../lib/videoStatus";
+import { detectPlatform } from "../lib/platform";
+import type { Paths } from "../lib/paths";
+import {
+ type ArchiveBuildOptions,
+} from "../lib/archiveOptions";
+
+export type ArchiveLiveChatOptions = {
+ paths: Paths;
+ onLog?: (msg: string) => void;
+ signal?: AbortSignal;
+ concurrency?: number;
+ channelSlugs?: string[];
+ build?: Partial<ArchiveBuildOptions>;
+};
+
+export type ArchiveLiveChatResult = {
+ archives: { slug: string; archivePath: string; liveChatCount: number }[];
+ totalLiveChats: number;
+ skipped: { slug: string; reason: string }[];
+};
+
+export type ArchiveCombinedLiveChatResult = {
+ archivePath: string | null;
+ channels: { slug: string; liveChatCount: number }[];
+ totalLiveChats: number;
+ skipped: { slug: string; reason: string }[];
+};
+
+const COMBINED_BASENAME = "all-live-chat";
+const COMBINED_STAGING_DIR = "all-live-chat";
+const ARCHIVE_INFIX = "live_chat";
+
+function archivesDir(paths: Paths): string {
+ return path.join(paths.transcriptsDir, "export", "archives");
+}
+
+// Keep live-chat staging isolated from transcript staging so concurrent runs
+// (e.g. on separate queues) can't trample each other.
+function stagingDir(paths: Paths): string {
+ return path.join(paths.transcriptsDir, "export", ".staging", "live-chat");
+}
+
+function channelManifest(ch: ChannelStat, liveChatCount: number) {
+ return {
+ slug: ch.slug,
+ name: ch.config.name ?? null,
+ handling: ch.config.handling,
+ platform:
+ ch.config.platform ?? detectPlatform(ch.config.url ?? null) ?? null,
+ url: ch.config.url ?? null,
+ liveChatCount,
+ generatedAt: new Date().toISOString(),
+ };
+}
+
+// Hard-link or transform-write every video's live_chat.cues.json from
+// <channelsDir>/<slug>/data/ into <destDir>/<videoId>/live_chat.cues.json.
+// Normalizes on demand so the stage tree is always self-consistent.
+async function stageChannel(
+ paths: Paths,
+ ch: ChannelStat,
+ destDir: string,
+ build: ArchiveBuildOptions,
+ log: (msg: string) => void,
+ limit: ReturnType<typeof pLimit>,
+): Promise<{ linkedCount: number; normalizedCount: number; normalizeFailed: number }> {
+ const dataDir = path.join(paths.channelsDir, ch.slug, "data");
+ await rm(destDir, { recursive: true, force: true });
+ await mkdir(destDir, { recursive: true });
+
+ let videoIds: string[];
+ try {
+ videoIds = await readdir(dataDir);
+ } catch {
+ videoIds = [];
+ }
+
+ log(`Archive live chat ${ch.slug}: scanning ${videoIds.length} videos`);
+
+ let normalizedCount = 0;
+ let linkedCount = 0;
+ let normalizeFailed = 0;
+
+ const requiresRewrite = !build.includeMetadata || build.prettyPrint;
+
+ await Promise.all(
+ videoIds.map((id) =>
+ limit(async () => {
+ const videoDir = path.join(dataDir, id);
+ try {
+ const s = await stat(videoDir);
+ if (!s.isDirectory()) return;
+ } catch {
+ return;
+ }
+ let outcome;
+ try {
+ outcome = await normalizeLiveChat({
+ videoDir,
+ channelSlug: ch.slug,
+ configName: ch.config.name,
+ });
+ } catch (err) {
+ normalizeFailed++;
+ log(` ! normalize ${ch.slug}/${id}: ${(err as Error).message}`);
+ return;
+ }
+ if (outcome.status === "skipped") return;
+ if (outcome.status === "wrote") normalizedCount++;
+ const dest = path.join(destDir, id, LIVE_CHAT_CUES_FILENAME);
+ await mkdir(path.dirname(dest), { recursive: true });
+ if (requiresRewrite) {
+ await writeTransformedCues(outcome.cuesPath, dest, build);
+ } else {
+ await link(outcome.cuesPath, dest);
+ }
+ linkedCount++;
+ }),
+ ),
+ );
+
+ if (normalizedCount > 0) {
+ log(` ${ch.slug}: normalized ${normalizedCount} live-chat cues files on demand`);
+ }
+ if (normalizeFailed > 0) {
+ log(` ${ch.slug}: ${normalizeFailed} videos failed to normalize`);
+ }
+
+ if (linkedCount > 0) {
+ await writeFile(
+ path.join(destDir, "channel.json"),
+ JSON.stringify(channelManifest(ch, linkedCount), null, 2) + "\n",
+ );
+ }
+
+ return { linkedCount, normalizedCount, normalizeFailed };
+}
+
+export async function archiveLiveChat(
+ opts: ArchiveLiveChatOptions,
+): Promise<ArchiveLiveChatResult> {
+ const build = resolveBuild(opts.build);
+ const log = opts.onLog ?? ((m: string) => console.log(m));
+ const limit = pLimit(opts.concurrency ?? 8);
+ const archives: ArchiveLiveChatResult["archives"] = [];
+ const skipped: ArchiveLiveChatResult["skipped"] = [];
+ let totalLiveChats = 0;
+
+ log(
+ `Format: ${build.format} level=${build.compressionLevel} metadata=${build.includeMetadata} pretty=${build.prettyPrint}`,
+ );
+
+ await mkdir(archivesDir(opts.paths), { recursive: true });
+ await mkdir(stagingDir(opts.paths), { recursive: true });
+
+ const allChannels = await listChannels(opts.paths);
+ const target = opts.channelSlugs
+ ? allChannels.filter((c) => opts.channelSlugs!.includes(c.slug))
+ : allChannels;
+
+ for (const ch of target) {
+ if (opts.signal?.aborted) {
+ log("Aborted.");
+ break;
+ }
+ const staging = path.join(stagingDir(opts.paths), ch.slug);
+ const archivePath = path.join(
+ archivesDir(opts.paths),
+ `${ch.slug}.${ARCHIVE_INFIX}.${archiveExtension(build.format)}`,
+ );
+ try {
+ const { linkedCount } = await stageChannel(
+ opts.paths,
+ ch,
+ staging,
+ build,
+ log,
+ limit,
+ );
+ if (linkedCount === 0) {
+ await rm(staging, { recursive: true, force: true });
+ skipped.push({ slug: ch.slug, reason: "no normalized live chat" });
+ log(` ${ch.slug}: skipped (no live chat to archive)`);
+ continue;
+ }
+ try {
+ await compressArchive(
+ archivePath,
+ stagingDir(opts.paths),
+ [ch.slug],
+ build,
+ );
+ } finally {
+ await rm(staging, { recursive: true, force: true });
+ }
+ log(
+ ` ${ch.slug}: ${linkedCount} live chats → ${path.basename(archivePath)}`,
+ );
+ archives.push({ slug: ch.slug, archivePath, liveChatCount: linkedCount });
+ totalLiveChats += linkedCount;
+ } catch (err) {
+ await rm(staging, { recursive: true, force: true }).catch(() => {});
+ skipped.push({ slug: ch.slug, reason: (err as Error).message });
+ log(` ! ${ch.slug}: ${(err as Error).message}`);
+ }
+ }
+
+ log("");
+ log(
+ `Wrote ${archives.length} archive${archives.length === 1 ? "" : "s"} containing ${totalLiveChats} live chats to:`,
+ );
+ log(` ${archivesDir(opts.paths)}`);
+ for (const a of archives) {
+ log(` - ${path.basename(a.archivePath)} (${a.liveChatCount} live chats)`);
+ }
+ if (skipped.length > 0) {
+ log("");
+ log(`Skipped ${skipped.length} channel(s):`);
+ for (const s of skipped) log(` - ${s.slug}: ${s.reason}`);
+ }
+
+ return { archives, totalLiveChats, skipped };
+}
+
+export async function archiveCombinedLiveChat(
+ opts: ArchiveLiveChatOptions,
+): Promise<ArchiveCombinedLiveChatResult> {
+ const build = resolveBuild(opts.build);
+ const log = opts.onLog ?? ((m: string) => console.log(m));
+ const limit = pLimit(opts.concurrency ?? 8);
+ const skipped: ArchiveCombinedLiveChatResult["skipped"] = [];
+ const channels: ArchiveCombinedLiveChatResult["channels"] = [];
+ let totalLiveChats = 0;
+
+ log(
+ `Format: ${build.format} level=${build.compressionLevel} metadata=${build.includeMetadata} pretty=${build.prettyPrint}`,
+ );
+
+ await mkdir(archivesDir(opts.paths), { recursive: true });
+ await mkdir(stagingDir(opts.paths), { recursive: true });
+
+ const combinedStaging = path.join(
+ stagingDir(opts.paths),
+ COMBINED_STAGING_DIR,
+ );
+ const archivePath = path.join(
+ archivesDir(opts.paths),
+ `${COMBINED_BASENAME}.${archiveExtension(build.format)}`,
+ );
+
+ await rm(combinedStaging, { recursive: true, force: true });
+ await mkdir(combinedStaging, { recursive: true });
+
+ try {
+ const allChannels = await listChannels(opts.paths);
+ const target = opts.channelSlugs
+ ? allChannels.filter((c) => opts.channelSlugs!.includes(c.slug))
+ : allChannels;
+
+ for (const ch of target) {
+ if (opts.signal?.aborted) {
+ log("Aborted.");
+ break;
+ }
+ const dest = path.join(combinedStaging, ch.slug);
+ try {
+ const { linkedCount } = await stageChannel(
+ opts.paths,
+ ch,
+ dest,
+ build,
+ log,
+ limit,
+ );
+ if (linkedCount === 0) {
+ await rm(dest, { recursive: true, force: true });
+ skipped.push({ slug: ch.slug, reason: "no normalized live chat" });
+ log(` ${ch.slug}: skipped (no live chat to archive)`);
+ continue;
+ }
+ channels.push({ slug: ch.slug, liveChatCount: linkedCount });
+ totalLiveChats += linkedCount;
+ } catch (err) {
+ await rm(dest, { recursive: true, force: true }).catch(() => {});
+ skipped.push({ slug: ch.slug, reason: (err as Error).message });
+ log(` ! ${ch.slug}: ${(err as Error).message}`);
+ }
+ }
+
+ if (channels.length === 0) {
+ log("No channels had normalisable live chat; nothing to archive.");
+ return {
+ archivePath: null,
+ channels,
+ totalLiveChats,
+ skipped,
+ };
+ }
+
+ await writeFile(
+ path.join(combinedStaging, "manifest.json"),
+ JSON.stringify(
+ {
+ version: 1,
+ generatedAt: new Date().toISOString(),
+ channelCount: channels.length,
+ liveChatCount: totalLiveChats,
+ channels,
+ },
+ null,
+ 2,
+ ) + "\n",
+ );
+
+ const tarMembers = ["manifest.json", ...channels.map((c) => c.slug)];
+ log(
+ `Compressing ${tarMembers.length - 1} channels into ${path.basename(archivePath)}…`,
+ );
+ await compressArchive(archivePath, combinedStaging, tarMembers, build);
+ log("");
+ log(
+ `Wrote ${path.basename(archivePath)} containing ${totalLiveChats} live chats across ${channels.length} channels to:`,
+ );
+ log(` ${archivePath}`);
+ if (skipped.length > 0) {
+ log("");
+ log(`Skipped ${skipped.length} channel(s):`);
+ for (const s of skipped) log(` - ${s.slug}: ${s.reason}`);
+ }
+ return {
+ archivePath,
+ channels,
+ totalLiveChats,
+ skipped,
+ };
+ } finally {
+ await rm(combinedStaging, { recursive: true, force: true });
+ }
+}
diff --git a/common/controller/archiveTranscripts.ts b/common/controller/archiveTranscripts.ts
@@ -86,7 +86,7 @@ function stagingDir(paths: Paths): string {
return path.join(paths.transcriptsDir, "export", ".staging");
}
-function resolveBuild(
+export function resolveBuild(
partial: Partial<ArchiveBuildOptions> | undefined,
): ArchiveBuildOptions {
const merged = { ...DEFAULT_ARCHIVE_OPTIONS, ...partial };
@@ -198,7 +198,7 @@ async function stageChannel(
return { linkedCount, normalizedCount, normalizeFailed };
}
-async function writeTransformedCues(
+export async function writeTransformedCues(
sourcePath: string,
destPath: string,
build: ArchiveBuildOptions,
@@ -218,13 +218,13 @@ async function writeTransformedCues(
await writeFile(destPath, body);
}
-function archiveExtension(format: ArchiveFormat): string {
+export function archiveExtension(format: ArchiveFormat): string {
return format;
}
// Compress a single staged top-level entry (a slug dir for per-channel
// mode, or one of multiple entries for combined mode) into archivePath.
-async function compressArchive(
+export async function compressArchive(
archivePath: string,
cwd: string,
members: string[],
diff --git a/common/controller/buildIndex.ts b/common/controller/buildIndex.ts
@@ -34,6 +34,10 @@ import {
isCuesJsonFresh,
readNormalizedTranscript,
} from "./normalizeTranscript";
+import {
+ isLiveChatCuesFresh,
+ readNormalizedLiveChat,
+} from "./normalizeLiveChat";
import type {
TranscriptSummary,
TranscriptDetail,
@@ -557,12 +561,24 @@ export async function buildIndex({
const parsedSubs: StoredSubs = [];
for (const t of s.subTracks) {
try {
- const raw = await readFile(
- path.join(path.dirname(s.metaPath), t.filename),
- "utf8",
- );
- const trackCues =
- t.track === "live_chat" ? parseLiveChat(raw) : parseVtt(raw);
+ let trackCues: Cue[] | null = null;
+ if (t.track === "live_chat") {
+ const liveChatFresh = await isLiveChatCuesFresh(videoFullDir);
+ if (liveChatFresh.fresh) {
+ const normalized = await readNormalizedLiveChat(
+ liveChatFresh.cuesPath,
+ );
+ if (normalized) trackCues = normalized.cues ?? [];
+ }
+ }
+ if (!trackCues) {
+ const raw = await readFile(
+ path.join(path.dirname(s.metaPath), t.filename),
+ "utf8",
+ );
+ trackCues =
+ t.track === "live_chat" ? parseLiveChat(raw) : parseVtt(raw);
+ }
if (trackCues.length > 0) {
parsedSubs.push({ track: t.track, cues: trackCues });
}
diff --git a/common/controller/normalizeAllLiveChat.ts b/common/controller/normalizeAllLiveChat.ts
@@ -0,0 +1,80 @@
+// Walk every channel/video and ensure each transcript.live_chat.json has a
+// sibling live_chat.cues.json. Mirrors normalizeAllTranscripts; cheap on
+// rebuilds because normalizeLiveChat short-circuits when the sidecar is
+// already at least as new as the metadata + raw payload.
+
+import path from "node:path";
+import { readdir } from "node:fs/promises";
+import pLimit from "p-limit";
+import { listChannels } from "./channels";
+import { normalizeLiveChat } from "./normalizeLiveChat";
+import type { Paths } from "../lib/paths";
+
+export type NormalizeAllLiveChatOptions = {
+ paths: Paths;
+ onLog?: (msg: string) => void;
+ signal?: AbortSignal;
+ concurrency?: number;
+};
+
+export type NormalizeAllLiveChatResult = {
+ wrote: number;
+ fresh: number;
+ skipped: number;
+ failed: number;
+};
+
+export async function normalizeAllLiveChat(
+ opts: NormalizeAllLiveChatOptions,
+): Promise<NormalizeAllLiveChatResult> {
+ const log = opts.onLog ?? ((m: string) => console.log(m));
+ const limit = pLimit(opts.concurrency ?? 8);
+ const channels = await listChannels(opts.paths);
+ const result: NormalizeAllLiveChatResult = {
+ wrote: 0,
+ fresh: 0,
+ skipped: 0,
+ failed: 0,
+ };
+ for (const ch of channels) {
+ if (opts.signal?.aborted) break;
+ const dataDir = path.join(opts.paths.channelsDir, ch.slug, "data");
+ const videoIds = await readdir(dataDir).catch(() => [] as string[]);
+ log(`Normalize live chat ${ch.slug}: ${videoIds.length} videos`);
+ let wrote = 0;
+ let fresh = 0;
+ let skipped = 0;
+ let failed = 0;
+ await Promise.all(
+ videoIds.map((id) =>
+ limit(async () => {
+ if (opts.signal?.aborted) return;
+ try {
+ const outcome = await normalizeLiveChat({
+ videoDir: path.join(dataDir, id),
+ channelSlug: ch.slug,
+ configName: ch.config.name,
+ });
+ if (outcome.status === "wrote") wrote++;
+ else if (outcome.status === "fresh") fresh++;
+ else skipped++;
+ } catch (err) {
+ failed++;
+ log(` ! ${ch.slug}/${id}: ${(err as Error).message}`);
+ }
+ }),
+ ),
+ );
+ log(
+ ` ${ch.slug}: wrote=${wrote} fresh=${fresh} skipped=${skipped} failed=${failed}`,
+ );
+ result.wrote += wrote;
+ result.fresh += fresh;
+ result.skipped += skipped;
+ result.failed += failed;
+ }
+ log(
+ `Done. wrote=${result.wrote} fresh=${result.fresh} skipped=${result.skipped} failed=${result.failed}`,
+ );
+ return result;
+}
diff --git a/common/controller/normalizeLiveChat.ts b/common/controller/normalizeLiveChat.ts
@@ -0,0 +1,141 @@
+// Produce a compact, self-describing parsed copy of transcript.live_chat.json
+// alongside the raw file so that downstream consumers (buildIndex, archives)
+// can skip re-parsing potentially huge live-chat JSON-lines payloads. Shape
+// matches NormalizedTranscript with source: "live_chat".
+
+import path from "node:path";
+import { readFile, rename, stat, writeFile } from "node:fs/promises";
+import { parseLiveChat } from "../lib/liveChat";
+import type { Cue } from "../lib/vtt";
+import { summarize, type RawMetadata } from "../lib/transcripts-server";
+import {
+ type NormalizedLiveChat,
+} from "./normalizeTranscript";
+import {
+ LIVE_CHAT_CUES_FILENAME,
+ LIVE_CHAT_FILENAME,
+ META_FILENAME,
+} from "../lib/videoStatus";
+
+export const LIVE_CHAT_FILE_VERSION = 1;
+
+export type NormalizeLiveChatOptions = {
+ videoDir: string;
+ channelSlug: string;
+ configName?: string;
+ log?: (msg: string) => void;
+ force?: boolean;
+};
+
+export type NormalizeLiveChatOutcome =
+ | { status: "wrote"; cuesPath: string }
+ | { status: "fresh"; cuesPath: string }
+ | { status: "skipped"; reason: "no-live-chat" | "no-metadata" };
+
+async function mtimeMs(p: string): Promise<number | null> {
+ try {
+ return (await stat(p)).mtimeMs;
+ } catch {
+ return null;
+ }
+}
+
+// TODO: live_chat files can be hundreds of MB. parseLiveChat already splits
+// on "\n", so a streaming readline variant is straightforward if we hit a
+// memory wall. For now this matches the readFile pattern used by the
+// transcript normalizer.
+export async function normalizeLiveChat(
+ opts: NormalizeLiveChatOptions,
+): Promise<NormalizeLiveChatOutcome> {
+ const metaPath = path.join(opts.videoDir, META_FILENAME);
+ const rawPath = path.join(opts.videoDir, LIVE_CHAT_FILENAME);
+ const cuesPath = path.join(opts.videoDir, LIVE_CHAT_CUES_FILENAME);
+
+ const [metaStatMs, rawStatMs, cuesMs] = await Promise.all([
+ mtimeMs(metaPath),
+ mtimeMs(rawPath),
+ mtimeMs(cuesPath),
+ ]);
+
+ if (metaStatMs === null) return { status: "skipped", reason: "no-metadata" };
+ if (rawStatMs === null) return { status: "skipped", reason: "no-live-chat" };
+
+ if (
+ !opts.force &&
+ cuesMs !== null &&
+ cuesMs >= metaStatMs &&
+ cuesMs >= rawStatMs
+ ) {
+ return { status: "fresh", cuesPath };
+ }
+
+ const metaRaw = await readFile(metaPath, "utf8");
+ const parsedMeta = JSON.parse(metaRaw) as RawMetadata;
+ const summary = summarize(
+ opts.channelSlug,
+ path.basename(opts.videoDir),
+ parsedMeta,
+ opts.configName,
+ );
+
+ const rawLiveChat = await readFile(rawPath, "utf8");
+ let cues: Cue[];
+ try {
+ cues = parseLiveChat(rawLiveChat);
+ } catch (err) {
+ throw new Error(
+ `Failed to parse ${LIVE_CHAT_FILENAME} for ${opts.channelSlug}/${path.basename(opts.videoDir)}: ${(err as Error).message}`,
+ );
+ }
+
+ const out: NormalizedLiveChat = {
+ version: LIVE_CHAT_FILE_VERSION,
+ source: "live_chat",
+ ...summary,
+ cues,
+ };
+
+ const tmp = `${cuesPath}.tmp-${process.pid}`;
+ await writeFile(tmp, JSON.stringify(out));
+ await rename(tmp, cuesPath);
+ opts.log?.(
+ `Normalized live chat ${opts.channelSlug}/${path.basename(opts.videoDir)} (${cues.length} cues)`,
+ );
+ return { status: "wrote", cuesPath };
+}
+
+export async function readNormalizedLiveChat(
+ cuesPath: string,
+): Promise<NormalizedLiveChat | null> {
+ try {
+ const raw = await readFile(cuesPath, "utf8");
+ const parsed = JSON.parse(raw) as NormalizedLiveChat;
+ if (typeof parsed.version !== "number") return null;
+ if (parsed.source !== "live_chat") return null;
+ return parsed;
+ } catch {
+ return null;
+ }
+}
+
+// Sidecar is fresh iff it exists and its mtime is >= both metadata.info.json
+// and transcript.live_chat.json. Used by buildIndex to decide whether to
+// trust the cached cues without re-parsing the raw payload.
+export async function isLiveChatCuesFresh(
+ videoDir: string,
+): Promise<{ fresh: boolean; cuesPath: string }> {
+ const cuesPath = path.join(videoDir, LIVE_CHAT_CUES_FILENAME);
+ const metaPath = path.join(videoDir, META_FILENAME);
+ const rawPath = path.join(videoDir, LIVE_CHAT_FILENAME);
+ const [cuesMs, metaMs, rawMs] = await Promise.all([
+ mtimeMs(cuesPath),
+ mtimeMs(metaPath),
+ mtimeMs(rawPath),
+ ]);
+ if (cuesMs === null || metaMs === null || rawMs === null) {
+ return { fresh: false, cuesPath };
+ }
+ if (cuesMs < metaMs) return { fresh: false, cuesPath };
+ if (cuesMs < rawMs) return { fresh: false, cuesPath };
+ return { fresh: true, cuesPath };
+}
diff --git a/common/controller/normalizeTranscript.ts b/common/controller/normalizeTranscript.ts
@@ -23,9 +23,11 @@ export const CUES_FILE_VERSION = 1;
export type NormalizedTranscript = TranscriptDetail & {
version: number;
- source: IndexTranscript["kind"];
+ source: IndexTranscript["kind"] | "live_chat";
};
+export type NormalizedLiveChat = NormalizedTranscript & { source: "live_chat" };
+
export type NormalizeOptions = {
videoDir: string;
channelSlug: string;
diff --git a/common/lib/videoStatus.ts b/common/lib/videoStatus.ts
@@ -21,6 +21,8 @@ export type IndexTranscript =
export const VTT_FILENAME = "transcript.en.vtt";
export const WHISPER_FILENAME = "transcript.json";
export const CUES_JSON_FILENAME = "transcript.cues.json";
+export const LIVE_CHAT_FILENAME = "transcript.live_chat.json";
+export const LIVE_CHAT_CUES_FILENAME = "live_chat.cues.json";
export const META_FILENAME = "metadata.info.json";
export type SubTrack = {
@@ -109,6 +111,7 @@ export async function readSubTracks(videoDir: string): Promise<SubTrack[]> {
for (const entry of entries) {
if (entry === VTT_FILENAME || entry === WHISPER_FILENAME) continue;
if (entry === CUES_JSON_FILENAME) continue;
+ if (entry === LIVE_CHAT_CUES_FILENAME) continue;
const m = entry.match(SUB_FILE_RE);
if (!m) continue;
const ext = m[2].toLowerCase();
diff --git a/editor/app/build/buildAction.ts b/editor/app/build/buildAction.ts
@@ -3,11 +3,16 @@
import { revalidatePath } from "next/cache";
import { buildIndex } from "yt-dlp-transcript-common/controller/buildIndex";
import { normalizeAllTranscripts } from "yt-dlp-transcript-common/controller/normalizeAll";
+import { normalizeAllLiveChat } from "yt-dlp-transcript-common/controller/normalizeAllLiveChat";
import {
archiveTranscripts,
archiveCombinedTranscripts,
type ArchiveBuildOptions,
} from "yt-dlp-transcript-common/controller/archiveTranscripts";
+import {
+ archiveLiveChat,
+ archiveCombinedLiveChat,
+} from "yt-dlp-transcript-common/controller/archiveLiveChat";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import {
runManagedCommand,
@@ -96,3 +101,47 @@ export async function archiveCombinedTranscriptsAction(
},
});
}
+
+export async function normalizeLiveChatAction(
+ queueKey?: string,
+): Promise<StreamActionResult> {
+ const paths = getPaths();
+ return runManagedFunction({
+ kind: "normalize-live-chat",
+ queueKey: queueKey === undefined ? DEFAULT_BUILD_QUEUE : queueKey.trim(),
+ paths,
+ fn: async (onLog, signal) => {
+ await normalizeAllLiveChat({ paths, onLog, signal });
+ },
+ });
+}
+
+export async function archiveLiveChatAction(
+ build: Partial<ArchiveBuildOptions>,
+ queueKey?: string,
+): Promise<StreamActionResult> {
+ const paths = getPaths();
+ return runManagedFunction({
+ kind: "archive-live-chat",
+ queueKey: queueKey === undefined ? DEFAULT_BUILD_QUEUE : queueKey.trim(),
+ paths,
+ fn: async (onLog, signal) => {
+ await archiveLiveChat({ paths, onLog, signal, build });
+ },
+ });
+}
+
+export async function archiveCombinedLiveChatAction(
+ build: Partial<ArchiveBuildOptions>,
+ queueKey?: string,
+): Promise<StreamActionResult> {
+ const paths = getPaths();
+ return runManagedFunction({
+ kind: "archive-combined-live-chat",
+ queueKey: queueKey === undefined ? DEFAULT_BUILD_QUEUE : queueKey.trim(),
+ paths,
+ fn: async (onLog, signal) => {
+ await archiveCombinedLiveChat({ paths, onLog, signal, build });
+ },
+ });
+}
diff --git a/editor/app/build/components/BuildButtons.tsx b/editor/app/build/components/BuildButtons.tsx
@@ -9,10 +9,13 @@ import {
import { QueueControl } from "../../components/QueueControl";
import { cancelJobAction } from "../../jobs/actions";
import {
+ archiveCombinedLiveChatAction,
archiveCombinedTranscriptsAction,
+ archiveLiveChatAction,
archiveTranscriptsAction,
buildExportAction,
buildIndexAction,
+ normalizeLiveChatAction,
normalizeTranscriptsAction,
} from "../buildAction";
import { ArchiveOptionsControl } from "./ArchiveOptionsControl";
@@ -29,12 +32,25 @@ export function BuildButtons({ existingQueues }: Props) {
const [normalizeQueue, setNormalizeQueue] = useState(DEFAULT_BUILD_QUEUE);
const [archiveQueue, setArchiveQueue] = useState(DEFAULT_BUILD_QUEUE);
const [combinedQueue, setCombinedQueue] = useState(DEFAULT_BUILD_QUEUE);
+ const [liveChatNormalizeQueue, setLiveChatNormalizeQueue] = useState(
+ DEFAULT_BUILD_QUEUE,
+ );
+ const [liveChatArchiveQueue, setLiveChatArchiveQueue] = useState(
+ DEFAULT_BUILD_QUEUE,
+ );
+ const [liveChatCombinedQueue, setLiveChatCombinedQueue] = useState(
+ DEFAULT_BUILD_QUEUE,
+ );
const [archiveOptions, setArchiveOptions] = useState<ArchiveBuildOptions>(
DEFAULT_ARCHIVE_OPTIONS,
);
const [combinedOptions, setCombinedOptions] = useState<ArchiveBuildOptions>(
DEFAULT_ARCHIVE_OPTIONS,
);
+ const [liveChatArchiveOptions, setLiveChatArchiveOptions] =
+ useState<ArchiveBuildOptions>(DEFAULT_ARCHIVE_OPTIONS);
+ const [liveChatCombinedOptions, setLiveChatCombinedOptions] =
+ useState<ArchiveBuildOptions>(DEFAULT_ARCHIVE_OPTIONS);
return (
<div className="flex flex-col gap-8">
@@ -190,6 +206,113 @@ export function BuildButtons({ existingQueues }: Props) {
}
/>
</section>
+
+ <section className="flex flex-col gap-3 border-t border-zinc-200 dark:border-zinc-800 pt-6">
+ <div>
+ <h2 className="text-lg font-semibold">Normalize live chat</h2>
+ <p className="text-sm text-zinc-500">
+ Walks every video and writes{" "}
+ <code>live_chat.cues.json</code> next to each raw{" "}
+ <code>transcript.live_chat.json</code>. Idempotent — skips
+ entries whose cues file is already up to date.
+ </p>
+ </div>
+ <StreamActionLog
+ trigger={() => normalizeLiveChatAction(liveChatNormalizeQueue)}
+ cancelAction={cancelJobAction}
+ buttonLabel="Normalize live chat"
+ runningLabel="Normalizing live chat…"
+ extraControls={
+ <QueueControl
+ value={liveChatNormalizeQueue}
+ onChange={setLiveChatNormalizeQueue}
+ defaultQueueKey={DEFAULT_BUILD_QUEUE}
+ existingQueues={existingQueues}
+ actionLabel="Normalize live chat"
+ />
+ }
+ />
+ </section>
+
+ <section className="flex flex-col gap-3 border-t border-zinc-200 dark:border-zinc-800 pt-6">
+ <div>
+ <h2 className="text-lg font-semibold">Build live chat archives</h2>
+ <p className="text-sm text-zinc-500">
+ Produces one <code><channel>.live_chat.tar.gz</code> per
+ channel under <code>transcripts/export/archives/</code>,
+ containing each video's parsed{" "}
+ <code>live_chat.cues.json</code> plus a{" "}
+ <code>channel.json</code> manifest. Normalizes missing live
+ chats on demand.
+ </p>
+ </div>
+ <StreamActionLog
+ trigger={() =>
+ archiveLiveChatAction(liveChatArchiveOptions, liveChatArchiveQueue)
+ }
+ cancelAction={cancelJobAction}
+ buttonLabel="Build live chat archives"
+ runningLabel="Building live chat archives…"
+ extraControls={
+ <div className="flex flex-wrap items-center gap-3">
+ <ArchiveOptionsControl
+ value={liveChatArchiveOptions}
+ onChange={setLiveChatArchiveOptions}
+ actionLabel="Build live chat archives"
+ />
+ <QueueControl
+ value={liveChatArchiveQueue}
+ onChange={setLiveChatArchiveQueue}
+ defaultQueueKey={DEFAULT_BUILD_QUEUE}
+ existingQueues={existingQueues}
+ actionLabel="Build live chat archives"
+ />
+ </div>
+ }
+ />
+ </section>
+
+ <section className="flex flex-col gap-3 border-t border-zinc-200 dark:border-zinc-800 pt-6">
+ <div>
+ <h2 className="text-lg font-semibold">
+ Build combined live chat archive
+ </h2>
+ <p className="text-sm text-zinc-500">
+ Produces a single <code>all-live-chat.tar.gz</code> under{" "}
+ <code>transcripts/export/archives/</code> containing every
+ channel's parsed live chats plus a top-level{" "}
+ <code>manifest.json</code> index. Normalizes missing live chats
+ on demand.
+ </p>
+ </div>
+ <StreamActionLog
+ trigger={() =>
+ archiveCombinedLiveChatAction(
+ liveChatCombinedOptions,
+ liveChatCombinedQueue,
+ )
+ }
+ cancelAction={cancelJobAction}
+ buttonLabel="Build combined live chat archive"
+ runningLabel="Building combined live chat archive…"
+ extraControls={
+ <div className="flex flex-wrap items-center gap-3">
+ <ArchiveOptionsControl
+ value={liveChatCombinedOptions}
+ onChange={setLiveChatCombinedOptions}
+ actionLabel="Build combined live chat archive"
+ />
+ <QueueControl
+ value={liveChatCombinedQueue}
+ onChange={setLiveChatCombinedQueue}
+ defaultQueueKey={DEFAULT_BUILD_QUEUE}
+ existingQueues={existingQueues}
+ actionLabel="Build combined live chat archive"
+ />
+ </div>
+ }
+ />
+ </section>
</div>
);
}
