// The converters' disk half: what a channels tree can tell a converter, as the // callbacks the converters take — a record's cues, the archive channel that // keeps a post, a post's record. The CLI is bin/reports-convert.ts. // // Reads only TEXT: `channels//data//transcript.cues.json` (behind // the text guard, lib/channelMedia.ts — a legacy channel or one mid-migration // is "no cues", with a warning, never a read of the wrong tree), and each // channel's `posts-archive` and `posts/*.jsonl`. Writes nothing. // // SERVER-ONLY (node:fs). import { readdir, readFile } from "node:fs/promises"; import path from "node:path"; import { assertChannelTextReadable } from "../channelMedia"; import { isSafeChannelSegment, isSafeIdSegment } from "../citations/moments"; import { readAllPosts, readSeenPostIds } from "../posts-server"; import type { Post } from "../posts"; import type { Cue, CuesOf, PostChannelOf } from "./convertShared"; import type { PostOf, PostRecord } from "./convertManifest"; function isCue(v: unknown): v is Cue { if (!v || typeof v !== "object") return false; const c = v as Record; return typeof c.start === "number" && typeof c.end === "number" && typeof c.text === "string" && c.end >= c.start; } // A record's cues off the channels tree, cached per record. A channel whose // text is not readable is one warning and no cues. export function diskCuesOf(channelsDir: string, warnings: string[] = []): CuesOf { const guarded = new Map>(); const cache = new Map>(); const readable = (slug: string) => { let p = guarded.get(slug); if (!p) { p = assertChannelTextReadable({ channelsDir }, slug).then( () => true, (err: Error) => { warnings.push(`${slug}: ${err.message} — no cues read from it`); return false; }, ); guarded.set(slug, p); } return p; }; return (channel, id) => { if (!isSafeChannelSegment(channel) || !isSafeIdSegment(id)) return Promise.resolve(null); const key = `${channel}/${id}`; let p = cache.get(key); if (!p) { p = (async () => { if (!(await readable(channel))) return null; let doc: unknown; try { doc = JSON.parse(await readFile(path.join(channelsDir, channel, "data", id, "transcript.cues.json"), "utf8")); } catch { return null; } const cues = (doc as { cues?: unknown })?.cues; if (!Array.isArray(cues)) return null; return cues.filter(isCue).sort((a, b) => a.start - b.start); })(); cache.set(key, p); } return p; }; } // The channel directories under the tree, by name. async function channelSlugs(channelsDir: string): Promise { const entries = await readdir(channelsDir, { withFileTypes: true }).catch(() => []); return entries.filter((e) => e.isDirectory() && isSafeChannelSegment(e.name)).map((e) => e.name).sort(); } const squash = (v: string) => v.toLowerCase().replace(/^@/, "").replace(/[^a-z0-9]+/g, ""); // The archive channel that keeps a post: every channel's posts archive, read // once; among several, the one whose slug carries the post's handle. export function diskPostChannelOf(channelsDir: string): PostChannelOf { let index: Promise> | null = null; const build = async () => { const byId = new Map(); for (const slug of await channelSlugs(channelsDir)) { for (const id of await readSeenPostIds(path.join(channelsDir, slug))) { byId.set(id, [...(byId.get(id) ?? []), slug]); } } return byId; }; return async ({ id, handle }) => { index ??= build(); const slugs = (await index).get(id) ?? []; if (slugs.length <= 1) return slugs[0] ?? null; const h = handle ? squash(handle.split(".")[0]) : ""; return (h && slugs.find((s) => squash(s).includes(h))) || slugs[0]; }; } // A post's record off its channel's archive, each channel read once. export function diskPostOf(channelsDir: string): PostOf { const channels = new Map>>(); return async (channel, id) => { if (!isSafeChannelSegment(channel)) return null; let p = channels.get(channel); if (!p) { p = readAllPosts(path.join(channelsDir, channel)).then( (posts) => new Map(posts.map((post) => [post.id, post])), () => new Map(), ); channels.set(channel, p); } const post = (await p).get(id); if (!post) return null; const rec: PostRecord = { platform: post.platform === "bluesky" ? "bluesky" : post.platform === "xenforo" ? "forum" : "x", url: post.url, createdAt: post.createdAt, author: post.author, }; if (post.authorName) rec.authorName = post.authorName; return rec; }; }