// Importing browser-saved forum thread pages into a forum-thread channel, end // to end over a temp corpus: the pages are parsed, posts merged (new ones // appended, an edited one updated), and pages that are not this thread's are // skipped by name. SYNTHETIC pages only. // // Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test controller/importForumPages.test.ts import { test } from "node:test"; import assert from "node:assert/strict"; import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; const ROOT = await mkdtemp(path.join(os.tmpdir(), "forum-import-")); process.env.TRANSCRIPTS_DIR = path.join(ROOT, "transcripts"); const { getPaths } = await import("../lib/paths"); const { importForumPages } = await import("./importForumPages"); const { readAllPosts } = await import("../lib/posts-server"); const { CHALLENGE_PAGE, THREAD_URL, threadPage } = await import("../social/__fixtures__/xenforoPages"); const T0 = 1_760_000_000; const post = (id: number, position: number, body: string, editedTs?: number) => ({ id, author: `Member${id % 3}`, userId: 10 + (id % 3), ts: T0 + position * 60, position, body, ...(editedTs ? { editedTs } : {}), }); async function channel(slug: string, config: Record) { const dir = path.join(getPaths().channelsDir, slug); await mkdir(dir, { recursive: true }); await writeFile(path.join(dir, "config.json"), JSON.stringify(config)); } await channel("teapots", { handling: "youtube", sourceKind: "social", platform: "xenforo", postFetcher: "xenforo-thread", url: THREAD_URL, socialHandle: "the-teapot-collectors-thread.4242", }); await channel("someone-on-bluesky", { handling: "youtube", sourceKind: "social", platform: "bluesky", url: "https://bsky.app/profile/someone.example", }); const SAVES = path.join(ROOT, "saves"); await mkdir(path.join(SAVES, "The Teapot Collectors Thread_files"), { recursive: true }); await writeFile( path.join(SAVES, "The Teapot Collectors Thread _ Page 2.html"), threadPage({ page: 2, last: 2, saved: true, posts: [post(21, 4, "Fourth."), post(22, 5, "Fifth.")] }), ); await writeFile( path.join(SAVES, "The Teapot Collectors Thread.html"), threadPage({ page: 1, last: 2, saved: true, posts: [post(11, 1, "First."), post(12, 2, "Second."), post(13, 3, "Third.")] }), ); await writeFile(path.join(SAVES, "check.html"), CHALLENGE_PAGE); await writeFile( path.join(SAVES, "other-thread.html"), threadPage({ page: 1, last: 1, posts: [post(99, 1, "Elsewhere.")] }).replace(/4242/g, "5151"), ); await writeFile(path.join(SAVES, "The Teapot Collectors Thread_files", "x.jpg"), "not a page"); test("imports a folder of saved pages: posts merged, strangers skipped by name", async () => { const log: string[] = []; const res = await importForumPages({ paths: getPaths(), slug: "teapots", inputs: [SAVES], onLog: (l) => log.push(l) }); assert.equal(res.ok, true, res.error); assert.equal(res.parsed, 5); assert.equal(res.written, 5); const byFile = Object.fromEntries(res.pages.map((p) => [p.file, p])); assert.equal(byFile["check.html"].skipped?.startsWith("not a thread page (challenge"), true); assert.match(byFile["other-thread.html"].skipped ?? "", /another thread \(5151, not 4242\)/); assert.equal(byFile["The Teapot Collectors Thread _ Page 2.html"].page, 2); assert.equal(res.pages.length, 4, "the _files folder is not walked"); const posts = await readAllPosts(path.join(getPaths().channelsDir, "teapots")); assert.deepEqual(posts.map((p) => p.id), ["22", "21", "13", "12", "11"]); assert.equal(posts[0].forum?.page, 2); assert.equal(posts[0].channelSlug, "teapots"); }); test("a later save of an edited post updates it; the same save again changes nothing", async () => { const file = path.join(ROOT, "page-2-again.html"); await writeFile( file, threadPage({ page: 2, last: 2, saved: true, posts: [post(21, 4, "Fourth, edited.", T0 + 5000), post(22, 5, "Fifth.")] }), ); const res = await importForumPages({ paths: getPaths(), slug: "teapots", inputs: [file] }); assert.deepEqual([res.written, res.updated, res.unchanged], [0, 1, 1]); const posts = await readAllPosts(path.join(getPaths().channelsDir, "teapots")); assert.equal(posts.find((p) => p.id === "21")?.text, "Fourth, edited."); assert.equal(posts.length, 5); const again = await importForumPages({ paths: getPaths(), slug: "teapots", inputs: [file] }); assert.deepEqual([again.written, again.updated, again.unchanged], [0, 0, 2]); }); test("a dry run writes nothing; refusals name the problem", async () => { const dry = await importForumPages({ paths: getPaths(), slug: "teapots", inputs: [SAVES], dryRun: true }); assert.equal(dry.ok, true); assert.equal(dry.written, 0); const notForum = await importForumPages({ paths: getPaths(), slug: "someone-on-bluesky", inputs: [SAVES] }); assert.match(notForum.error ?? "", /not a forum-thread channel/); const missing = await importForumPages({ paths: getPaths(), slug: "teapots", inputs: [path.join(ROOT, "nope")] }); assert.match(missing.error ?? "", /No such file or directory/); const noChannel = await importForumPages({ paths: getPaths(), slug: "nobody", inputs: [SAVES] }); assert.match(noChannel.error ?? "", /No such channel/); });