// Import forum thread pages the operator SAVED from their own browser ("Save // page as", complete or HTML only) into a forum-thread channel. The second way // in beside the live fetcher, through the same parser (xenforoParse.ts) and // the same store (posts-server.ts): new posts are appended, a post already // archived is updated when the save holds a newer record of it (an edit), and // one that arrives unchanged is left alone. // // Server-only (node:fs). Nothing here touches the network: a saved page's // assets are read from the page as it is, never fetched. import { sameXenforoForum } from "../lib/platform"; import { readdir, readFile, stat } from "node:fs/promises"; import path from "node:path"; import { readChannelConfig } from "./channels"; import { isSocialChannel } from "../lib/channelConfig"; import type { Paths } from "../lib/paths"; import type { Post } from "../lib/posts"; import { upsertPosts } from "../lib/posts-server"; import { classifyForumPage, parseXenforoThreadPage, parseXenforoThreadUrl, } from "../social/xenforoParse"; export type ImportForumPagesOptions = { paths: Paths; slug: string; // Files and directories. A directory contributes the .html/.htm files in // it (not the `_files/` asset folders a browser saves beside them). inputs: ReadonlyArray; // Parse and count, write nothing. dryRun?: boolean; onLog?: (line: string) => void; }; export type ImportedPage = { file: string; page?: number; posts: number; // Why the file contributed nothing, when it did not. skipped?: string; }; export type ImportForumPagesResult = { ok: boolean; error?: string; pages: ImportedPage[]; parsed: number; written: number; updated: number; unchanged: number; }; const PAGE_EXT = /\.(html?|xhtml)$/i; // The page files the inputs name, sorted, each once. export async function listSavedPages(inputs: ReadonlyArray): Promise { const out = new Set(); for (const input of inputs) { const abs = path.resolve(input); const st = await stat(abs).catch(() => null); if (!st) throw new Error(`No such file or directory: ${input}`); if (st.isDirectory()) { for (const e of await readdir(abs, { withFileTypes: true })) { if (e.isFile() && PAGE_EXT.test(e.name)) out.add(path.join(abs, e.name)); } } else { out.add(abs); } } return [...out].sort(); } export async function importForumPages(opts: ImportForumPagesOptions): Promise { const log = (line: string) => opts.onLog?.(line); const fail = (error: string): ImportForumPagesResult => ({ ok: false, error, pages: [], parsed: 0, written: 0, updated: 0, unchanged: 0, }); const config = await readChannelConfig(opts.paths, opts.slug); if (!config) return fail(`No such channel: ${opts.slug}`); if (!isSocialChannel(config) || config.platform !== "xenforo") { return fail(`${opts.slug} is not a forum-thread channel (platform "xenforo").`); } const thread = parseXenforoThreadUrl(config.url ?? ""); if (!thread) return fail(`${opts.slug}'s URL is not a XenForo thread URL.`); let files: string[]; try { files = await listSavedPages(opts.inputs); } catch (err) { return fail((err as Error).message); } if (files.length === 0) return fail("No saved pages (.html) found in what was given."); const pages: ImportedPage[] = []; const posts: Post[] = []; for (const file of files) { const name = path.basename(file); let html: string; try { html = await readFile(file, "utf8"); } catch (err) { pages.push({ file: name, posts: 0, skipped: `unreadable: ${(err as Error).message}` }); continue; } const block = classifyForumPage(html); if (block) { pages.push({ file: name, posts: 0, skipped: `not a thread page (${block.kind}: ${block.detail})` }); continue; } const parsed = parseXenforoThreadPage(html, { channelSlug: opts.slug, pageUrl: thread.base }); if (parsed.threadId && parsed.threadId !== thread.threadId) { pages.push({ file: name, page: parsed.page, posts: 0, skipped: `a page of another thread (${parsed.threadId}, not ${thread.threadId})`, }); continue; } if (parsed.host && !sameXenforoForum(parsed.host, thread.host)) { pages.push({ file: name, page: parsed.page, posts: 0, skipped: `a page of another forum (${parsed.host})` }); continue; } pages.push({ file: name, page: parsed.page, posts: parsed.posts.length }); posts.push(...parsed.posts); log(`${name}: page ${parsed.page}/${parsed.lastPage}, ${parsed.posts.length} post(s).`); } if (opts.dryRun) { log(`Dry run: ${posts.length} post(s) parsed from ${files.length} file(s); nothing written.`); return { ok: true, pages, parsed: posts.length, written: 0, updated: 0, unchanged: 0 }; } const channelRoot = path.join(opts.paths.channelsDir, opts.slug); const res = await upsertPosts(channelRoot, posts); log( `Imported ${posts.length} post(s) from ${files.length} file(s): ${res.written} new, ` + `${res.updated} updated, ${res.unchanged} unchanged.`, ); return { ok: true, pages, parsed: posts.length, written: res.written, updated: res.updated, unchanged: res.unchanged, }; }