// Capture specific archived posts of a social channel: a screenshot of each // post as its platform renders it, and its attached media, into // `channels//posts-media//` (social/postCapture.ts owns the layout). // // The fetcher does the capturing (`SocialFetcher.captureByIds`); this lands // what it learned about each post's liveness in the channel's availability // sidecar, `posts-availability.json`, through the same merge the deleted-post // sweep uses — a deleted or walled post is recorded there, history appended // only on a change, and nothing is recorded from ignorance (a login wall says // nothing about the post). // // Only ids already in the channel's posts archive are captured: a capture is // of the archive, and an id from somewhere else is refused by name. Each id's // archived text and links go to the fetcher with it, so a post that is an X // Article's link is known as one before its page is opened. import path from "node:path"; import { readChannelConfig } from "./channels"; import type { Paths } from "../lib/paths"; import type { PostAvailability } from "../lib/posts"; import { isSocialChannel } from "../lib/channelConfig"; import { alwaysCookies, resolveCookiePolicy, type CookiePolicyInputs, } from "../lib/cookiePolicy"; import { mergePostAvailability, readAllPosts, readPostAvailability, readSeenPostIds, writePostAvailability, } from "../lib/posts-server"; import { handleFromAccountUrl, resolveSocialFetcher, type PostCaptureOutcome, type SocialFetcher, } from "../social/fetchers"; import { postsMediaDir } from "../social/postCapture"; import "../social/blueskyFetcher"; import "../social/xGalleryDlFetcher"; import "../social/xPlaywrightFetcher"; import "../social/xNitterFetcher"; import "../social/xenforoFetcher"; import { resolveXCookieSourceFor, type XLoginSettings, } from "../social/xBrowserLogin"; export type CapturePostsOptions = { paths: Paths; slug: string; settings: CookiePolicyInputs & XLoginSettings; ids: ReadonlyArray; // Take the screenshot / download the media. Both default to true. shots?: boolean; media?: boolean; // Capture again what is already captured. force?: boolean; // Also open and save the X Article a post links to. Defaults to true. articles?: boolean; onLog?: (line: string) => void; signal?: AbortSignal; // The job's drain: stop between posts. drain?: AbortSignal; }; export type CapturePostsResult = { ok: boolean; outcomes: PostCaptureOutcome[]; needsCookies?: boolean; error?: string; }; // Why this fetcher cannot capture posts, or null when it can. Shared with the // server action, so the refusal is one sentence everywhere. export function capturePostsProblem( fetcher: Pick | undefined, ): string | null { if (!fetcher) return "This channel has no post fetcher."; return typeof fetcher.captureByIds === "function" ? null : `${fetcher.label} cannot capture posts.`; } export const NOTHING_TO_CAPTURE = 'Nothing to capture: both the screenshot and the media are turned off (to read only the articles, ask for "articles": true).'; // Both halves off is nothing to do — unless the articles were asked for by // name: their default alone does not make a run. export function nothingToCapture(o: { shots?: boolean; media?: boolean; articles?: boolean; }): boolean { return o.shots === false && o.media === false && o.articles !== true; } // The ids not in the channel's archive, or null when every one is. export function strayCaptureIds( ids: ReadonlyArray, archived: ReadonlySet, ): string[] | null { const stray = ids.filter((id) => !archived.has(id)); return stray.length ? stray : null; } export function strayIdsRefusal(slug: string, stray: string[]): string { return `${stray.length} id(s) not in ${slug}'s posts archive: ${stray.join(", ")}`; } export async function capturePosts( opts: CapturePostsOptions, ): Promise { const { paths, slug, settings, onLog } = opts; const log = (line: string) => onLog?.(line); const fail = (error: string): CapturePostsResult => ({ ok: false, outcomes: [], error }); const channelRoot = path.join(paths.channelsDir, slug); const ids = [...new Set(opts.ids)]; if (nothingToCapture(opts)) return fail(NOTHING_TO_CAPTURE); if (ids.length === 0) return fail("No post ids to capture."); const config = await readChannelConfig(paths, slug); if (!config) return fail(`No such channel: ${slug}`); if (!isSocialChannel(config)) return fail(`${slug} is not a social channel.`); const accountUrl = config.url ?? ""; const fetcher = resolveSocialFetcher(config.postFetcher, accountUrl); const problem = capturePostsProblem(fetcher); if (problem || !fetcher?.captureByIds) return fail(problem ?? "This channel has no post fetcher."); const handle = config.socialHandle ?? handleFromAccountUrl(accountUrl) ?? ""; if (!handle) return fail(`Could not determine an account handle for ${slug}`); const stray = strayCaptureIds(ids, await readSeenPostIds(channelRoot)); if (stray) return fail(strayIdsRefusal(slug, stray)); // The archived record of each id: its text and links for the article links // in it (X), its URL and media for a forum post (the capture opens the one // and downloads the other). type Archived = { text: string; links: string[]; url?: string; media?: { kind: string; url: string; name?: string }[]; }; let archived: Map | undefined; const forum = fetcher.platform === "xenforo"; if ((opts.articles !== false && fetcher.platform === "twitter") || forum) { const wanted = new Set(ids); archived = new Map(); for (const p of await readAllPosts(channelRoot)) { if (!wanted.has(p.id)) continue; archived.set(p.id, { text: p.text, links: p.links ?? [], ...(forum ? { url: p.url, ...(p.media ? { media: p.media } : {}) } : {}), }); } } const policy = resolveCookiePolicy(settings, config); const xLogin = fetcher.platform === "twitter" ? await resolveXCookieSourceFor(paths, settings, policy.cookies) : undefined; log( `Capturing ${ids.length} post(s) of ${slug} via ${fetcher.label} (@${handle}):` + [ opts.shots === false ? "" : " screenshot", opts.media === false ? "" : " media", opts.articles === false || fetcher.platform !== "twitter" ? "" : " articles", ].join("") + (opts.force ? ", again where already captured" : "") + ".", ); const controller = new AbortController(); let result; try { result = await fetcher.captureByIds({ ids, handle, accountUrl, ...(config.postPagePauseSeconds ? { pagePauseMs: config.postPagePauseSeconds * 1000 } : {}), outDir: postsMediaDir(channelRoot), shots: opts.shots, media: opts.media, force: opts.force, articles: opts.articles, archived, cookies: alwaysCookies(policy), cookieSource: xLogin?.source, browserCookies: xLogin?.browserSpec, signal: opts.signal ?? controller.signal, drain: opts.drain, onLog: log, }); } catch (err) { const message = (err as Error).message; log(`[error] ${message}`); return fail(message); } // What the pages said about each post, folded into the sidecar. const observations = new Map(); for (const o of result.outcomes) { if (o.availability) observations.set(o.id, o.availability); } if (observations.size > 0) { const { map, newlyDeleted, changed } = mergePostAvailability( await readPostAvailability(channelRoot), observations, new Date().toISOString(), ); await writePostAvailability(channelRoot, map); log( `Availability: ${observations.size} recorded, ${changed} changed` + (newlyDeleted.length ? `, newly deleted: ${newlyDeleted.join(", ")}` : "") + ".", ); } const count = (state: string) => result.outcomes.filter((o) => o.state === state).length; log( `Done: ${count("captured")} captured, ${count("deleted")} deleted, ` + `${count("unavailable")} unavailable, ${count("error")} failed` + (count("login-wall") ? `, ${count("login-wall")} met a login wall` : "") + `; ${ids.length - result.outcomes.length} not attempted or already captured.`, ); // A run that stopped at a login is a failed run (the job says so); one that // was cancelled or met deleted posts is not. if (result.needsCookies) { return { ok: false, outcomes: result.outcomes, needsCookies: true, error: result.stoppedEarly ?? (forum ? "The capture needs the forum session (Connect)." : "The capture needs an X login."), }; } const stoppedByOperator = (opts.signal ?? controller.signal).aborted || Boolean(opts.drain?.aborted); if (result.stoppedEarly && !stoppedByOperator) { return { ok: false, outcomes: result.outcomes, error: result.stoppedEarly }; } return { ok: true, outcomes: result.outcomes }; }