// "Are these archived posts still up?" — the posts analogue of // controller/checkAvailability.ts. // // For a commentary archive this is arguably the most valuable sweep there is: // a deleted post is exactly the artefact the archive exists to preserve, and // the only way to know one was deleted is to have looked. // // Same discipline as the video checker: // - the append-only JSONL is never rewritten; verdicts live in a sidecar // that the index merges into the served pages // - history is appended only when a verdict CHANGES, preserving the moment a // post was first seen deleted // - a failed check records `error`, never `deleted` — we never report a // deletion from ignorance // // Modes mirror the video checker's: `stale` re-checks the least recently // checked first (the default, cheap and incremental), `unchecked` only covers // posts never looked at, `all` re-checks everything. import path from "node:path"; import { readChannelConfig } from "./channels"; import type { Paths } from "../lib/paths"; import { isSocialChannel } from "../lib/channelConfig"; import { mergePostAvailability, readAllPosts, readPostAvailability, writePostAvailability, } from "../lib/posts-server"; import type { PostAvailability, PostAvailabilityMap } from "../lib/posts"; import { handleFromAccountUrl, resolveSocialFetcher, } from "../social/fetchers"; import "../social/blueskyFetcher"; import "../social/xenforoFetcher"; import "../social/xGalleryDlFetcher"; import "../social/xPlaywrightFetcher"; import "../social/xNitterFetcher"; export type CheckPostAvailabilityMode = "stale" | "unchecked" | "all"; export const CHECK_POST_AVAILABILITY_MODES: CheckPostAvailabilityMode[] = [ "stale", "unchecked", "all", ]; // Default ceiling per run. Bluesky answers 25 posts per request so it could // take far more, but a per-post browser check (X) cannot — one bound keeps the // job predictable on both. export const DEFAULT_POST_CHECK_LIMIT = 200; export type CheckPostAvailabilityOptions = { paths: Paths; slug: string; mode?: CheckPostAvailabilityMode; limit?: number; onLog?: (line: string) => void; signal?: AbortSignal; }; export type CheckPostAvailabilityResult = { ok: boolean; checked: number; deleted: number; newlyDeleted: string[]; changed: number; remaining: number; error?: string; }; // Which posts to look at this run, oldest-checked first so repeated runs sweep // the whole archive rather than re-checking the same head every time. export function selectPostsToCheck( ids: ReadonlyArray, availability: PostAvailabilityMap, mode: CheckPostAvailabilityMode, limit: number, ): string[] { const candidates = ids.filter((id) => { const rec = availability[id]; if (mode === "all") return true; if (mode === "unchecked") return !rec; // "stale": everything is a candidate, ordered by how long ago we looked. return true; }); if (mode === "stale") { candidates.sort((a, b) => { // Never-checked first, then least-recently-checked. const ra = availability[a]?.checkedAt ?? ""; const rb = availability[b]?.checkedAt ?? ""; return ra.localeCompare(rb); }); } return candidates.slice(0, Math.max(0, limit)); } export async function checkPostAvailability( opts: CheckPostAvailabilityOptions, ): Promise { const { paths, slug, onLog, signal } = opts; const mode = opts.mode ?? "stale"; const limit = opts.limit ?? DEFAULT_POST_CHECK_LIMIT; const log = (line: string) => onLog?.(line); const channelRoot = path.join(paths.channelsDir, slug); const empty = { checked: 0, deleted: 0, newlyDeleted: [] as string[], changed: 0, remaining: 0, }; const config = await readChannelConfig(paths, slug); if (!config) { return { ok: false, ...empty, error: `No such channel: ${slug}` }; } if (!isSocialChannel(config)) { return { ok: false, ...empty, error: `${slug} is not a social channel.` }; } const fetcher = resolveSocialFetcher(config.postFetcher, config.url); if (!fetcher) { return { ok: false, ...empty, error: `No post fetcher configured for ${slug}.` }; } if (!fetcher.checkAvailability) { // Better to say so than to guess. gallery-dl, for instance, has no cheap // per-post liveness probe, so its channels should switch fetcher to sweep. return { ok: false, ...empty, error: `${fetcher.label} cannot check post availability. ` + `Switch this channel's scraping method to one that can.`, }; } const handle = config.socialHandle ?? handleFromAccountUrl(config.url ?? "") ?? ""; if (!handle) { return { ok: false, ...empty, error: `No account handle for ${slug}.` }; } const posts = await readAllPosts(channelRoot); if (posts.length === 0) { return { ok: true, ...empty }; } const availability = await readPostAvailability(channelRoot); const ids = posts.map((p) => p.id); const batch = selectPostsToCheck(ids, availability, mode, limit); if (batch.length === 0) { log(`Nothing to check for ${slug} (mode: ${mode}).`); return { ok: true, ...empty }; } log( `Checking ${batch.length} of ${ids.length} archived post(s) for ${slug} ` + `via ${fetcher.label} (mode: ${mode}).`, ); const controller = new AbortController(); let observations: Map; try { observations = await fetcher.checkAvailability({ ids: batch, handle, signal: signal ?? controller.signal, onLog: log, }); } catch (err) { const message = (err as Error).message; log(`[error] ${message}`); return { ok: false, ...empty, error: message }; } const checkedAt = new Date().toISOString(); const { map, newlyDeleted, changed } = mergePostAvailability( availability, observations, checkedAt, ); await writePostAvailability(channelRoot, map); const deleted = Object.values(map).filter( (r) => r.availability === "deleted", ).length; const checkedIds = new Set(Object.keys(map)); const remaining = ids.filter((id) => !checkedIds.has(id)).length; if (newlyDeleted.length > 0) { log( `${newlyDeleted.length} post(s) newly detected as DELETED: ${newlyDeleted.slice(0, 10).join(", ")}` + (newlyDeleted.length > 10 ? ` (+${newlyDeleted.length - 10} more)` : ""), ); } log( `Done: ${observations.size} checked, ${changed} changed, ${deleted} deleted overall, ` + `${remaining} never checked.`, ); return { ok: true, checked: observations.size, deleted, newlyDeleted, changed, remaining, }; }