#!/usr/bin/env tsx // Fetch social posts for one channel into its on-disk posts corpus. // // pnpm --filter yt-dlp-transcript-common exec tsx bin/fetch-posts.ts \ // --slug [--full | --older [--floor YYYY-MM-DD] [--from YYYY-MM-DD] [--force]] [--limit N] [--pages N] // // --pages caps how many pages one run reads, for a source read page by page (a // forum thread: its latest N pages, newest first; the next run continues where // this one stopped). // // --full re-walks the account's whole history instead of stopping at the // stored watermark. New posts are still deduped against the posts-archive, so // it repairs a gap rather than creating duplicates. // // --older walks the account's history backwards from the oldest archived post // (X: search windows), below what the timeline reaches; --floor is the date it // stops at; --from starts it afresh at a date, replacing its saved position // (and a "complete") — for a gap above one surviving old post. Its position // is saved apart from the timeline's, so a later run // continues it and a normal fetch is unaffected. An account that shows no // posts (nothing archived, and the last timeline fetch read none) is refused // a search walk unless --force. import { getPaths } from "../lib/paths"; import { getSettings } from "../lib/settings"; import { fetchPosts } from "../controller/fetchPosts"; import { parseFlags } from "./_parseFlags"; const flags = parseFlags(process.argv.slice(2)); const slug = flags.slug; if (!slug) { console.error( "Usage: fetch-posts.ts --slug [--full | --older [--floor YYYY-MM-DD] [--from YYYY-MM-DD] [--force]] [--limit N] [--pages N]", ); process.exit(2); } const limitRaw = flags.limit ? Number(flags.limit) : undefined; const limit = typeof limitRaw === "number" && Number.isFinite(limitRaw) && limitRaw > 0 ? Math.floor(limitRaw) : undefined; const pagesRaw = flags.pages ? Number(flags.pages) : undefined; const pages = typeof pagesRaw === "number" && Number.isFinite(pagesRaw) && pagesRaw > 0 ? Math.floor(pagesRaw) : undefined; fetchPosts({ paths: getPaths(), slug, settings: getSettings(), full: flags.full === "true", older: flags.older === "true", floor: flags.floor, from: flags.from, force: flags.force === "true", limit, pages, onLog: (line) => console.log(line), }) .then((result) => { if (!result.ok) { console.error(`Failed: ${result.error}`); process.exit(1); } console.log( `Done: ${result.written} new, ${result.skipped} skipped, ` + `${result.complete ? "caught up" : "more history remains"}.`, ); }) .catch((err) => { console.error(err); process.exit(1); });