commit 709175ead6010f26be56b8e8b971bbf62d51b6fc
parent fa2f521343c6c9a2b4a94e7b9871d66926ea8c4b
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sat, 3 Oct 2026 13:25:53 -0400
common: X older-posts backfill through search windows
The gallery-dl X fetcher gains fetchOlder: it walks an account's history
backwards from the oldest archived post, one search window at a time
(from:<handle> since:<day> until:<day>, three months per window), through
gallery-dl's twitter search extractor. Posts go through the same normalizer
and shard writer. Progress is saved mid-window (the oldest post read is the
resume point, passed back as max_id:) and at every window boundary, in
posts-state.json's own 'older' field; the timeline cursor is untouched and
a normal fetch carries 'older' through without reading it. The walk ends at
a floor date, the account's creation date, or after four empty windows in a
row, and records that it is complete.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
5 files changed, 916 insertions(+), 32 deletions(-)
diff --git a/common/controller/fetchPosts.ts b/common/controller/fetchPosts.ts
@@ -21,16 +21,25 @@ import {
} from "../lib/cookiePolicy";
import {
latestPostCreatedAt,
+ oldestPostCreatedAt,
readPostFetchState,
readSeenPostIds,
writePostFetchState,
writePosts,
+ type OlderBackfillPosition,
+ type OlderBackfillState,
type PostFetchState,
} from "../lib/posts-server";
import {
handleFromAccountUrl,
resolveSocialFetcher,
+ type SocialFetcher,
} from "../social/fetchers";
+import {
+ firstOlderWindow,
+ isUtcDay,
+ olderFloorDay,
+} from "../social/olderBackfill";
// Registering the built-in fetchers is a side effect of importing them. Keep
// this list here (rather than inside fetchers.ts) so the registry module stays
// free of imports from the heavier fetchers.
@@ -42,6 +51,7 @@ import "../social/xGalleryDlFetcher";
// only ever used when a channel opts into it via postFetcher: "x-playwright".
import "../social/xPlaywrightFetcher";
import "../social/xNitterFetcher";
+import type { XCookieSource } from "../social/xCookieSource";
import {
resolveXCookieSourceFor,
type XLoginSettings,
@@ -57,11 +67,37 @@ export type FetchPostsOptions = {
// posts are still deduped against the archive, so this is a safe repair
// operation rather than a duplicate-maker.
full?: boolean;
+ // Walk the account's history BACKWARDS from the oldest archived post, with
+ // the fetcher's `fetchOlder` (X: search windows), instead of fetching new
+ // posts. Its position is kept in posts-state.json's `older`, apart from the
+ // timeline cursor. Not with `full`.
+ older?: boolean;
+ // The older walk's floor date (YYYY-MM-DD): it stops there. Remembered for
+ // later runs of the walk. Default: none (it stops after a run of empty
+ // windows, or at the account's creation date).
+ floor?: string;
+ // The older walk's pause between two windows. Default: the fetcher's own.
+ olderWindowPauseMs?: number;
limit?: number;
onLog?: (line: string) => void;
signal?: AbortSignal;
};
+// The refusal for both walks at once, shared with the server action so the
+// button, the ops route and this controller say the same thing.
+export const FULL_AND_OLDER_REFUSAL =
+ "A full re-fetch and an older-posts fetch are different walks — run one at a time.";
+
+// Why this fetcher cannot fetch older posts, or null when it can.
+export function olderPostsProblem(
+ fetcher: Pick<SocialFetcher, "label" | "fetchOlder"> | undefined,
+): string | null {
+ if (!fetcher) return "This channel has no post fetcher.";
+ return typeof fetcher.fetchOlder === "function"
+ ? null
+ : `${fetcher.label} cannot fetch older posts.`;
+}
+
export type FetchPostsResult = {
ok: boolean;
written: number;
@@ -78,6 +114,19 @@ export async function fetchPosts(
const log = (line: string) => onLog?.(line);
const channelRoot = path.join(paths.channelsDir, slug);
+ if (opts.full && opts.older) {
+ return { ok: false, written: 0, skipped: 0, complete: false, error: FULL_AND_OLDER_REFUSAL };
+ }
+ if (opts.floor !== undefined && !isUtcDay(opts.floor)) {
+ return {
+ ok: false,
+ written: 0,
+ skipped: 0,
+ complete: false,
+ error: `"${opts.floor}" is not a date (YYYY-MM-DD)`,
+ };
+ }
+
const config = await readChannelConfig(paths, slug);
if (!config) {
return { ok: false, written: 0, skipped: 0, complete: false, error: `No such channel: ${slug}` };
@@ -118,6 +167,30 @@ export async function fetchPosts(
const seenIds = await readSeenPostIds(channelRoot);
const priorState = await readPostFetchState(channelRoot);
+
+ if (opts.older) {
+ const problem = olderPostsProblem(fetcher);
+ if (problem) {
+ return { ok: false, written: 0, skipped: 0, complete: false, error: problem };
+ }
+ const policy = resolveCookiePolicy(settings, config);
+ const xLogin =
+ fetcher.platform === "twitter"
+ ? await resolveXCookieSourceFor(paths, settings, policy.cookies)
+ : undefined;
+ return fetchOlderPosts({
+ opts,
+ fetcher,
+ channelRoot,
+ handle,
+ accountUrl,
+ seenIds,
+ priorState,
+ cookies: alwaysCookies(policy),
+ cookieSource: xLogin?.source,
+ browserCookies: xLogin?.browserSpec,
+ });
+ }
// A stored cursor means the previous run stopped early (hit its limit, or was
// cancelled). Resume the backfill from there rather than applying the
// watermark, which would otherwise leave that history permanently missing.
@@ -156,6 +229,9 @@ export async function fetchPosts(
const state: PostFetchState = {
lastFetchedAt: new Date().toISOString(),
};
+ // The older walk's position rides along untouched: a normal fetch neither
+ // reads it nor drops it.
+ if (priorState?.older) state.older = priorState.older;
// Progress a long run saves as it goes (a fetcher that streams calls this
// between pages): the posts so far, then the resume point they are safe
@@ -258,3 +334,169 @@ export async function fetchPosts(
error: result.error,
};
}
+
+// The older-posts walk (`FetchPostsOptions.older`). Its own position in
+// posts-state.json's `older`; every other field of the state is carried as it
+// was, so the timeline cursor and watermark are exactly what the last normal
+// fetch left. It does not stamp the channel's lastSyncedAt: it is not a sync,
+// and the scheduler's next new-posts fetch stays where it was.
+async function fetchOlderPosts(ctx: {
+ opts: FetchPostsOptions;
+ fetcher: SocialFetcher;
+ channelRoot: string;
+ handle: string;
+ accountUrl: string;
+ seenIds: ReadonlySet<string>;
+ priorState: PostFetchState | null;
+ cookies?: string;
+ cookieSource?: XCookieSource;
+ browserCookies?: string;
+}): Promise<FetchPostsResult> {
+ const { opts, fetcher, channelRoot, handle, priorState } = ctx;
+ const { slug } = opts;
+ const log = (line: string) => opts.onLog?.(line);
+ const fetchOlder = fetcher.fetchOlder!;
+ const prior = priorState?.older;
+
+ if (prior?.complete) {
+ log(
+ `Older posts for ${slug} are already complete` +
+ (prior.completedAt ? ` (since ${prior.completedAt})` : "") +
+ (prior.completeReason ? `: ${prior.completeReason}` : "") +
+ ". Nothing to walk.",
+ );
+ return { ok: true, written: 0, skipped: 0, complete: true };
+ }
+
+ const floor = opts.floor ?? prior?.floor;
+ let accountCreatedAt = prior?.accountCreatedAt;
+ const startedAt = new Date().toISOString();
+ let position: OlderBackfillPosition;
+ if (prior) {
+ position = {
+ since: prior.since,
+ until: prior.until,
+ emptyWindows: prior.emptyWindows ?? 0,
+ ...(prior.maxId ? { maxId: prior.maxId } : {}),
+ };
+ } else {
+ const oldest = await oldestPostCreatedAt(channelRoot);
+ position = firstOlderWindow({
+ oldestArchivedAt: oldest ?? undefined,
+ floor: olderFloorDay(floor, accountCreatedAt),
+ });
+ }
+
+ log(
+ `Fetching older posts for ${slug} via ${fetcher.label} (@${handle}): ` +
+ (prior ? "resuming at " : "starting at ") +
+ `${position.since} – ${position.until}` +
+ (floor ? `, down to ${floor}` : "") +
+ `; ${ctx.seenIds.size} already archived.`,
+ );
+
+ let written = 0;
+ let skipped = 0;
+ const shards = new Set<string>();
+ const olderState = (
+ at: OlderBackfillPosition,
+ extra: Partial<OlderBackfillState> = {},
+ ): OlderBackfillState => ({
+ since: at.since,
+ until: at.until,
+ emptyWindows: at.emptyWindows,
+ ...(at.maxId ? { maxId: at.maxId } : {}),
+ ...(floor ? { floor } : {}),
+ ...(accountCreatedAt ? { accountCreatedAt } : {}),
+ lastRunAt: startedAt,
+ lastWritten: written,
+ ...extra,
+ });
+ // Everything but `older` is carried from the state as it was.
+ const save = (older: OlderBackfillState, needsCookies?: boolean) =>
+ writePostFetchState(channelRoot, {
+ ...(priorState ?? {}),
+ ...(needsCookies ? { needsCookies: true } : {}),
+ older,
+ });
+
+ const onCheckpoint = async (cp: {
+ posts: Post[];
+ position: OlderBackfillPosition;
+ accountCreatedAt?: string;
+ }) => {
+ const w = await writePosts(channelRoot, cp.posts);
+ written += w.written;
+ skipped += w.skipped;
+ for (const shard of w.shards) shards.add(shard);
+ accountCreatedAt ??= cp.accountCreatedAt;
+ await save(olderState(cp.position));
+ if (w.written > 0) {
+ log(`Saved ${w.written} older post(s) (${written} so far this run); resume point recorded.`);
+ }
+ };
+
+ const controller = new AbortController();
+ let result;
+ try {
+ result = await fetchOlder({
+ accountUrl: ctx.accountUrl,
+ handle,
+ channelSlug: slug,
+ seenIds: ctx.seenIds,
+ cookies: ctx.cookies,
+ cookieSource: ctx.cookieSource,
+ browserCookies: ctx.browserCookies,
+ limit: opts.limit,
+ signal: opts.signal ?? controller.signal,
+ onLog: log,
+ position,
+ floor,
+ accountCreatedAt,
+ windowPauseMs: opts.olderWindowPauseMs,
+ onCheckpoint,
+ });
+ } catch (err) {
+ const message = (err as Error).message;
+ log(`[error] ${message}`);
+ // Whatever was checkpointed is on disk with its position; keep that one.
+ const now = (await readPostFetchState(channelRoot))?.older;
+ await save({ ...(now ?? olderState(position)), lastError: message });
+ return { ok: false, written, skipped, complete: false, error: message };
+ }
+
+ accountCreatedAt ??= result.accountCreatedAt;
+ const final = await writePosts(channelRoot, result.posts);
+ written += final.written;
+ skipped += final.skipped;
+ for (const shard of final.shards) shards.add(shard);
+ log(
+ `Wrote ${written} older post(s), skipped ${skipped} already archived` +
+ (shards.size > 0 ? ` (shards: ${[...shards].join(", ")})` : "") +
+ (result.complete ? " — older posts complete." : " — more history remains."),
+ );
+ if (result.error) log(`[error] ${result.error}`);
+ await save(
+ olderState(
+ result.position,
+ result.complete
+ ? {
+ complete: true,
+ completedAt: new Date().toISOString(),
+ ...(result.completeReason ? { completeReason: result.completeReason } : {}),
+ }
+ : result.error
+ ? { lastError: result.error }
+ : {},
+ ),
+ result.needsCookies,
+ );
+ return {
+ ok: !result.error,
+ written,
+ skipped,
+ complete: result.complete,
+ needsCookies: result.needsCookies,
+ error: result.error,
+ };
+}
diff --git a/common/lib/posts-server.ts b/common/lib/posts-server.ts
@@ -211,6 +211,24 @@ export async function latestPostCreatedAt(
return null;
}
+// The oldest archived post's createdAt: where the older-posts backfill starts.
+// Only the oldest non-empty shard is read (listPostShards sorts oldest first).
+export async function oldestPostCreatedAt(
+ channelRoot: string,
+): Promise<string | null> {
+ const shards = await listPostShards(channelRoot);
+ for (const shard of shards) {
+ const posts = await readPostShard(channelRoot, shard);
+ if (posts.length === 0) continue;
+ let oldest = posts[0].createdAt;
+ for (const post of posts) {
+ if (post.createdAt < oldest) oldest = post.createdAt;
+ }
+ return oldest;
+ }
+ return null;
+}
+
// ---------------------------------------------------------------------------
// Availability sidecar ("is this post still up?")
// ---------------------------------------------------------------------------
@@ -310,7 +328,42 @@ export type PostFetchState = {
// the post-corpus analogue of the video pipeline's needs_auth outcome.
needsCookies?: boolean;
lastFetchedCount?: number;
+ // The TIMELINE walk's resume point (gallery-dl's cursor). Only a normal fetch
+ // reads it; the older-posts backfill keeps its own position in `older`.
cursor?: string;
+ // The older-posts backfill (a fetcher's `fetchOlder`), which walks the
+ // account's history backwards through search windows. Kept apart from
+ // `cursor` so a normal fetch never resumes backwards, and carried through
+ // every write a normal fetch makes.
+ older?: OlderBackfillState;
+};
+
+// Where the older-posts backfill stands. The window is in UTC days, as X's
+// `since:`/`until:` search operators take them: posts created on or after
+// `since` and before `until`.
+export type OlderBackfillPosition = {
+ since: string; // YYYY-MM-DD
+ until: string; // YYYY-MM-DD
+ // Resume INSIDE the window: only posts with an id at or below this one (the
+ // oldest post the walk has read in this window).
+ maxId?: string;
+ // Windows in a row, ending with the last one walked, that held no posts.
+ emptyWindows: number;
+};
+
+export type OlderBackfillState = OlderBackfillPosition & {
+ // The date the walk stops at (YYYY-MM-DD), when one was given.
+ floor?: string;
+ // The account's creation time, once a fetched post has carried it: the walk
+ // never goes below it.
+ accountCreatedAt?: string;
+ complete?: boolean;
+ completedAt?: string;
+ // Why the walk ended, as a sentence ("reached the floor date 2020-01-01").
+ completeReason?: string;
+ lastRunAt?: string;
+ lastWritten?: number;
+ lastError?: string;
};
export const POST_FETCH_STATE_FILENAME = "posts-state.json";
diff --git a/common/social/fetchers.ts b/common/social/fetchers.ts
@@ -14,6 +14,7 @@
// in the client bundle.
import type { Post, PostAvailability, PostPlatform } from "../lib/posts";
+import type { OlderBackfillPosition } from "../lib/posts-server";
import type { XCookieSource } from "./xCookieSource";
export type PostFetchInput = {
@@ -80,6 +81,54 @@ export type PostFetchResult = {
error?: string;
};
+// Input for the older-posts backfill (`SocialFetcher.fetchOlder`): walk the
+// account's history BACKWARDS from `position`, a window at a time, below what
+// the timeline walk can reach. Login, limit, signal and log are the same as a
+// normal fetch's; there is no watermark and no timeline cursor.
+export type OlderPostFetchInput = Pick<
+ PostFetchInput,
+ | "accountUrl"
+ | "handle"
+ | "channelSlug"
+ | "seenIds"
+ | "cookies"
+ | "cookieSource"
+ | "browserCookies"
+ | "limit"
+ | "signal"
+ | "onLog"
+> & {
+ // Where to start: the stored position of an unfinished walk, or the first
+ // window below the oldest archived post (olderBackfill.ts, firstOlderWindow).
+ position: OlderBackfillPosition;
+ // The date the walk stops at (YYYY-MM-DD), when one is set.
+ floor?: string;
+ // The account's creation time, when an earlier run learned it.
+ accountCreatedAt?: string;
+ // Pause between two windows' searches, so a run of quiet windows is not a
+ // burst of requests. Default: the fetcher's own.
+ windowPauseMs?: number;
+ // Save progress during the walk: posts not yet handed back, and the position
+ // they are safe under. Posts passed here are NOT returned again.
+ onCheckpoint?: (checkpoint: {
+ posts: Post[];
+ position: OlderBackfillPosition;
+ accountCreatedAt?: string;
+ }) => Promise<void>;
+};
+
+export type OlderPostFetchResult = {
+ posts: Post[];
+ // True when the walk has reached its end (`completeReason` says which).
+ complete: boolean;
+ completeReason?: string;
+ // Where the next run resumes, when the walk is not complete.
+ position: OlderBackfillPosition;
+ accountCreatedAt?: string;
+ needsCookies?: boolean;
+ error?: string;
+};
+
export type SocialFetcherProbe = {
ok: boolean;
// Display name for the account, used to autofill the channel form.
@@ -132,6 +181,10 @@ export type SocialFetcher = {
checkAvailability?(
input: PostAvailabilityInput,
): Promise<Map<string, PostAvailability>>;
+ // Optional: walk the account's history backwards below what `fetch` can
+ // reach (X: search windows). A fetcher without it has no such backfill, and
+ // the channel page offers no "Fetch older posts" for it.
+ fetchOlder?(input: OlderPostFetchInput): Promise<OlderPostFetchResult>;
};
// Populated by registerSocialFetcher() from each fetcher module. Indirection
diff --git a/common/social/olderBackfill.ts b/common/social/olderBackfill.ts
@@ -0,0 +1,152 @@
+// The older-posts backfill's window arithmetic, pure (no I/O) so every step of
+// the walk is testable without a fetcher.
+//
+// The walk goes BACKWARDS from the oldest archived post in fixed windows of
+// UTC days — the unit X's `since:`/`until:` search operators take. Each window
+// is one bounded search; the next window ends where the last one began, so the
+// windows tile the history with no gap and no overlap. The walk ends at a floor
+// date (given, or the account's creation date once a post has carried it), or
+// after a run of windows in a row that held no posts.
+
+import type {
+ OlderBackfillPosition,
+ OlderBackfillState,
+} from "../lib/posts-server";
+
+// One window's span. Three months keeps one search bounded (a run that stops
+// mid-window resumes inside it) without spending a request per quiet week.
+export const OLDER_WINDOW_MONTHS = 3;
+
+// Windows in a row with no posts before the walk calls the history done: four
+// three-month windows, a year with nothing.
+export const OLDER_MAX_EMPTY_WINDOWS = 4;
+
+const DAY_RE = /^\d{4}-\d{2}-\d{2}$/;
+
+// A real YYYY-MM-DD calendar day (2021-02-30 is not one).
+export function isUtcDay(value: string): boolean {
+ if (!DAY_RE.test(value)) return false;
+ const ms = Date.parse(`${value}T00:00:00Z`);
+ return Number.isFinite(ms) && new Date(ms).toISOString().slice(0, 10) === value;
+}
+
+// The UTC day an instant falls on.
+export function utcDayOf(instant: string | Date): string {
+ const d = typeof instant === "string" ? new Date(instant) : instant;
+ return d.toISOString().slice(0, 10);
+}
+
+export function addUtcDays(day: string, days: number): string {
+ const [y, m, d] = day.split("-").map(Number);
+ return new Date(Date.UTC(y, m - 1, d + days)).toISOString().slice(0, 10);
+}
+
+// Calendar months, clamped to the target month's last day (2024-05-31 minus
+// three months is 2024-02-29, not March 2nd).
+export function addUtcMonths(day: string, months: number): string {
+ const [y, m, d] = day.split("-").map(Number);
+ const target = new Date(Date.UTC(y, m - 1 + months, 1));
+ const lastDay = new Date(
+ Date.UTC(target.getUTCFullYear(), target.getUTCMonth() + 1, 0),
+ ).getUTCDate();
+ target.setUTCDate(Math.min(d, lastDay));
+ return target.toISOString().slice(0, 10);
+}
+
+const later = (a: string, b: string | undefined): string =>
+ b !== undefined && b > a ? b : a;
+
+// Where the walk may not go below: the given floor date, or the day the
+// account was created, whichever is later.
+export function olderFloorDay(
+ floor: string | undefined,
+ accountCreatedAt: string | undefined,
+): string | undefined {
+ const created = accountCreatedAt ? utcDayOf(accountCreatedAt) : undefined;
+ if (floor && created) return later(floor, created);
+ return floor ?? created;
+}
+
+// The first window: it ends the day AFTER the oldest archived post (`until` is
+// exclusive), so the rest of that day is read too — the posts already on disk
+// are skipped as duplicates. With nothing archived it ends tomorrow.
+export function firstOlderWindow(opts: {
+ oldestArchivedAt?: string;
+ now?: Date;
+ months?: number;
+ floor?: string;
+}): OlderBackfillPosition {
+ const months = opts.months ?? OLDER_WINDOW_MONTHS;
+ const until = addUtcDays(
+ utcDayOf(opts.oldestArchivedAt ?? opts.now ?? new Date()),
+ 1,
+ );
+ return {
+ since: later(addUtcMonths(until, -months), opts.floor),
+ until,
+ emptyWindows: 0,
+ };
+}
+
+// True when the window starts at or below the floor: it is the walk's last.
+export function isAtFloor(
+ position: OlderBackfillPosition,
+ floor: string | undefined,
+): boolean {
+ return floor !== undefined && position.since <= floor;
+}
+
+export type OlderWindowStep =
+ | { complete: true; reason: string; emptyWindows: number }
+ | { complete: false; next: OlderBackfillPosition };
+
+// What follows a window walked to its end. `hadPosts` counts any post the
+// window held, archived already or not — a window full of known posts is not
+// an empty one.
+export function stepOlderWindow(
+ position: OlderBackfillPosition,
+ opts: {
+ hadPosts: boolean;
+ floor?: string;
+ months?: number;
+ maxEmptyWindows?: number;
+ },
+): OlderWindowStep {
+ const months = opts.months ?? OLDER_WINDOW_MONTHS;
+ const maxEmpty = opts.maxEmptyWindows ?? OLDER_MAX_EMPTY_WINDOWS;
+ const emptyWindows = opts.hadPosts ? 0 : position.emptyWindows + 1;
+ if (isAtFloor(position, opts.floor)) {
+ return {
+ complete: true,
+ reason: `reached ${opts.floor}, the earliest date to walk to`,
+ emptyWindows,
+ };
+ }
+ if (emptyWindows >= maxEmpty) {
+ return {
+ complete: true,
+ reason: `${emptyWindows} windows of ${months} months in a row held no posts, back to ${position.since}`,
+ emptyWindows,
+ };
+ }
+ const until = position.since;
+ return {
+ complete: false,
+ next: {
+ since: later(addUtcMonths(until, -months), opts.floor),
+ until,
+ emptyWindows,
+ },
+ };
+}
+
+// One line for the channel page.
+export function describeOlderBackfill(
+ state: OlderBackfillState | undefined,
+): string {
+ if (!state) return "not started";
+ if (state.complete) {
+ return `complete${state.completeReason ? ` — ${state.completeReason}` : ""}`;
+ }
+ return `walked back to ${state.until}; next ${state.since} – ${state.until}`;
+}
diff --git a/common/social/xGalleryDlFetcher.ts b/common/social/xGalleryDlFetcher.ts
@@ -29,11 +29,15 @@ import type { Readable } from "node:stream";
import { execa } from "execa";
import { getPaths } from "../lib/paths";
import type { Post } from "../lib/posts";
-import { normalizeXTweet, type XTweetRaw } from "./xNormalize";
+import type { OlderBackfillPosition } from "../lib/posts-server";
+import { normalizeXTweet, xCreatedAt, type XTweetRaw } from "./xNormalize";
+import { olderFloorDay, stepOlderWindow } from "./olderBackfill";
import { readXSessionStatus, xCookieFile } from "./xSessionBroker";
import type { XCookieSource } from "./xCookieSource";
import {
registerSocialFetcher,
+ type OlderPostFetchInput,
+ type OlderPostFetchResult,
type PostFetchInput,
type PostFetchResult,
type SocialFetcher,
@@ -81,22 +85,39 @@ export function timelineUrlFor(accountUrl: string): string {
}
}
-export function buildGalleryDlArgs(opts: {
- accountUrl: string;
+type GalleryDlLoginAndCap = {
cookies?: string;
// A cookies.txt exported by the Playwright session broker
// (xSessionBroker.ts). PREFERRED over `cookies`: a live browser profile keeps
// the session fresh, which is the fix for X cookies expiring in days.
cookieFile?: string;
limit?: number;
- // gallery-dl's own resume point (`extractor.twitter.cursor`), as logged by a
- // previous streamed run — see GalleryDlStream.
- cursor?: string;
- // A long fetch, read as it runs: log at debug level so gallery-dl prints its
- // "Cursor: …" line after every page. Off for the probe, whose error message
- // is the tail of stderr and must not be buried in debug noise.
- stream?: boolean;
-}): string[] {
+};
+
+export function buildGalleryDlArgs(
+ opts: GalleryDlLoginAndCap & {
+ accountUrl: string;
+ // gallery-dl's own resume point (`extractor.twitter.cursor`), as logged by
+ // a previous streamed run — see GalleryDlStream.
+ cursor?: string;
+ // A long fetch, read as it runs: log at debug level so gallery-dl prints
+ // its "Cursor: …" line after every page. Off for the probe, whose error
+ // message is the tail of stderr and must not be buried in debug noise.
+ stream?: boolean;
+ },
+): string[] {
+ const args = galleryDlCommonArgs(opts);
+ if (opts.cursor) {
+ args.push("-o", `extractor.twitter.cursor=${opts.cursor}`);
+ }
+ if (opts.stream) args.push("--verbose");
+ args.push(timelineUrlFor(opts.accountUrl));
+ return args;
+}
+
+// The flags every gallery-dl read shares: metadata only, streamed, text tweets
+// with their retweets and replies, the login, and the --range cap.
+function galleryDlCommonArgs(opts: GalleryDlLoginAndCap): string[] {
const args = [
// One JSON object per item on stdout — we want metadata, never files.
"--dump-json",
@@ -141,14 +162,115 @@ export function buildGalleryDlArgs(opts: {
if (opts.limit && opts.limit > 0) {
args.push("--range", `1-${Math.floor(opts.limit)}`);
}
- if (opts.cursor) {
- args.push("-o", `extractor.twitter.cursor=${opts.cursor}`);
+ return args;
+}
+
+// ---------------------------------------------------------------------------
+// Search: the older-posts backfill
+// ---------------------------------------------------------------------------
+//
+// X's timeline only pages back so far; below that, an account's posts are
+// reachable through search. gallery-dl's `twitter:search` extractor takes a
+// search URL (`https://x.com/search?q=<query>`), reads the query's `from:` user
+// for the records' `user`, and pages with `max_id:` (its default
+// `search-pagination`): after each page it rewrites the query's `max_id:` to
+// just below the oldest tweet it has read. That pagination logs no cursor, so
+// this walk's resume point is its own: the oldest post id read in the window,
+// passed back as `max_id:` — which gallery-dl's rewrite then replaces, rather
+// than adding a second one.
+//
+// Search needs a logged-in session; a guest run is refused before spawning.
+
+// An X handle as search's `from:` takes it. Anything else (a space, a quote, a
+// colon) would change the query's meaning rather than name an account.
+const X_HANDLE_RE = /^[A-Za-z0-9_]{1,50}$/;
+
+export function xSearchHandle(handle: string): string {
+ const bare = handle.trim().replace(/^@/, "");
+ if (!X_HANDLE_RE.test(bare)) {
+ throw new Error(
+ `"${handle}" is not an X handle search can take (letters, digits and "_")`,
+ );
}
- if (opts.stream) args.push("--verbose");
- args.push(timelineUrlFor(opts.accountUrl));
+ return bare;
+}
+
+// The query for one window. `include:nativeretweets` keeps the account's
+// reposts, as the timeline walk does (search leaves them out by default).
+export function xSearchQuery(
+ handle: string,
+ position: Pick<OlderBackfillPosition, "since" | "until" | "maxId">,
+): string {
+ const terms = [
+ `from:${xSearchHandle(handle)}`,
+ `since:${position.since}`,
+ `until:${position.until}`,
+ "include:nativeretweets",
+ ];
+ if (position.maxId) {
+ if (!/^\d+$/.test(position.maxId)) {
+ throw new Error(`"${position.maxId}" is not a post id`);
+ }
+ terms.push(`max_id:${position.maxId}`);
+ }
+ return terms.join(" ");
+}
+
+// The search URL gallery-dl's search extractor matches. The query is
+// percent-encoded whole (a space is %20, a colon %3A): gallery-dl turns "+"
+// into a space BEFORE it unquotes, so an encoded "+" survives as itself.
+// `f=live` is X's own "Latest" tab, which is also what gallery-dl asks for.
+export function xSearchUrl(query: string): string {
+ return `https://x.com/search?q=${encodeURIComponent(query)}&f=live`;
+}
+
+export function buildGalleryDlSearchArgs(
+ opts: GalleryDlLoginAndCap & {
+ handle: string;
+ position: Pick<OlderBackfillPosition, "since" | "until" | "maxId">;
+ },
+): string[] {
+ const args = galleryDlCommonArgs(opts);
+ args.push(
+ // Newest first, paged by max_id — the order the resume point relies on.
+ "-o",
+ "extractor.twitter.search-results=latest",
+ "-o",
+ "extractor.twitter.search-pagination=max_id",
+ );
+ args.push(xSearchUrl(xSearchQuery(opts.handle, opts.position)));
return args;
}
+// Inclusive: the post at the resume point is read again (and skipped as a
+// duplicate) rather than risk an off-by-one past it.
+function olderId(a: string | undefined, b: string): string {
+ if (a === undefined) return b;
+ try {
+ return BigInt(b) < BigInt(a) ? b : a;
+ } catch {
+ return a;
+ }
+}
+
+// Pause between two windows' searches. Each window is its own gallery-dl run
+// (a user lookup, then the search pages), so a stretch of empty windows would
+// otherwise be a burst of requests a second apart.
+export const OLDER_WINDOW_PAUSE_MS = 15_000;
+
+function pause(ms: number, signal: AbortSignal): Promise<void> {
+ if (ms <= 0 || signal.aborted) return Promise.resolve();
+ return new Promise((resolve) => {
+ const t = setTimeout(done, ms);
+ function done() {
+ clearTimeout(t);
+ signal.removeEventListener("abort", done);
+ resolve();
+ }
+ signal.addEventListener("abort", done, { once: true });
+ });
+}
+
// The timeline extractor's cursor is `<state>[_<tweet id>]/<API cursor>`, and
// "1/" is its first state with no API cursor: walk from the top. A run that
// stopped before gallery-dl logged any cursor resumes from here rather than
@@ -202,6 +324,11 @@ export class GalleryDlStream {
rateLimited = false;
stopReached = false;
cursor?: string;
+ // The oldest post id read this run (any post, archived or not) — a search
+ // walk's resume point.
+ oldestId?: string;
+ // The account's creation time, from a record whose `user` is the account.
+ accountCreatedAt?: string;
private previousCursor?: string;
private savedCursor?: string;
private pending: Post[] = [];
@@ -216,6 +343,8 @@ export class GalleryDlStream {
seenIds: ReadonlySet<string>;
since?: string;
stopAtKnown: boolean;
+ // The account's handle, to recognise its `user` record.
+ accountHandle?: string;
onLog?: (line: string) => void;
},
) {}
@@ -267,6 +396,14 @@ export class GalleryDlStream {
return { posts, cursor };
}
+ // Every post not yet handed over, whatever the cursor — for a walk whose
+ // resume point comes from the records themselves (oldestId), not stderr.
+ takePending(): Post[] {
+ const posts = this.pending;
+ this.pending = [];
+ return posts;
+ }
+
// A checkpoint that could not be written goes back into the queue, so the
// final write retries it.
requeue(posts: Post[]): void {
@@ -294,6 +431,18 @@ export class GalleryDlStream {
if (!post || this.ids.has(post.id)) return;
this.ids.add(post.id);
this.distinct++;
+ if (/^\d+$/.test(post.id)) this.oldestId = olderId(this.oldestId, post.id);
+ if (!this.accountCreatedAt && this.opts.accountHandle) {
+ const user = rec.user as Record<string, unknown> | undefined;
+ if (
+ user &&
+ typeof user.name === "string" &&
+ user.name.toLowerCase() === this.opts.accountHandle.toLowerCase() &&
+ typeof user.date === "string"
+ ) {
+ this.accountCreatedAt = xCreatedAt({ date: user.date });
+ }
+ }
const isKnown = this.opts.seenIds.has(post.id);
const isOlder =
!isKnown && Boolean(this.opts.since && post.createdAt <= this.opts.since);
@@ -416,6 +565,28 @@ function flattenRecords(items: unknown[]): XTweetRaw[] {
return out;
}
+// The login source (galleryDlCookieChoice) for one run, logged: the operator's
+// browser, or the session broker's exported jar — kept fresh by a live browser
+// profile, so it survives the cookie expiry that otherwise breaks gallery-dl
+// within days. One resolution for the timeline walk and the search walk.
+async function galleryDlLoginFor(
+ input: Pick<PostFetchInput, "cookieSource" | "browserCookies" | "cookies" | "onLog">,
+): Promise<{ cookies?: string; cookieFile?: string }> {
+ const paths = getPaths();
+ const status = await readXSessionStatus(paths);
+ const choice = galleryDlCookieChoice({
+ source: input.cookieSource,
+ browserCookies: input.browserCookies,
+ jarFile:
+ status.hasCookies && status.looksAuthenticated
+ ? xCookieFile(paths)
+ : undefined,
+ alwaysCookies: input.cookies,
+ });
+ input.onLog?.(choice.note);
+ return { cookies: choice.cookies, cookieFile: choice.cookieFile };
+}
+
// Feed a pipe to `onLine` a line at a time; resolves once it has closed.
function readLines(
input: Readable | null | undefined,
@@ -429,6 +600,232 @@ function readLines(
});
}
+const SEARCH_NEEDS_LOGIN =
+ "X search needs a logged-in session: connect an X account on /settings, or " +
+ "set cookiesFromBrowser for this channel (or globally), then run it again.";
+
+// The older-posts backfill: search windows walked backwards from `position`
+// (olderBackfill.ts does the window arithmetic). One gallery-dl run per
+// window, a pause between windows, the same per-run budget as a timeline
+// backfill across all of them. Progress is saved mid-window (the oldest post
+// read so far is the resume point) and at every window boundary.
+export async function fetchOlderViaSearch(
+ input: OlderPostFetchInput,
+): Promise<OlderPostFetchResult> {
+ const { channelSlug, seenIds, signal, onLog } = input;
+ const bin = getPaths().galleryDlBin;
+ let handle: string;
+ try {
+ handle = xSearchHandle(input.handle);
+ } catch (err) {
+ return {
+ posts: [],
+ complete: false,
+ position: input.position,
+ error: (err as Error).message,
+ };
+ }
+ const { cookies, cookieFile } = await galleryDlLoginFor(input);
+ if (!cookies && !cookieFile) {
+ onLog?.(`[auth] ${SEARCH_NEEDS_LOGIN}`);
+ return {
+ posts: [],
+ complete: false,
+ position: input.position,
+ needsCookies: true,
+ error: SEARCH_NEEDS_LOGIN,
+ };
+ }
+
+ const deadline = Date.now() + BACKFILL_BUDGET_MS;
+ const pauseMs = input.windowPauseMs ?? OLDER_WINDOW_PAUSE_MS;
+ let position: OlderBackfillPosition = { ...input.position };
+ let accountCreatedAt = input.accountCreatedAt;
+ // Posts not handed to a checkpoint, returned for the final write.
+ const carried: Post[] = [];
+ let checkpointsOn = Boolean(input.onCheckpoint);
+ let read = 0;
+
+ const save = async (posts: Post[], at: OlderBackfillPosition) => {
+ if (!checkpointsOn || !input.onCheckpoint) {
+ carried.push(...posts);
+ return;
+ }
+ try {
+ await input.onCheckpoint({ posts, position: at, accountCreatedAt });
+ } catch (err) {
+ // Keep the posts for the final write and stop checkpointing, so a saved
+ // position never runs ahead of what is on disk.
+ checkpointsOn = false;
+ carried.push(...posts);
+ onLog?.(`[warn] could not save progress mid-run: ${(err as Error).message}`);
+ }
+ };
+ const stopped = (
+ at: OlderBackfillPosition,
+ extra: Partial<OlderPostFetchResult> = {},
+ ): OlderPostFetchResult => ({
+ posts: carried,
+ complete: false,
+ position: at,
+ accountCreatedAt,
+ ...extra,
+ });
+
+ onLog?.(
+ `Searching @${handle}'s posts backwards, ${position.since} – ${position.until}` +
+ (position.maxId ? " from the saved resume point" : "") +
+ ` (up to ${Math.round(BACKFILL_BUDGET_MS / 60_000)} min this run; progress is saved as it goes)`,
+ );
+
+ for (;;) {
+ const floor = olderFloorDay(input.floor, accountCreatedAt);
+ if (floor !== undefined && position.until <= floor) {
+ return {
+ posts: carried,
+ complete: true,
+ completeReason: `the archive already reaches back to ${floor}, the earliest date to walk to`,
+ position,
+ accountCreatedAt,
+ };
+ }
+ const remainingMs = deadline - Date.now();
+ if (remainingMs <= 0) {
+ onLog?.(`Stopped at this run's ${Math.round(BACKFILL_BUDGET_MS / 60_000)}-minute budget; the next run resumes at ${position.since} – ${position.until}.`);
+ return stopped(position);
+ }
+ const remainingLimit = input.limit ? input.limit - read : undefined;
+ if (remainingLimit !== undefined && remainingLimit <= 0) {
+ return stopped(position);
+ }
+
+ const windowStart = position;
+ const stream = new GalleryDlStream({
+ channelSlug,
+ seenIds,
+ stopAtKnown: false,
+ accountHandle: handle,
+ onLog,
+ });
+ const args = buildGalleryDlSearchArgs({
+ handle,
+ position: windowStart,
+ cookies,
+ cookieFile,
+ limit: remainingLimit,
+ });
+ const subprocess = execa(bin, args, {
+ reject: false,
+ buffer: false,
+ stdin: "ignore",
+ timeout: remainingMs,
+ cancelSignal: signal,
+ env: { PYTHONUNBUFFERED: "1" },
+ });
+
+ // Mid-window checkpoints, one at a time and in order. The resume point is
+ // the oldest post read so far, which every post taken here is at or above.
+ let checkpoints: Promise<void> = Promise.resolve();
+ let lastCheckpointAt = Date.now();
+ const maybeCheckpoint = () => {
+ if (!checkpointsOn) return;
+ const due =
+ stream.pendingCount() >= CHECKPOINT_EVERY_POSTS ||
+ (stream.pendingCount() > 0 &&
+ Date.now() - lastCheckpointAt >= CHECKPOINT_EVERY_MS);
+ if (!due || !stream.oldestId) return;
+ lastCheckpointAt = Date.now();
+ const posts = stream.takePending();
+ const at = { ...windowStart, maxId: stream.oldestId };
+ checkpoints = checkpoints.then(() => save(posts, at));
+ };
+ const stdoutDone = readLines(subprocess.stdout, (line) => {
+ stream.pushStdout(line);
+ maybeCheckpoint();
+ });
+ const stderrDone = readLines(subprocess.stderr, (line) => {
+ stream.pushStderr(line);
+ maybeCheckpoint();
+ });
+ const res = await subprocess;
+ await Promise.all([stdoutDone, stderrDone]);
+ await checkpoints;
+
+ const spawnError = `${res.code ?? ""} ${res.message ?? ""}`;
+ if (res.failed && /ENOENT/.test(spawnError) && stream.distinct === 0) {
+ throw new Error(
+ `gallery-dl not found (looked for "${bin}"; set GALLERY_DL_BIN)`,
+ );
+ }
+
+ read += stream.distinct;
+ accountCreatedAt ??= stream.accountCreatedAt;
+ carried.push(...stream.finish());
+ const here: OlderBackfillPosition = {
+ ...windowStart,
+ maxId: stream.oldestId ?? windowStart.maxId,
+ };
+ if (stream.rateLimited) {
+ onLog?.("[rate-limit] X throttled this search — gallery-dl paused between pages.");
+ }
+ onLog?.(
+ `${windowStart.since} – ${windowStart.until}: read ${stream.distinct} post(s), ${stream.newPosts} new` +
+ (stream.known ? `, ${stream.known} already archived` : ""),
+ );
+
+ if (res.isCanceled || signal.aborted || res.timedOut) {
+ onLog?.(
+ res.timedOut
+ ? `Stopped at this run's ${Math.round(BACKFILL_BUDGET_MS / 60_000)}-minute budget; the next run resumes where this one left off.`
+ : "Cancelled; the next run resumes where this one left off.",
+ );
+ return stopped(here);
+ }
+ if (res.exitCode !== 0) {
+ const tail = stream.stderrTail();
+ if (looksLikeAuthFailure(tail)) {
+ onLog?.(`[auth] gallery-dl could not authenticate to X:\n${tail}`);
+ return stopped(here, { needsCookies: true, error: SEARCH_NEEDS_LOGIN });
+ }
+ return stopped(here, {
+ error: `gallery-dl exited ${res.exitCode ?? res.signal ?? "abnormally"}: ${tail || "(no output)"}`,
+ });
+ }
+ if (remainingLimit !== undefined && stream.distinct >= remainingLimit) {
+ return stopped(here);
+ }
+
+ // The window is walked to its end.
+ const step = stepOlderWindow(windowStart, {
+ // A window resumed below a saved point held posts above it.
+ hadPosts: stream.distinct > 0 || Boolean(windowStart.maxId),
+ floor: olderFloorDay(input.floor, accountCreatedAt),
+ });
+ if (step.complete) {
+ onLog?.(`Older-posts backfill complete: ${step.reason}.`);
+ return {
+ posts: carried,
+ complete: true,
+ completeReason: step.reason,
+ position: { since: windowStart.since, until: windowStart.until, emptyWindows: step.emptyWindows },
+ accountCreatedAt,
+ };
+ }
+ position = step.next;
+ // Every window boundary is saved: the posts so far, and the next window.
+ await save(carried.splice(0), position);
+ if (signal.aborted) {
+ onLog?.("Cancelled; the next run resumes where this one left off.");
+ return stopped(position);
+ }
+ await pause(pauseMs, signal);
+ if (signal.aborted) {
+ onLog?.("Cancelled; the next run resumes where this one left off.");
+ return stopped(position);
+ }
+ }
+}
+
export const xGalleryDlFetcher: SocialFetcher = {
id: "x-gallery-dl",
label: "X / Twitter (gallery-dl)",
@@ -498,23 +895,8 @@ export const xGalleryDlFetcher: SocialFetcher = {
async fetch(input: PostFetchInput): Promise<PostFetchResult> {
const { accountUrl, channelSlug, since, seenIds, limit, signal, onLog } =
input;
- const paths = getPaths();
- const bin = paths.galleryDlBin;
- // The login source (galleryDlCookieChoice): the operator's browser, or the
- // session broker's exported jar — kept fresh by a live browser profile, so
- // it survives the cookie expiry that otherwise breaks gallery-dl within days.
- const status = await readXSessionStatus(paths);
- const choice = galleryDlCookieChoice({
- source: input.cookieSource,
- browserCookies: input.browserCookies,
- jarFile:
- status.hasCookies && status.looksAuthenticated
- ? xCookieFile(paths)
- : undefined,
- alwaysCookies: input.cookies,
- });
- onLog?.(choice.note);
- const { cookies, cookieFile } = choice;
+ const bin = getPaths().galleryDlBin;
+ const { cookies, cookieFile } = await galleryDlLoginFor(input);
const args = buildGalleryDlArgs({
accountUrl,
cookies,
@@ -673,6 +1055,8 @@ export const xGalleryDlFetcher: SocialFetcher = {
? { posts, complete: false, cursor: resumeAt }
: { posts, complete: true };
},
+
+ fetchOlder: fetchOlderViaSearch,
};
registerSocialFetcher(xGalleryDlFetcher);
