// The older-posts backfill's window arithmetic, pure (no I/O) so every step of // the walk is testable without a fetcher. // // The walk goes BACKWARDS from the oldest archived post in fixed windows of // UTC days — the unit X's `since:`/`until:` search operators take. Each window // is one bounded search; the next window ends where the last one began, so the // windows tile the history with no gap and no overlap. The walk ends at a floor // date (given, or the account's creation date once a post has carried it), or // after a run of windows in a row that held no posts. import type { OlderBackfillPosition, OlderBackfillState, } from "../lib/posts-server"; // One window's span. Three months keeps one search bounded (a run that stops // mid-window resumes inside it) without spending a request per quiet week. export const OLDER_WINDOW_MONTHS = 3; // Windows in a row with no posts before the walk calls the history done: four // three-month windows, a year with nothing. export const OLDER_MAX_EMPTY_WINDOWS = 4; // The same, for a walk over an account with nothing archived that has found // nothing in this run either: two windows. Searching an empty account window // after window only repeats empty searches. export const OLDER_MAX_EMPTY_WINDOWS_NOTHING_ARCHIVED = 2; const DAY_RE = /^\d{4}-\d{2}-\d{2}$/; // A real YYYY-MM-DD calendar day (2021-02-30 is not one). export function isUtcDay(value: string): boolean { if (!DAY_RE.test(value)) return false; const ms = Date.parse(`${value}T00:00:00Z`); return Number.isFinite(ms) && new Date(ms).toISOString().slice(0, 10) === value; } // The UTC day an instant falls on. export function utcDayOf(instant: string | Date): string { const d = typeof instant === "string" ? new Date(instant) : instant; return d.toISOString().slice(0, 10); } export function addUtcDays(day: string, days: number): string { const [y, m, d] = day.split("-").map(Number); return new Date(Date.UTC(y, m - 1, d + days)).toISOString().slice(0, 10); } // Calendar months, clamped to the target month's last day (2024-05-31 minus // three months is 2024-02-29, not March 2nd). export function addUtcMonths(day: string, months: number): string { const [y, m, d] = day.split("-").map(Number); const target = new Date(Date.UTC(y, m - 1 + months, 1)); const lastDay = new Date( Date.UTC(target.getUTCFullYear(), target.getUTCMonth() + 1, 0), ).getUTCDate(); target.setUTCDate(Math.min(d, lastDay)); return target.toISOString().slice(0, 10); } const later = (a: string, b: string | undefined): string => b !== undefined && b > a ? b : a; // Where the walk may not go below: the given floor date, or the day the // account was created, whichever is later. export function olderFloorDay( floor: string | undefined, accountCreatedAt: string | undefined, ): string | undefined { const created = accountCreatedAt ? utcDayOf(accountCreatedAt) : undefined; if (floor && created) return later(floor, created); return floor ?? created; } // The first window: it ends the day AFTER the oldest archived post (`until` is // exclusive), so the rest of that day is read too — the posts already on disk // are skipped as duplicates. With nothing archived it ends tomorrow. export function firstOlderWindow(opts: { oldestArchivedAt?: string; now?: Date; months?: number; floor?: string; }): OlderBackfillPosition { const months = opts.months ?? OLDER_WINDOW_MONTHS; const until = addUtcDays( utcDayOf(opts.oldestArchivedAt ?? opts.now ?? new Date()), 1, ); return { since: later(addUtcMonths(until, -months), opts.floor), until, emptyWindows: 0, }; } // True when the window starts at or below the floor: it is the walk's last. export function isAtFloor( position: OlderBackfillPosition, floor: string | undefined, ): boolean { return floor !== undefined && position.since <= floor; } export type OlderWindowStep = | { complete: true; reason: string; emptyWindows: number } | { complete: false; next: OlderBackfillPosition }; // What follows a window walked to its end. `hadPosts` counts any post the // window held, archived already or not — a window full of known posts is not // an empty one. `nothingFound`: nothing is archived and this run has read no // post — the walk ends after OLDER_MAX_EMPTY_WINDOWS_NOTHING_ARCHIVED empty // windows, not a year of them. export function stepOlderWindow( position: OlderBackfillPosition, opts: { hadPosts: boolean; floor?: string; months?: number; maxEmptyWindows?: number; nothingFound?: boolean; }, ): OlderWindowStep { const months = opts.months ?? OLDER_WINDOW_MONTHS; const maxEmpty = opts.maxEmptyWindows ?? (opts.nothingFound ? OLDER_MAX_EMPTY_WINDOWS_NOTHING_ARCHIVED : OLDER_MAX_EMPTY_WINDOWS); const emptyWindows = opts.hadPosts ? 0 : position.emptyWindows + 1; if (isAtFloor(position, opts.floor)) { return { complete: true, reason: `reached ${opts.floor}, the earliest date to walk to`, emptyWindows, }; } if (emptyWindows >= maxEmpty) { return { complete: true, reason: `${emptyWindows} windows of ${months} months in a row held no posts, back to ${position.since}` + (opts.nothingFound ? ", with nothing archived and none found — an empty account is not searched further" : ""), emptyWindows, }; } const until = position.since; return { complete: false, next: { since: later(addUtcMonths(until, -months), opts.floor), until, emptyWindows, }, }; } // One line for the channel page. export function describeOlderBackfill( state: OlderBackfillState | undefined, ): string { if (!state) return "not started"; if (state.complete) { return `complete${state.completeReason ? ` — ${state.completeReason}` : ""}`; } return `walked back to ${state.until}; next ${state.since} – ${state.until}`; }