commit bfcf9918b7e39acd16e77f58d56037d4e4758bc7
parent c3f3f4e94f69ae7f10f49269366147b7d270da02
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sun, 4 Oct 2026 22:10:17 -0400
Merge x-empty-account-drain (X fetches: an older walk refuses an account that shows no posts unless forced, stops after two empty windows when nothing is archived; Drain stops a fetch at the next resume point)
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
17 files changed, 620 insertions(+), 27 deletions(-)
diff --git a/common/bin/archilyzer.ts b/common/bin/archilyzer.ts
@@ -252,7 +252,7 @@ export const COMMANDS: Command[] = [
script(["duplicates"], "duplicate-shorts.ts",
"[--threshold N] [--all-durations] [--blocking title|duration|both] [--near F] [--tolerance N] … on-demand duplicate detection (after index + stats)", 8192),
script(["posts", "fetch"], "fetch-posts.ts",
- "--slug <channel> [--full | --older [--floor YYYY-MM-DD]] [--limit N] fetch a social channel's posts into its posts corpus (--older: walk back below the oldest archived post)"),
+ "--slug <channel> [--full | --older [--floor YYYY-MM-DD] [--force]] [--limit N] fetch a social channel's posts into its posts corpus (--older: walk back below the oldest archived post)"),
script(["posts", "check"], "check-post-availability.ts",
"--slug <channel> [--mode stale|unchecked|all] [--limit N] which archived posts were deleted at the source"),
script(["diarize", "backfill"], "diarize-backfill.ts",
diff --git a/common/bin/fetch-posts.ts b/common/bin/fetch-posts.ts
@@ -2,7 +2,7 @@
// Fetch social posts for one channel into its on-disk posts corpus.
//
// pnpm --filter yt-dlp-transcript-common exec tsx bin/fetch-posts.ts \
-// --slug <channel-slug> [--full | --older [--floor YYYY-MM-DD]] [--limit N]
+// --slug <channel-slug> [--full | --older [--floor YYYY-MM-DD] [--force]] [--limit N]
//
// --full re-walks the account's whole history instead of stopping at the
// stored watermark. New posts are still deduped against the posts-archive, so
@@ -11,7 +11,9 @@
// --older walks the account's history backwards from the oldest archived post
// (X: search windows), below what the timeline reaches; --floor is the date it
// stops at. Its position is saved apart from the timeline's, so a later run
-// continues it and a normal fetch is unaffected.
+// continues it and a normal fetch is unaffected. An account that shows no
+// posts (nothing archived, and the last timeline fetch read none) is refused
+// a search walk unless --force.
import { getPaths } from "../lib/paths";
import { getSettings } from "../lib/settings";
@@ -22,7 +24,7 @@ const flags = parseFlags(process.argv.slice(2));
const slug = flags.slug;
if (!slug) {
console.error(
- "Usage: fetch-posts.ts --slug <channel-slug> [--full | --older [--floor YYYY-MM-DD]] [--limit N]",
+ "Usage: fetch-posts.ts --slug <channel-slug> [--full | --older [--floor YYYY-MM-DD] [--force]] [--limit N]",
);
process.exit(2);
}
@@ -40,6 +42,7 @@ fetchPosts({
full: flags.full === "true",
older: flags.older === "true",
floor: flags.floor,
+ force: flags.force === "true",
limit,
onLog: (line) => console.log(line),
})
diff --git a/common/controller/fetchOlderPosts.test.ts b/common/controller/fetchOlderPosts.test.ts
@@ -3,7 +3,9 @@
// the walk steps back a window at a time, writes through the same shard
// writer, records its own position apart from the timeline cursor, stops at a
// floor / the account's creation date / a run of empty windows, and a normal
-// fetch never resumes backwards from it.
+// fetch never resumes backwards from it. An account that shows no posts is
+// refused a walk unless forced, and a forced one ends after two empty
+// windows; a drain stops the walk at a resume point and keeps it.
//
// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test controller/fetchOlderPosts.test.ts
@@ -91,7 +93,13 @@ async function makeChannel(slug: string): Promise<string> {
return channelRoot;
}
-const { fetchPosts, FULL_AND_OLDER_REFUSAL, olderPostsProblem } = await import("./fetchPosts");
+const {
+ fetchPosts,
+ FULL_AND_OLDER_REFUSAL,
+ EMPTY_ACCOUNT_OLDER_REFUSAL,
+ emptyAccountOlderProblem,
+ olderPostsProblem,
+} = await import("./fetchPosts");
const {
readPostFetchState,
readSeenPostIds,
@@ -396,3 +404,191 @@ test("full and older together are refused, and so is a fetcher with no older wal
assert.equal(olderPostsProblem({ label: "Bluesky" }), "Bluesky cannot fetch older posts.");
assert.equal(olderPostsProblem(undefined), "This channel has no post fetcher.");
});
+
+// An account that shows no posts: nothing archived, and the last timeline
+// fetch finished having read none.
+async function emptyAccount(slug: string): Promise<string> {
+ const channelRoot = await makeChannel(slug);
+ await writePostFetchState(channelRoot, {
+ lastFetchedAt: "2026-01-01T00:00:00.000Z",
+ lastFetchedCount: 0,
+ });
+ return channelRoot;
+}
+
+test("an older walk over an account that shows no posts is refused, and searches nothing", async () => {
+ const slug = "older-empty";
+ const channelRoot = await emptyAccount(slug);
+ const before = (await argvLines()).length;
+ const result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ olderWindowPauseMs: 0,
+ });
+ assert.equal(result.ok, false);
+ assert.equal(result.error, EMPTY_ACCOUNT_OLDER_REFUSAL);
+ assert.equal((await argvLines()).length, before);
+ // Nothing is recorded: the refusal is not a walk.
+ assert.equal((await readPostFetchState(channelRoot))?.older, undefined);
+});
+
+test("forced, the walk over an empty account ends after two empty windows, not a year of them", async () => {
+ const slug = "older-empty-forced";
+ const channelRoot = await emptyAccount(slug);
+ const before = (await argvLines()).length;
+ const log: string[] = [];
+ const result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ force: true,
+ olderWindowPauseMs: 0,
+ onLog: (l) => log.push(l),
+ });
+ assert.equal(result.ok, true, log.join("\n"));
+ assert.equal(result.complete, true);
+ assert.equal(result.written, 0);
+ // From today back: two quiet windows, then it stops.
+ assert.equal((await argvLines()).length - before, 2);
+ const older = (await readPostFetchState(channelRoot))?.older;
+ assert.equal(older?.complete, true);
+ assert.equal(older?.emptyWindows, 2);
+ assert.match(older?.completeReason ?? "", /2 windows of 3 months in a row held no posts/);
+ assert.match(older?.completeReason ?? "", /nothing archived and none found/);
+ assert.ok(log.some((l) => /walking anyway, as forced/.test(l)), log.join("\n"));
+ assert.ok(log.some((l) => /backfill complete: .*nothing archived/.test(l)), log.join("\n"));
+});
+
+test("the refusal reads only a clean timeline fetch: a stopped or failed one, or none yet, does not refuse", async () => {
+ const state = { lastFetchedAt: "2026-01-01T00:00:00.000Z", lastFetchedCount: 0 };
+ assert.equal(emptyAccountOlderProblem(0, state), EMPTY_ACCOUNT_OLDER_REFUSAL);
+ assert.equal(emptyAccountOlderProblem(1, state), null);
+ assert.equal(emptyAccountOlderProblem(0, null), null);
+ assert.equal(emptyAccountOlderProblem(0, { ...state, lastFetchedCount: 3 }), null);
+ assert.equal(emptyAccountOlderProblem(0, { ...state, cursor: "1/X" }), null);
+ assert.equal(emptyAccountOlderProblem(0, { ...state, lastError: "boom" }), null);
+ assert.equal(emptyAccountOlderProblem(0, { ...state, needsCookies: true }), null);
+});
+
+test("with nothing archived but posts found this run, the year-of-empties rule still ends the walk", async () => {
+ const slug = "older-found";
+ const channelRoot = await makeChannel(slug);
+ // A walk already under way at the posts, with nothing yet on disk.
+ await writePostFetchState(channelRoot, {
+ older: { since: "2020-12-11", until: "2021-03-11", emptyWindows: 0 },
+ });
+ const before = (await argvLines()).length;
+ const result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ olderWindowPauseMs: 0,
+ });
+ assert.equal(result.ok, true);
+ assert.equal(result.complete, true);
+ // Every post is new, the archived one's included.
+ assert.equal(result.written, 5);
+ assert.equal((await argvLines()).length - before, 10);
+ const older = (await readPostFetchState(channelRoot))?.older;
+ assert.match(older?.completeReason ?? "", /^4 windows of 3 months in a row held no posts, back to [\d-]+$/);
+});
+
+test("a drain mid-window stops the search at once and keeps the resume point inside the window", async () => {
+ const slug = "older-drain-window";
+ const channelRoot = await makeChannel(slug);
+ await seed(channelRoot, slug);
+ const drain = new AbortController();
+ const log: string[] = [];
+ const started = Date.now();
+ process.env.FAKE_MODE = "stall";
+ let result;
+ try {
+ result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ olderWindowPauseMs: 0,
+ drain: drain.signal,
+ onLog: (l) => {
+ log.push(l);
+ if (/rate limit/.test(l)) drain.abort();
+ },
+ });
+ } finally {
+ delete process.env.FAKE_MODE;
+ }
+ assert.equal(result.ok, true, log.join("\n"));
+ assert.equal(result.drained, true);
+ assert.equal(result.complete, false);
+ assert.ok(Date.now() - started < 15_000, `took ${Date.now() - started} ms`);
+ const older = (await readPostFetchState(channelRoot))?.older;
+ assert.deepEqual(
+ { since: older?.since, until: older?.until, maxId: older?.maxId, lastError: older?.lastError },
+ { since: "2020-12-11", until: "2021-03-11", maxId: idOf(POST_DATES[0]), lastError: undefined },
+ );
+ assert.ok(log.some((l) => /Drained; the next run resumes at 2020-12-11 – 2021-03-11 below/.test(l)), log.join("\n"));
+});
+
+test("a drain during the pause between windows ends it at once, at the next window, before another search", async () => {
+ const slug = "older-drain-gap";
+ const channelRoot = await makeChannel(slug);
+ await seed(channelRoot, slug);
+ const before = (await argvLines()).length;
+ const drain = new AbortController();
+ const log: string[] = [];
+ const started = Date.now();
+ const result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ // A gap the test would notice.
+ olderWindowPauseMs: 60_000,
+ drain: drain.signal,
+ onLog: (l) => {
+ log.push(l);
+ // The first window's tally: its boundary is saved next, then the pause.
+ if (/^2020-12-11 – 2021-03-11: read/.test(l)) setTimeout(() => drain.abort(), 500);
+ },
+ });
+ assert.equal(result.ok, true, log.join("\n"));
+ assert.equal(result.drained, true);
+ assert.equal(result.written, 2);
+ assert.ok(Date.now() - started < 15_000, `took ${Date.now() - started} ms`);
+ assert.equal((await argvLines()).length - before, 1);
+ const older = (await readPostFetchState(channelRoot))?.older;
+ assert.deepEqual(
+ { since: older?.since, until: older?.until, maxId: older?.maxId },
+ { since: "2020-09-11", until: "2020-12-11", maxId: undefined },
+ );
+ assert.ok(log.some((l) => /Drained; the next run resumes at 2020-09-11 – 2020-12-11\.$/.test(l)), log.join("\n"));
+});
+
+test("a drain already set when a window ends stops before the pause, never waiting it out", async () => {
+ const slug = "older-drain-boundary";
+ const channelRoot = await makeChannel(slug);
+ await seed(channelRoot, slug);
+ const before = (await argvLines()).length;
+ const drain = new AbortController();
+ const started = Date.now();
+ const result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ olderWindowPauseMs: 60_000,
+ drain: drain.signal,
+ onLog: (l) => {
+ if (/^2020-12-11 – 2021-03-11: read/.test(l)) drain.abort();
+ },
+ });
+ assert.equal(result.drained, true);
+ assert.ok(Date.now() - started < 15_000, `took ${Date.now() - started} ms`);
+ assert.equal((await argvLines()).length - before, 1);
+ assert.equal((await readPostFetchState(channelRoot))?.older?.since, "2020-09-11");
+});
diff --git a/common/controller/fetchPosts.test.ts b/common/controller/fetchPosts.test.ts
@@ -1,6 +1,7 @@
// fetchPosts over the gallery-dl X fetcher, end to end against a fake binary:
// a run that is cancelled (or times out) part-way keeps what it read and the
-// resume point, and the next run continues from there.
+// resume point, and the next run continues from there. A drained run stops at
+// a page boundary, keeps that resume point, and is not a failure.
//
// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test controller/fetchPosts.test.ts
@@ -24,6 +25,8 @@ process.env.FAKE_ARGS_LOG = ARGS_LOG;
// blocks on a "rate limit" (the real stall); given the cursor after page 2 it
// emits page 3 and exits cleanly. FAKE_MODE=known emits page 1 then endless
// tweets older than the watermark, to prove the early stop kills it.
+// FAKE_MODE=pages emits six pages, each followed (after a pause) by an info
+// line, then its cursor: a walk with a page boundary every few hundred ms.
await writeFile(
BIN,
`#!/usr/bin/env node
@@ -59,6 +62,20 @@ if (process.env.FAKE_MODE === "known") {
log("[twitter][debug] Cursor: 1/CRASH-P1");
log("[twitter][error] AbortExtraction: 404 Not Found (GraphQL query id rotated)");
process.exit(4);
+} else if (process.env.FAKE_MODE === "pages") {
+ let p = 0;
+ const next = () => {
+ page(p);
+ setTimeout(() => {
+ log("[twitter][info] page " + p + " read");
+ setTimeout(() => {
+ log("[twitter][debug] Cursor: 1/PAGES-P" + p);
+ p++;
+ if (p < 6) setTimeout(next, 300); else process.exit(0);
+ }, 300);
+ }, 100);
+ };
+ next();
} else if (cursor === "1/AFTER-P2") {
page(2);
process.exit(0);
@@ -197,3 +214,111 @@ test("an auth failure that read nothing keeps the resume point it had, and inven
assert.equal(state?.needsCookies, true);
assert.equal(state?.cursor, undefined);
});
+
+async function makeChannel(slug: string): Promise<string> {
+ const root = path.join(ROOT, "transcripts", "channels", slug);
+ await mkdir(root, { recursive: true });
+ await writeFile(
+ path.join(root, "config.json"),
+ JSON.stringify({
+ handling: "transcribe",
+ sourceKind: "social",
+ postFetcher: "x-gallery-dl",
+ socialHandle: "faketester",
+ platform: "twitter",
+ name: "Fake (X)",
+ url: "https://x.com/faketester",
+ }),
+ );
+ return root;
+}
+
+test("a drain mid-page waits for the page's cursor, stops there and keeps it; the run is not a failure", async () => {
+ const slug = "fake-x-drain-page";
+ const root = await makeChannel(slug);
+ const drain = new AbortController();
+ const log: string[] = [];
+ process.env.FAKE_MODE = "pages";
+ let result;
+ try {
+ result = await fetchPosts({
+ paths: (await import("../lib/paths")).getPaths(),
+ slug,
+ settings: {},
+ drain: drain.signal,
+ onLog: (line) => {
+ log.push(line);
+ // Page 1's records are in; its cursor is not yet.
+ if (/page 1 read/.test(line)) drain.abort();
+ },
+ });
+ } finally {
+ delete process.env.FAKE_MODE;
+ }
+ assert.equal(result.ok, true, log.join("\n"));
+ assert.equal(result.drained, true);
+ assert.equal(result.complete, false);
+ // Pages 0 and 1, and not one more.
+ assert.equal(result.written, 6);
+ assert.equal((await readSeenPostIds(root)).size, 6);
+ const state = await readPostFetchState(root);
+ assert.equal(state?.cursor, "1/PAGES-P1");
+ assert.equal(state?.lastError, undefined);
+ assert.equal(log.some((l) => /page 2 read/.test(l)), false, log.join("\n"));
+ assert.ok(log.some((l) => /Drained; the next run resumes from the last page read/.test(l)), log.join("\n"));
+
+ // The next run resumes there (cancelled once it has started: only its argv
+ // is wanted).
+ const cancel = new AbortController();
+ await fetchPosts({
+ paths: (await import("../lib/paths")).getPaths(),
+ slug,
+ settings: {},
+ signal: cancel.signal,
+ onLog: (line) => {
+ if (/^Running /.test(line)) setTimeout(() => cancel.abort(), 200);
+ },
+ });
+ assert.ok((await lastArgv()).includes("extractor.twitter.cursor=1/PAGES-P1"));
+});
+
+test("a drain while gallery-dl waits out a rate limit stops it at once, keeping the last page's cursor", async () => {
+ const slug = "fake-x-drain-wait";
+ const root = await makeChannel(slug);
+ const drain = new AbortController();
+ const log: string[] = [];
+ const started = Date.now();
+ const result = await fetchPosts({
+ paths: (await import("../lib/paths")).getPaths(),
+ slug,
+ settings: {},
+ drain: drain.signal,
+ onLog: (line) => {
+ log.push(line);
+ if (/rate limit/.test(line)) drain.abort();
+ },
+ });
+ assert.equal(result.ok, true, log.join("\n"));
+ assert.equal(result.drained, true);
+ assert.equal(result.written, 6);
+ // Not left asleep for its 60 s.
+ assert.ok(Date.now() - started < 15_000, `took ${Date.now() - started} ms`);
+ assert.equal((await readPostFetchState(root))?.cursor, "1/AFTER-P2");
+});
+
+test("a drain before the fetch starts reads nothing and spawns nothing", async () => {
+ const slug = "fake-x-drain-early";
+ const root = await makeChannel(slug);
+ const drain = new AbortController();
+ drain.abort();
+ const before = (await readFile(ARGS_LOG, "utf8")).trim().split("\n").length;
+ const result = await fetchPosts({
+ paths: (await import("../lib/paths")).getPaths(),
+ slug,
+ settings: {},
+ drain: drain.signal,
+ });
+ assert.deepEqual(result, { ok: true, written: 0, skipped: 0, complete: false, drained: true });
+ assert.equal((await readFile(ARGS_LOG, "utf8")).trim().split("\n").length, before);
+ assert.equal(await readPostFetchState(root), null);
+});
diff --git a/common/controller/fetchPosts.ts b/common/controller/fetchPosts.ts
@@ -78,9 +78,15 @@ export type FetchPostsOptions = {
floor?: string;
// The older walk's pause between two windows. Default: the fetcher's own.
olderWindowPauseMs?: number;
+ // Walk older posts even on an account that shows none (see
+ // emptyAccountOlderProblem). Only with `older`.
+ force?: boolean;
limit?: number;
onLog?: (line: string) => void;
signal?: AbortSignal;
+ // The job's Drain: the fetch stops at its next resume point and keeps it,
+ // and the run ends as a success, not a failure.
+ drain?: AbortSignal;
};
// The refusal for both walks at once, shared with the server action so the
@@ -98,6 +104,25 @@ export function olderPostsProblem(
: `${fetcher.label} cannot fetch older posts.`;
}
+// An older walk over an account that shows no posts is refused unless forced:
+// nothing is archived, and the last timeline fetch finished cleanly having read
+// nothing. Searching such an account window after window is nothing but empty
+// searches. A timeline run that stopped part-way, failed or needed a login
+// says nothing about the account, so it is not grounds to refuse.
+export const EMPTY_ACCOUNT_OLDER_REFUSAL =
+ "The account shows no posts: nothing is archived and the last timeline fetch read none. " +
+ 'A search walk would only repeat empty searches — run it with "force" to walk anyway.';
+
+export function emptyAccountOlderProblem(
+ archived: number,
+ state: Pick<PostFetchState, "lastFetchedCount" | "cursor" | "lastError" | "needsCookies"> | null,
+): string | null {
+ if (archived > 0 || !state) return null;
+ if (state.lastFetchedCount !== 0) return null;
+ if (state.cursor || state.lastError || state.needsCookies) return null;
+ return EMPTY_ACCOUNT_OLDER_REFUSAL;
+}
+
export type FetchPostsResult = {
ok: boolean;
written: number;
@@ -105,6 +130,8 @@ export type FetchPostsResult = {
complete: boolean;
error?: string;
needsCookies?: boolean;
+ // Stopped on the job's Drain, at a resume point the next run continues from.
+ drained?: boolean;
};
export async function fetchPosts(
@@ -165,6 +192,11 @@ export async function fetchPosts(
};
}
+ if (opts.drain?.aborted) {
+ log("Drained before the fetch started; nothing was read.");
+ return { ok: true, written: 0, skipped: 0, complete: false, drained: true };
+ }
+
const seenIds = await readSeenPostIds(channelRoot);
const priorState = await readPostFetchState(channelRoot);
@@ -270,6 +302,7 @@ export async function fetchPosts(
stopAtKnown: !opts.full,
onCheckpoint,
signal: effectiveSignal,
+ drain: opts.drain,
onLog: log,
});
} catch (err) {
@@ -332,6 +365,7 @@ export async function fetchPosts(
complete: result.complete,
needsCookies: result.needsCookies,
error: result.error,
+ ...(result.drained ? { drained: true } : {}),
};
}
@@ -368,6 +402,13 @@ async function fetchOlderPosts(ctx: {
return { ok: true, written: 0, skipped: 0, complete: true };
}
+ const empty = emptyAccountOlderProblem(ctx.seenIds.size, priorState);
+ if (empty && !opts.force) {
+ log(empty);
+ return { ok: false, written: 0, skipped: 0, complete: false, error: empty };
+ }
+ if (empty) log("[warn] The account shows no posts; walking anyway, as forced.");
+
const floor = opts.floor ?? prior?.floor;
let accountCreatedAt = prior?.accountCreatedAt;
const startedAt = new Date().toISOString();
@@ -449,6 +490,7 @@ async function fetchOlderPosts(ctx: {
browserCookies: ctx.browserCookies,
limit: opts.limit,
signal: opts.signal ?? controller.signal,
+ drain: opts.drain,
onLog: log,
position,
floor,
@@ -498,5 +540,6 @@ async function fetchOlderPosts(ctx: {
complete: result.complete,
needsCookies: result.needsCookies,
error: result.error,
+ ...(result.drained ? { drained: true } : {}),
};
}
diff --git a/common/social/fetchers.ts b/common/social/fetchers.ts
@@ -63,6 +63,10 @@ export type PostFetchInput = {
// again in the result. A fetcher that pages in one shot never calls it.
onCheckpoint?: (checkpoint: { posts: Post[]; cursor: string }) => Promise<void>;
signal: AbortSignal;
+ // A soft stop (the job's Drain): the run ends at its next resume point —
+ // between two pages — keeping that point, rather than walking on. `signal`
+ // cancels outright. A fetcher that pages in one shot may ignore it.
+ drain?: AbortSignal;
// Progress/diagnostic sink, wired to the job log.
onLog?: (line: string) => void;
};
@@ -80,6 +84,8 @@ export type PostFetchResult = {
// The run failed part-way. `posts` and `cursor` still hold what it got, so
// the controller saves them before reporting the failure.
error?: string;
+ // The run stopped on `drain` at a resume point (`cursor`). Not a failure.
+ drained?: boolean;
};
// Input for the older-posts backfill (`SocialFetcher.fetchOlder`): walk the
@@ -97,6 +103,7 @@ export type OlderPostFetchInput = Pick<
| "browserCookies"
| "limit"
| "signal"
+ | "drain"
| "onLog"
> & {
// Where to start: the stored position of an unfinished walk, or the first
@@ -128,6 +135,8 @@ export type OlderPostFetchResult = {
accountCreatedAt?: string;
needsCookies?: boolean;
error?: string;
+ // The walk stopped on `drain`; `position` is where the next run resumes.
+ drained?: boolean;
};
export type SocialFetcherProbe = {
diff --git a/common/social/olderBackfill.test.ts b/common/social/olderBackfill.test.ts
@@ -67,6 +67,23 @@ test("windows step back with no gap, count empty ones, and end after a run of th
]);
});
+test("nothing archived and nothing found: the walk ends after two empty windows, and says why", () => {
+ const first = stepOlderWindow(
+ { since: "2026-07-05", until: "2026-10-05", emptyWindows: 0 },
+ { hadPosts: false, nothingFound: true },
+ );
+ assert.equal(first.complete, false);
+ if (first.complete) return;
+ const second = stepOlderWindow(first.next, { hadPosts: false, nothingFound: true });
+ assert.equal(second.complete, true);
+ if (!second.complete) return;
+ assert.equal(second.emptyWindows, 2);
+ assert.match(second.reason, /^2 windows of 3 months in a row held no posts, back to 2026-04-05, with nothing archived and none found/);
+ // Without it, the same two windows are not the end: a year of them is.
+ const year = stepOlderWindow(first.next, { hadPosts: false });
+ assert.equal(year.complete, false);
+});
+
test("a window with posts resets the empty count", () => {
const step = stepOlderWindow(
{ since: "2020-01-01", until: "2020-04-01", emptyWindows: 3 },
diff --git a/common/social/olderBackfill.ts b/common/social/olderBackfill.ts
@@ -21,6 +21,11 @@ export const OLDER_WINDOW_MONTHS = 3;
// three-month windows, a year with nothing.
export const OLDER_MAX_EMPTY_WINDOWS = 4;
+// The same, for a walk over an account with nothing archived that has found
+// nothing in this run either: two windows. Searching an empty account window
+// after window only repeats empty searches.
+export const OLDER_MAX_EMPTY_WINDOWS_NOTHING_ARCHIVED = 2;
+
const DAY_RE = /^\d{4}-\d{2}-\d{2}$/;
// A real YYYY-MM-DD calendar day (2021-02-30 is not one).
@@ -102,7 +107,9 @@ export type OlderWindowStep =
// What follows a window walked to its end. `hadPosts` counts any post the
// window held, archived already or not — a window full of known posts is not
-// an empty one.
+// an empty one. `nothingFound`: nothing is archived and this run has read no
+// post — the walk ends after OLDER_MAX_EMPTY_WINDOWS_NOTHING_ARCHIVED empty
+// windows, not a year of them.
export function stepOlderWindow(
position: OlderBackfillPosition,
opts: {
@@ -110,10 +117,13 @@ export function stepOlderWindow(
floor?: string;
months?: number;
maxEmptyWindows?: number;
+ nothingFound?: boolean;
},
): OlderWindowStep {
const months = opts.months ?? OLDER_WINDOW_MONTHS;
- const maxEmpty = opts.maxEmptyWindows ?? OLDER_MAX_EMPTY_WINDOWS;
+ const maxEmpty =
+ opts.maxEmptyWindows ??
+ (opts.nothingFound ? OLDER_MAX_EMPTY_WINDOWS_NOTHING_ARCHIVED : OLDER_MAX_EMPTY_WINDOWS);
const emptyWindows = opts.hadPosts ? 0 : position.emptyWindows + 1;
if (isAtFloor(position, opts.floor)) {
return {
@@ -125,7 +135,11 @@ export function stepOlderWindow(
if (emptyWindows >= maxEmpty) {
return {
complete: true,
- reason: `${emptyWindows} windows of ${months} months in a row held no posts, back to ${position.since}`,
+ reason:
+ `${emptyWindows} windows of ${months} months in a row held no posts, back to ${position.since}` +
+ (opts.nothingFound
+ ? ", with nothing archived and none found — an empty account is not searched further"
+ : ""),
emptyWindows,
};
}
diff --git a/common/social/xGalleryDlFetcher.test.ts b/common/social/xGalleryDlFetcher.test.ts
@@ -176,10 +176,23 @@ test("stream: a checkpoint carries the PREVIOUS cursor; the final cursor is the
test("stream: debug noise is dropped; info lines are logged and rate limits noticed", () => {
const logged: string[] = [];
const s = newStream({ onLog: (l) => logged.push(l) });
- s.pushStderr("[urllib3.connectionpool][debug] https://x.com:443 \"GET /i/api HTTP/1.1\" 200");
- s.pushStderr("[twitter][info] Waiting for 14 minutes until 15:11:22 (rate limit)");
+ assert.equal(
+ s.pushStderr("[urllib3.connectionpool][debug] https://x.com:443 \"GET /i/api HTTP/1.1\" 200"),
+ undefined,
+ );
+ // A wait on the rate limit, and a page's cursor, are each a point between
+ // pages — where a drain may stop the run.
+ assert.equal(
+ s.pushStderr("[twitter][info] Waiting for 14 minutes until 15:11:22 (rate limit)"),
+ "rate-limit",
+ );
+ assert.equal(s.pushStderr("[twitter][debug] Cursor: 1/ABC"), "cursor");
+ assert.equal(s.pushStderr("[twitter][info] something else"), undefined);
assert.equal(s.rateLimited, true);
- assert.deepEqual(logged, ["gallery-dl: [twitter][info] Waiting for 14 minutes until 15:11:22 (rate limit)"]);
+ assert.deepEqual(logged, [
+ "gallery-dl: [twitter][info] Waiting for 14 minutes until 15:11:22 (rate limit)",
+ "gallery-dl: [twitter][info] something else",
+ ]);
assert.match(s.stderrTail(), /rate limit/);
});
diff --git a/common/social/xGalleryDlFetcher.ts b/common/social/xGalleryDlFetcher.ts
@@ -574,18 +574,22 @@ export class GalleryDlStream {
for (const rec of records) this.addRecord(rec);
}
- pushStderr(line: string): void {
+ // Says what the line was when it marks a point between two pages: a page's
+ // cursor, or a wait on X's rate limit.
+ pushStderr(line: string): "cursor" | "rate-limit" | undefined {
const cursor = CURSOR_LINE.exec(line);
if (cursor) {
this.previousCursor = this.cursor;
this.cursor = cursor[1];
- return;
+ return "cursor";
}
- if (DEBUG_LINE.test(line) || !line.trim()) return;
- if (RATE_LIMIT_LINE.test(line)) this.rateLimited = true;
+ if (DEBUG_LINE.test(line) || !line.trim()) return undefined;
+ const waiting = RATE_LIMIT_LINE.test(line);
+ if (waiting) this.rateLimited = true;
this.tail.push(line);
if (this.tail.length > STDERR_TAIL_LINES) this.tail.shift();
this.opts.onLog?.(`gallery-dl: ${line.trim()}`);
+ return waiting ? "rate-limit" : undefined;
}
// The last few non-debug stderr lines, for an error message.
@@ -854,6 +858,12 @@ export async function fetchOlderViaSearch(
// A fixed pause when the caller names one (tests pass 0); otherwise a fresh
// random draw before every window.
const pauseFor = () => input.windowPauseMs ?? olderWindowPauseMs();
+ const { drain } = input;
+ // A drain ends the pause between windows as a cancel does.
+ const pauseSignal = drain ? AbortSignal.any([signal, drain]) : signal;
+ // Nothing archived: a walk that finds nothing either ends sooner
+ // (stepOlderWindow's `nothingFound`).
+ const nothingArchived = seenIds.size === 0;
let position: OlderBackfillPosition = { ...input.position };
let accountCreatedAt = input.accountCreatedAt;
// Posts not handed to a checkpoint, returned for the final write.
@@ -886,6 +896,13 @@ export async function fetchOlderViaSearch(
accountCreatedAt,
...extra,
});
+ const drainedAt = (at: OlderBackfillPosition): OlderPostFetchResult => {
+ onLog?.(
+ `Drained; the next run resumes at ${at.since} – ${at.until}` +
+ (at.maxId ? " below the oldest post read." : "."),
+ );
+ return stopped(at, { drained: true });
+ };
onLog?.(
`Searching @${handle}'s posts backwards, ${position.since} – ${position.until}` +
@@ -913,6 +930,7 @@ export async function fetchOlderViaSearch(
if (remainingLimit !== undefined && remainingLimit <= 0) {
return stopped(position);
}
+ if (drain?.aborted) return drainedAt(position);
const windowStart = position;
const stream = new GalleryDlStream({
@@ -937,6 +955,17 @@ export async function fetchOlderViaSearch(
cancelSignal: signal,
env: { PYTHONUNBUFFERED: "1" },
});
+ // DRAIN mid-window stops the search at once: every post read is its own
+ // resume point (the oldest one read becomes the window's max_id), so there
+ // is no page boundary to wait for. SIGTERM, then execa's SIGKILL if
+ // gallery-dl has not exited a few seconds later.
+ let drainedHere = false;
+ const onDrain = () => {
+ drainedHere = true;
+ onLog?.("Drain requested: stopping this window's search.");
+ subprocess.kill("SIGTERM");
+ };
+ drain?.addEventListener("abort", onDrain, { once: true });
// Mid-window checkpoints, one at a time and in order. The resume point is
// the oldest post read so far, which every post taken here is at or above.
@@ -963,6 +992,7 @@ export async function fetchOlderViaSearch(
maybeCheckpoint();
});
const res = await subprocess;
+ drain?.removeEventListener("abort", onDrain);
await Promise.all([stdoutDone, stderrDone]);
await checkpoints;
@@ -996,6 +1026,7 @@ export async function fetchOlderViaSearch(
);
return stopped(here);
}
+ if (drainedHere) return drainedAt(here);
if (res.exitCode !== 0) {
const tail = stream.stderrTail();
if (looksLikeAuthFailure(tail)) {
@@ -1015,6 +1046,7 @@ export async function fetchOlderViaSearch(
// A window resumed below a saved point held posts above it.
hadPosts: stream.distinct > 0 || Boolean(windowStart.maxId),
floor: olderFloorDay(input.floor, accountCreatedAt),
+ nothingFound: nothingArchived && read === 0,
});
if (step.complete) {
onLog?.(`Older-posts backfill complete: ${step.reason}.`);
@@ -1033,11 +1065,14 @@ export async function fetchOlderViaSearch(
onLog?.("Cancelled; the next run resumes where this one left off.");
return stopped(position);
}
- await pause(pauseFor(), signal);
+ // Before the gap, not after it: a drain never waits out a pause.
+ if (drain?.aborted) return drainedAt(position);
+ await pause(pauseFor(), pauseSignal);
if (signal.aborted) {
onLog?.("Cancelled; the next run resumes where this one left off.");
return stopped(position);
}
+ if (drain?.aborted) return drainedAt(position);
}
}
@@ -1182,8 +1217,28 @@ export const xGalleryDlFetcher: SocialFetcher = {
});
};
+ // DRAIN stops gallery-dl at a page boundary — the "Cursor:" line it logs
+ // after each page, which is a resume point — so the next run does not
+ // repeat a page's request. Between pages already (a cursor just logged,
+ // or a rate-limit wait), it stops at once. SIGTERM, then execa's SIGKILL
+ // if gallery-dl has not exited a few seconds later; the resume point is
+ // the latest cursor, safe once both pipes have drained.
let stoppedAtKnown = false;
+ let drained = false;
+ let betweenPages = false;
+ const stopForDrain = () => {
+ if (drained || stoppedAtKnown) return;
+ drained = true;
+ onLog?.("Drain requested: stopping gallery-dl at this page boundary.");
+ subprocess.kill("SIGTERM");
+ };
+ const onDrain = () => {
+ if (betweenPages) stopForDrain();
+ };
+ input.drain?.addEventListener("abort", onDrain, { once: true });
+
const stdoutDone = readLines(subprocess.stdout, (line) => {
+ if (line.trim()) betweenPages = false;
stream.pushStdout(line);
if (stream.stopReached && !stoppedAtKnown) {
stoppedAtKnown = true;
@@ -1194,11 +1249,15 @@ export const xGalleryDlFetcher: SocialFetcher = {
}
});
const stderrDone = readLines(subprocess.stderr, (line) => {
- stream.pushStderr(line);
+ if (stream.pushStderr(line)) {
+ betweenPages = true;
+ if (input.drain?.aborted) stopForDrain();
+ }
maybeCheckpoint();
});
const res = await subprocess;
+ input.drain?.removeEventListener("abort", onDrain);
await Promise.all([stdoutDone, stderrDone]);
await checkpoints;
@@ -1246,6 +1305,15 @@ export const xGalleryDlFetcher: SocialFetcher = {
return { posts, complete: false, cursor: resumeAt };
}
+ if (drained) {
+ onLog?.(
+ resumeAt
+ ? "Drained; the next run resumes from the last page read."
+ : "Drained before the first page; the next run starts where this one did.",
+ );
+ return { posts, complete: false, cursor: resumeAt, drained: true };
+ }
+
if (res.exitCode !== 0) {
const tail = stream.stderrTail();
if (looksLikeAuthFailure(tail)) {
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,7 @@
# Changelog
## [Unreleased]
+- **X post fetches stop on Drain, and an account with no posts is not searched.** Draining a `fetch-posts` job used to do nothing until gallery-dl finished its whole run. Now the timeline fetch stops at the next page boundary (at once when gallery-dl is between pages or waiting out a rate limit), the older-posts walk stops its current window's search at once and never starts the 45–120 second pause between windows, and both keep their resume point: the job ends done, not failed, and the log says "Drained; the next run resumes …". An older-posts walk is refused when nothing is archived and the last timeline fetch finished having read no posts, since it would only repeat empty searches; `"force": true` (`--force` on `archilyzer posts fetch`) walks anyway. A walk with nothing archived that finds nothing ends after two empty three-month windows instead of four, and records why; a walk that has posts keeps the year-of-empty-windows rule. Capture-posts already stopped between posts on Drain.
- **Capturing an X post that is an Article also saves the article.** An X Article (a long-form post) is archived as nothing but its link, and gallery-dl cannot read its body. When `pnpm ops capture-posts` meets a post whose archived text, or whose card on the page, links to an article, it now opens the article in the same X profile, after the same 4–10 second pause, and saves beside the post's capture: `article.json` (the title, author, date and every heading, paragraph, quote, list item, image, link and embedded post in reading order, an embedded post by its URL), `article.md` (the same as readable text), `article.png` (the whole article as shown, cut off at 16,000 pixels tall and marked `trimmed` when longer), `article.html` (the article as X served it, so it can be read again without going back to X) and the article's pictures as `article-img-1.jpg`, `article-img-2.png`, … at full size, fetched through the same browser session. `capture.json` records the article's state, title, block count and every file's size and SHA-256. `"articles": false` leaves articles alone; `"shots": false, "media": false, "articles": true` reads only the articles. An article already captured, deleted or unavailable is not opened again unless `"force": true`; one that failed is tried on the next run. If X asks to log in on the article page, the job stops there as it does for a post. The post viewer's capture panel in the editor shows the article's title with a link to `article.md`.
- **A report video can play a clip that has only sound, and a clip can be a file beside the manifest.** When a clip's source has no picture, `build-video.mjs` plays it under a poster: a card with the clip's channel, title and date, the size of the picture area, with the sound's waveform moving along its foot (`render.audioPoster.waveform: false` keeps it still). The segment matches every other one in size, frame rate and sound, and the header, footer and on-screen deck are drawn over it as over footage. A video's saved sound (`audio.mp3` and the like in its folder) is now a source the build can cut from, after every saved picture: before any download when the clip has no picture to fetch (`"audioOnly": true` on the clip, `"preferLocalAudio": true` in `render`, a podcast or feed record, or a record with no page). A clip that should have a picture is not quietly played from its sound: with `--no-network` or `--skip-fetch`, one whose picture is missing still stops the build, listed as needing a download with a note that its sound is on disk, so `--no-network` still proves every picture is there. Add `--audio-fallback` to play such clips from their sound under the poster instead; they are listed apart (and logged as `audio-fallback`), and clips that have no picture to fetch are listed as playing from audio only rather than refused. A clip may also give `"src"` (a video or audio file) and `"cues"` (its transcript, either a `transcript.cues.json` or a `parakeet-stitch` transcript), both relative to the manifest, instead of a channel and video: it plays the whole file unless `start`/`end` cut inside it, `resolve-windows.mjs` widens it with those cues, it gets no QR unless it has a `citeUrl`, and a path that leaves the manifest's folder (or an absolute one, without `"allowAbsoluteSrc": true` in `render`), a missing file or an unreadable transcript stops the build before anything runs, naming the clip.
- **A report build cuts from media already on disk before it downloads anything, and `--no-network` makes sure it never does.** For each clip, `build-video.mjs` now looks, in order, in the project's own `out/clips-raw`, in the clip windows the editor fetched into the channel (`channels/<slug>/data/<id>/clips/`), and in a saved whole source video (through the saved-video store's pointer, or a `source-media` file still in the video's folder), and cuts from the first that holds the clip plus its fetch pad; only when none does is the window downloaded. A file that is a link to a drive that is not mounted counts as not there, and the next place is tried. The build prints one line per clip naming where its source came from (`raw-cache`, `corpus-window`, `saved-video`, or a network fetch). With `--no-network`, every clip's source is found before anything is rendered, and if any clip would need a download the build stops at once and lists each one (its position in the timeline, channel, video and the span it needs). umtool's clip bench reads the same three places, so a clip it shows as fetched is one the build cuts from without downloading.
diff --git a/editor/app/api/ops/fetch-posts/route.test.ts b/editor/app/api/ops/fetch-posts/route.test.ts
@@ -0,0 +1,75 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+// Run with:
+// pnpm -C editor exec tsx --test "app/api/ops/fetch-posts/route.test.ts"
+//
+// The body's shape and the refusals that come before any job: every case here
+// is answered from the disk alone, so a temp corpus with one X channel is the
+// whole world — no job is queued and nothing reaches X.
+
+const ROOT = await mkdtemp(path.join(os.tmpdir(), "fetch-posts-route-"));
+const SLUG = "demo-x";
+const CHANNEL = path.join(ROOT, "channels", SLUG);
+// Set before the route (and getPaths, which caches) is first imported.
+process.env.WORKER_TOKEN = "test-token";
+process.env.TRANSCRIPTS_DIR = ROOT;
+process.env.SETTINGS_FILE = path.join(ROOT, "settings.json");
+await mkdir(CHANNEL, { recursive: true });
+await writeFile(
+ path.join(CHANNEL, "config.json"),
+ JSON.stringify({
+ handling: "transcribe",
+ sourceKind: "social",
+ platform: "twitter",
+ postFetcher: "x-gallery-dl",
+ socialHandle: "example_user",
+ name: "Example (X)",
+ url: "https://x.com/example_user",
+ }),
+);
+// An account that shows no posts: nothing archived, and the last timeline
+// fetch finished having read none.
+await writeFile(
+ path.join(CHANNEL, "posts-state.json"),
+ JSON.stringify({ lastFetchedAt: "2026-01-01T00:00:00.000Z", lastFetchedCount: 0 }),
+);
+const { POST } = await import("./route");
+const { EMPTY_ACCOUNT_OLDER_REFUSAL } = await import(
+ "yt-dlp-transcript-common/controller/fetchPosts"
+);
+test.after(() => rm(ROOT, { recursive: true, force: true }));
+
+async function post(body: Record<string, unknown>): Promise<{ status: number; error: string }> {
+ const res = await POST(
+ new Request("http://localhost/api/ops/fetch-posts", {
+ method: "POST",
+ headers: {
+ authorization: "Bearer test-token",
+ "content-type": "application/json",
+ },
+ body: JSON.stringify(body),
+ }),
+ );
+ return { status: res.status, error: ((await res.json()) as { error?: string }).error ?? "" };
+}
+
+test("force must be a boolean, and applies only to an older walk", async () => {
+ const bad = await post({ slug: SLUG, older: true, force: "yes" });
+ assert.equal(bad.status, 400);
+ assert.match(bad.error, /"force" must be a boolean/);
+ const alone = await post({ slug: SLUG, force: true });
+ assert.equal(alone.status, 400);
+ assert.equal(alone.error, '"force" applies only to an older-posts fetch.');
+});
+
+test("an older walk over an account that shows no posts is refused before any job", async () => {
+ const res = await post({ slug: SLUG, older: true });
+ assert.equal(res.status, 400);
+ assert.equal(res.error, EMPTY_ACCOUNT_OLDER_REFUSAL);
+ // No job was ever written.
+ assert.deepEqual(await readdir(path.join(ROOT, ".jobs")).catch(() => []), []);
+});
diff --git a/editor/app/api/ops/fetch-posts/route.ts b/editor/app/api/ops/fetch-posts/route.ts
@@ -10,7 +10,7 @@ import {
export const dynamic = "force-dynamic";
-// POST { slug, full?, older?, floor?, limit?, queueKey? } -> { ok: true, jobId }
+// POST { slug, full?, older?, floor?, force?, limit?, queueKey? } -> { ok: true, jobId }
//
// A social channel's "Fetch posts" button, over HTTP — and its "Re-fetch full
// history" (`full`) and "Fetch older posts" (`older`, with an optional `floor`
@@ -19,11 +19,13 @@ export const dynamic = "force-dynamic";
//
// Every refusal is the action's own sentence: a channel that is not a social
// one, `full` and `older` together, a fetcher with no older walk, a floor
-// without `older` or that is not a date.
+// without `older` or that is not a date, `force` without `older`, and an older
+// walk over an account that shows no posts — nothing archived, and the last
+// timeline fetch read none — unless `force` is true.
export async function POST(request: Request) {
return ops(
request,
- ["slug", "full", "older", "floor", "limit", "queueKey"],
+ ["slug", "full", "older", "floor", "force", "limit", "queueKey"],
async (body) => {
const slug = reqSlug(body, "slug");
return jobResponse(
@@ -34,6 +36,7 @@ export async function POST(request: Request) {
optPositiveInt(body, "limit"),
optBool(body, "older"),
optString(body, "floor"),
+ optBool(body, "force"),
),
);
},
diff --git a/editor/app/channels/[slug]/socialActions.ts b/editor/app/channels/[slug]/socialActions.ts
@@ -21,6 +21,7 @@ import {
} from "yt-dlp-transcript-common/controller/channels";
import { isSocialChannel } from "yt-dlp-transcript-common/lib/channelConfig";
import {
+ emptyAccountOlderProblem,
fetchPosts,
FULL_AND_OLDER_REFUSAL,
olderPostsProblem,
@@ -38,7 +39,10 @@ import {
strayCaptureIds,
strayIdsRefusal,
} from "yt-dlp-transcript-common/controller/capturePosts";
-import { readSeenPostIds } from "yt-dlp-transcript-common/lib/posts-server";
+import {
+ readPostFetchState,
+ readSeenPostIds,
+} from "yt-dlp-transcript-common/lib/posts-server";
import {
getSocialFetcher,
listSocialFetchers,
@@ -141,7 +145,8 @@ export async function checkPostAvailabilityAction(
// (the fetcher's `fetchOlder` — X: search windows) instead of fetching new
// posts; `floor` (YYYY-MM-DD) is the date that walk stops at. Both are refused
// HERE, before a job exists, when they cannot run: with `full`, on a fetcher
-// that has no older walk, or with a floor that is not a date.
+// that has no older walk, with a floor that is not a date, or on an account
+// that shows no posts (emptyAccountOlderProblem) unless `force`.
export async function fetchPostsAction(
slug: string,
queueKey?: string,
@@ -149,11 +154,15 @@ export async function fetchPostsAction(
limit?: number,
older?: boolean,
floor?: string,
+ force?: boolean,
): Promise<StreamActionResult> {
if (full && older) return { ok: false, error: FULL_AND_OLDER_REFUSAL };
if (floor !== undefined && !older) {
return { ok: false, error: "A floor date applies only to an older-posts fetch." };
}
+ if (force && !older) {
+ return { ok: false, error: "\"force\" applies only to an older-posts fetch." };
+ }
if (floor !== undefined && !isUtcDay(floor)) {
return { ok: false, error: `"${floor}" is not a date (YYYY-MM-DD).` };
}
@@ -169,6 +178,14 @@ export async function fetchPostsAction(
resolveSocialFetcher(config.postFetcher, config.url),
);
if (problem) return { ok: false, error: problem };
+ if (!force) {
+ const channelRoot = path.join(paths.channelsDir, slug);
+ const empty = emptyAccountOlderProblem(
+ (await readSeenPostIds(channelRoot)).size,
+ await readPostFetchState(channelRoot),
+ );
+ if (empty) return { ok: false, error: empty };
+ }
}
// queueKeyForUrl() already routes x.com / bsky.app to platform:x.com /
@@ -184,9 +201,9 @@ export async function fetchPostsAction(
spec: {
kind: "fetch-posts",
slug,
- params: { queueKey, full, limit, older, floor },
+ params: { queueKey, full, limit, older, floor, force },
},
- fn: async (onLog, signal) => {
+ fn: async (onLog, signal, _progress, ctx) => {
const result = await fetchPosts({
paths,
slug,
@@ -194,9 +211,13 @@ export async function fetchPostsAction(
full,
older,
floor,
+ force,
limit,
onLog,
signal,
+ // A drained fetch keeps its resume point and returns ok: the job
+ // ends done, not failed.
+ drain: ctx.drainSignal,
});
safeRevalidate([`/channels/${slug}`]);
// Surface a failed fetch as a failed JOB (the managed wrapper turns a
diff --git a/editor/app/jobs/jobReplayRegistry.ts b/editor/app/jobs/jobReplayRegistry.ts
@@ -260,6 +260,7 @@ export const JOB_REPLAY_HANDLERS: Record<string, ReplayHandler> = {
num(p.limit),
bool(p.older),
str(p.floor),
+ bool(p.force),
);
},
// The ids are the spec's own (a capture is OF specific posts, unlike a
diff --git a/scripts/archilyzer-ops.mjs b/scripts/archilyzer-ops.mjs
@@ -330,7 +330,10 @@ export function usage() {
' re-walks the whole timeline; "older": true walks back from the oldest',
" archived post through search (X; needs a login), saving its place for",
' the next run, down to "floor": "YYYY-MM-DD" when given. "limit": N caps',
- ' the posts one run reads. "full" and "older" together are refused.',
+ ' the posts one run reads. "full" and "older" together are refused. An',
+ ' older walk over an account that shows no posts (nothing archived, and',
+ ' the last timeline fetch read none) is refused unless "force": true. A',
+ " drained fetch stops at its next resume point and the next run resumes.",
"",
'capture-posts captures archived posts of a social channel (X): a',
' screenshot of each through the connected X profile, and its attached',
diff --git a/scripts/archilyzer-ops.test.mjs b/scripts/archilyzer-ops.test.mjs
@@ -389,6 +389,7 @@ test("fetch-posts is a POST to its route, named in the usage", () => {
assert.equal(p.wait, true);
assert.match(usage(), /Actions:.*transcribe-bucket, fetch-posts/);
assert.match(usage(), /"older": true walks back from the oldest/);
+ assert.match(usage(), /read none\) is refused unless "force": true/);
});
// A post capture: a POST to its route, the body passed through untouched — the
