commit ea1e02ae3fafc5f997645efc31c739179abb87fc
parent b33f845548f1e021de48bcb617b9b8c399924034
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sun, 4 Oct 2026 22:07:41 -0400
common: tests for the empty-account refusal (+force), the two-empty-window stop, the year rule with posts found, and drain mid-page, mid-wait, mid-window and in the gap
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
4 files changed, 357 insertions(+), 6 deletions(-)
diff --git a/common/controller/fetchOlderPosts.test.ts b/common/controller/fetchOlderPosts.test.ts
@@ -3,7 +3,9 @@
// the walk steps back a window at a time, writes through the same shard
// writer, records its own position apart from the timeline cursor, stops at a
// floor / the account's creation date / a run of empty windows, and a normal
-// fetch never resumes backwards from it.
+// fetch never resumes backwards from it. An account that shows no posts is
+// refused a walk unless forced, and a forced one ends after two empty
+// windows; a drain stops the walk at a resume point and keeps it.
//
// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test controller/fetchOlderPosts.test.ts
@@ -91,7 +93,13 @@ async function makeChannel(slug: string): Promise<string> {
return channelRoot;
}
-const { fetchPosts, FULL_AND_OLDER_REFUSAL, olderPostsProblem } = await import("./fetchPosts");
+const {
+ fetchPosts,
+ FULL_AND_OLDER_REFUSAL,
+ EMPTY_ACCOUNT_OLDER_REFUSAL,
+ emptyAccountOlderProblem,
+ olderPostsProblem,
+} = await import("./fetchPosts");
const {
readPostFetchState,
readSeenPostIds,
@@ -396,3 +404,191 @@ test("full and older together are refused, and so is a fetcher with no older wal
assert.equal(olderPostsProblem({ label: "Bluesky" }), "Bluesky cannot fetch older posts.");
assert.equal(olderPostsProblem(undefined), "This channel has no post fetcher.");
});
+
+// An account that shows no posts: nothing archived, and the last timeline
+// fetch finished having read none.
+async function emptyAccount(slug: string): Promise<string> {
+ const channelRoot = await makeChannel(slug);
+ await writePostFetchState(channelRoot, {
+ lastFetchedAt: "2026-01-01T00:00:00.000Z",
+ lastFetchedCount: 0,
+ });
+ return channelRoot;
+}
+
+test("an older walk over an account that shows no posts is refused, and searches nothing", async () => {
+ const slug = "older-empty";
+ const channelRoot = await emptyAccount(slug);
+ const before = (await argvLines()).length;
+ const result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ olderWindowPauseMs: 0,
+ });
+ assert.equal(result.ok, false);
+ assert.equal(result.error, EMPTY_ACCOUNT_OLDER_REFUSAL);
+ assert.equal((await argvLines()).length, before);
+ // Nothing is recorded: the refusal is not a walk.
+ assert.equal((await readPostFetchState(channelRoot))?.older, undefined);
+});
+
+test("forced, the walk over an empty account ends after two empty windows, not a year of them", async () => {
+ const slug = "older-empty-forced";
+ const channelRoot = await emptyAccount(slug);
+ const before = (await argvLines()).length;
+ const log: string[] = [];
+ const result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ force: true,
+ olderWindowPauseMs: 0,
+ onLog: (l) => log.push(l),
+ });
+ assert.equal(result.ok, true, log.join("\n"));
+ assert.equal(result.complete, true);
+ assert.equal(result.written, 0);
+ // From today back: two quiet windows, then it stops.
+ assert.equal((await argvLines()).length - before, 2);
+ const older = (await readPostFetchState(channelRoot))?.older;
+ assert.equal(older?.complete, true);
+ assert.equal(older?.emptyWindows, 2);
+ assert.match(older?.completeReason ?? "", /2 windows of 3 months in a row held no posts/);
+ assert.match(older?.completeReason ?? "", /nothing archived and none found/);
+ assert.ok(log.some((l) => /walking anyway, as forced/.test(l)), log.join("\n"));
+ assert.ok(log.some((l) => /backfill complete: .*nothing archived/.test(l)), log.join("\n"));
+});
+
+test("the refusal reads only a clean timeline fetch: a stopped or failed one, or none yet, does not refuse", async () => {
+ const state = { lastFetchedAt: "2026-01-01T00:00:00.000Z", lastFetchedCount: 0 };
+ assert.equal(emptyAccountOlderProblem(0, state), EMPTY_ACCOUNT_OLDER_REFUSAL);
+ assert.equal(emptyAccountOlderProblem(1, state), null);
+ assert.equal(emptyAccountOlderProblem(0, null), null);
+ assert.equal(emptyAccountOlderProblem(0, { ...state, lastFetchedCount: 3 }), null);
+ assert.equal(emptyAccountOlderProblem(0, { ...state, cursor: "1/X" }), null);
+ assert.equal(emptyAccountOlderProblem(0, { ...state, lastError: "boom" }), null);
+ assert.equal(emptyAccountOlderProblem(0, { ...state, needsCookies: true }), null);
+});
+
+test("with nothing archived but posts found this run, the year-of-empties rule still ends the walk", async () => {
+ const slug = "older-found";
+ const channelRoot = await makeChannel(slug);
+ // A walk already under way at the posts, with nothing yet on disk.
+ await writePostFetchState(channelRoot, {
+ older: { since: "2020-12-11", until: "2021-03-11", emptyWindows: 0 },
+ });
+ const before = (await argvLines()).length;
+ const result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ olderWindowPauseMs: 0,
+ });
+ assert.equal(result.ok, true);
+ assert.equal(result.complete, true);
+ // Every post is new, the archived one's included.
+ assert.equal(result.written, 5);
+ assert.equal((await argvLines()).length - before, 10);
+ const older = (await readPostFetchState(channelRoot))?.older;
+ assert.match(older?.completeReason ?? "", /^4 windows of 3 months in a row held no posts, back to [\d-]+$/);
+});
+
+test("a drain mid-window stops the search at once and keeps the resume point inside the window", async () => {
+ const slug = "older-drain-window";
+ const channelRoot = await makeChannel(slug);
+ await seed(channelRoot, slug);
+ const drain = new AbortController();
+ const log: string[] = [];
+ const started = Date.now();
+ process.env.FAKE_MODE = "stall";
+ let result;
+ try {
+ result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ olderWindowPauseMs: 0,
+ drain: drain.signal,
+ onLog: (l) => {
+ log.push(l);
+ if (/rate limit/.test(l)) drain.abort();
+ },
+ });
+ } finally {
+ delete process.env.FAKE_MODE;
+ }
+ assert.equal(result.ok, true, log.join("\n"));
+ assert.equal(result.drained, true);
+ assert.equal(result.complete, false);
+ assert.ok(Date.now() - started < 15_000, `took ${Date.now() - started} ms`);
+ const older = (await readPostFetchState(channelRoot))?.older;
+ assert.deepEqual(
+ { since: older?.since, until: older?.until, maxId: older?.maxId, lastError: older?.lastError },
+ { since: "2020-12-11", until: "2021-03-11", maxId: idOf(POST_DATES[0]), lastError: undefined },
+ );
+ assert.ok(log.some((l) => /Drained; the next run resumes at 2020-12-11 – 2021-03-11 below/.test(l)), log.join("\n"));
+});
+
+test("a drain during the pause between windows ends it at once, at the next window, before another search", async () => {
+ const slug = "older-drain-gap";
+ const channelRoot = await makeChannel(slug);
+ await seed(channelRoot, slug);
+ const before = (await argvLines()).length;
+ const drain = new AbortController();
+ const log: string[] = [];
+ const started = Date.now();
+ const result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ // A gap the test would notice.
+ olderWindowPauseMs: 60_000,
+ drain: drain.signal,
+ onLog: (l) => {
+ log.push(l);
+ // The first window's tally: its boundary is saved next, then the pause.
+ if (/^2020-12-11 – 2021-03-11: read/.test(l)) setTimeout(() => drain.abort(), 500);
+ },
+ });
+ assert.equal(result.ok, true, log.join("\n"));
+ assert.equal(result.drained, true);
+ assert.equal(result.written, 2);
+ assert.ok(Date.now() - started < 15_000, `took ${Date.now() - started} ms`);
+ assert.equal((await argvLines()).length - before, 1);
+ const older = (await readPostFetchState(channelRoot))?.older;
+ assert.deepEqual(
+ { since: older?.since, until: older?.until, maxId: older?.maxId },
+ { since: "2020-09-11", until: "2020-12-11", maxId: undefined },
+ );
+ assert.ok(log.some((l) => /Drained; the next run resumes at 2020-09-11 – 2020-12-11\.$/.test(l)), log.join("\n"));
+});
+
+test("a drain already set when a window ends stops before the pause, never waiting it out", async () => {
+ const slug = "older-drain-boundary";
+ const channelRoot = await makeChannel(slug);
+ await seed(channelRoot, slug);
+ const before = (await argvLines()).length;
+ const drain = new AbortController();
+ const started = Date.now();
+ const result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ olderWindowPauseMs: 60_000,
+ drain: drain.signal,
+ onLog: (l) => {
+ if (/^2020-12-11 – 2021-03-11: read/.test(l)) drain.abort();
+ },
+ });
+ assert.equal(result.drained, true);
+ assert.ok(Date.now() - started < 15_000, `took ${Date.now() - started} ms`);
+ assert.equal((await argvLines()).length - before, 1);
+ assert.equal((await readPostFetchState(channelRoot))?.older?.since, "2020-09-11");
+});
diff --git a/common/controller/fetchPosts.test.ts b/common/controller/fetchPosts.test.ts
@@ -1,6 +1,7 @@
// fetchPosts over the gallery-dl X fetcher, end to end against a fake binary:
// a run that is cancelled (or times out) part-way keeps what it read and the
-// resume point, and the next run continues from there.
+// resume point, and the next run continues from there. A drained run stops at
+// a page boundary, keeps that resume point, and is not a failure.
//
// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test controller/fetchPosts.test.ts
@@ -24,6 +25,8 @@ process.env.FAKE_ARGS_LOG = ARGS_LOG;
// blocks on a "rate limit" (the real stall); given the cursor after page 2 it
// emits page 3 and exits cleanly. FAKE_MODE=known emits page 1 then endless
// tweets older than the watermark, to prove the early stop kills it.
+// FAKE_MODE=pages emits six pages, each followed (after a pause) by an info
+// line, then its cursor: a walk with a page boundary every few hundred ms.
await writeFile(
BIN,
`#!/usr/bin/env node
@@ -59,6 +62,20 @@ if (process.env.FAKE_MODE === "known") {
log("[twitter][debug] Cursor: 1/CRASH-P1");
log("[twitter][error] AbortExtraction: 404 Not Found (GraphQL query id rotated)");
process.exit(4);
+} else if (process.env.FAKE_MODE === "pages") {
+ let p = 0;
+ const next = () => {
+ page(p);
+ setTimeout(() => {
+ log("[twitter][info] page " + p + " read");
+ setTimeout(() => {
+ log("[twitter][debug] Cursor: 1/PAGES-P" + p);
+ p++;
+ if (p < 6) setTimeout(next, 300); else process.exit(0);
+ }, 300);
+ }, 100);
+ };
+ next();
} else if (cursor === "1/AFTER-P2") {
page(2);
process.exit(0);
@@ -197,3 +214,111 @@ test("an auth failure that read nothing keeps the resume point it had, and inven
assert.equal(state?.needsCookies, true);
assert.equal(state?.cursor, undefined);
});
+
+async function makeChannel(slug: string): Promise<string> {
+ const root = path.join(ROOT, "transcripts", "channels", slug);
+ await mkdir(root, { recursive: true });
+ await writeFile(
+ path.join(root, "config.json"),
+ JSON.stringify({
+ handling: "transcribe",
+ sourceKind: "social",
+ postFetcher: "x-gallery-dl",
+ socialHandle: "faketester",
+ platform: "twitter",
+ name: "Fake (X)",
+ url: "https://x.com/faketester",
+ }),
+ );
+ return root;
+}
+
+test("a drain mid-page waits for the page's cursor, stops there and keeps it; the run is not a failure", async () => {
+ const slug = "fake-x-drain-page";
+ const root = await makeChannel(slug);
+ const drain = new AbortController();
+ const log: string[] = [];
+ process.env.FAKE_MODE = "pages";
+ let result;
+ try {
+ result = await fetchPosts({
+ paths: (await import("../lib/paths")).getPaths(),
+ slug,
+ settings: {},
+ drain: drain.signal,
+ onLog: (line) => {
+ log.push(line);
+ // Page 1's records are in; its cursor is not yet.
+ if (/page 1 read/.test(line)) drain.abort();
+ },
+ });
+ } finally {
+ delete process.env.FAKE_MODE;
+ }
+ assert.equal(result.ok, true, log.join("\n"));
+ assert.equal(result.drained, true);
+ assert.equal(result.complete, false);
+ // Pages 0 and 1, and not one more.
+ assert.equal(result.written, 6);
+ assert.equal((await readSeenPostIds(root)).size, 6);
+ const state = await readPostFetchState(root);
+ assert.equal(state?.cursor, "1/PAGES-P1");
+ assert.equal(state?.lastError, undefined);
+ assert.equal(log.some((l) => /page 2 read/.test(l)), false, log.join("\n"));
+ assert.ok(log.some((l) => /Drained; the next run resumes from the last page read/.test(l)), log.join("\n"));
+
+ // The next run resumes there (cancelled once it has started: only its argv
+ // is wanted).
+ const cancel = new AbortController();
+ await fetchPosts({
+ paths: (await import("../lib/paths")).getPaths(),
+ slug,
+ settings: {},
+ signal: cancel.signal,
+ onLog: (line) => {
+ if (/^Running /.test(line)) setTimeout(() => cancel.abort(), 200);
+ },
+ });
+ assert.ok((await lastArgv()).includes("extractor.twitter.cursor=1/PAGES-P1"));
+});
+
+test("a drain while gallery-dl waits out a rate limit stops it at once, keeping the last page's cursor", async () => {
+ const slug = "fake-x-drain-wait";
+ const root = await makeChannel(slug);
+ const drain = new AbortController();
+ const log: string[] = [];
+ const started = Date.now();
+ const result = await fetchPosts({
+ paths: (await import("../lib/paths")).getPaths(),
+ slug,
+ settings: {},
+ drain: drain.signal,
+ onLog: (line) => {
+ log.push(line);
+ if (/rate limit/.test(line)) drain.abort();
+ },
+ });
+ assert.equal(result.ok, true, log.join("\n"));
+ assert.equal(result.drained, true);
+ assert.equal(result.written, 6);
+ // Not left asleep for its 60 s.
+ assert.ok(Date.now() - started < 15_000, `took ${Date.now() - started} ms`);
+ assert.equal((await readPostFetchState(root))?.cursor, "1/AFTER-P2");
+});
+
+test("a drain before the fetch starts reads nothing and spawns nothing", async () => {
+ const slug = "fake-x-drain-early";
+ const root = await makeChannel(slug);
+ const drain = new AbortController();
+ drain.abort();
+ const before = (await readFile(ARGS_LOG, "utf8")).trim().split("\n").length;
+ const result = await fetchPosts({
+ paths: (await import("../lib/paths")).getPaths(),
+ slug,
+ settings: {},
+ drain: drain.signal,
+ });
+ assert.deepEqual(result, { ok: true, written: 0, skipped: 0, complete: false, drained: true });
+ assert.equal((await readFile(ARGS_LOG, "utf8")).trim().split("\n").length, before);
+ assert.equal(await readPostFetchState(root), null);
+});
diff --git a/common/social/olderBackfill.test.ts b/common/social/olderBackfill.test.ts
@@ -67,6 +67,23 @@ test("windows step back with no gap, count empty ones, and end after a run of th
]);
});
+test("nothing archived and nothing found: the walk ends after two empty windows, and says why", () => {
+ const first = stepOlderWindow(
+ { since: "2026-07-05", until: "2026-10-05", emptyWindows: 0 },
+ { hadPosts: false, nothingFound: true },
+ );
+ assert.equal(first.complete, false);
+ if (first.complete) return;
+ const second = stepOlderWindow(first.next, { hadPosts: false, nothingFound: true });
+ assert.equal(second.complete, true);
+ if (!second.complete) return;
+ assert.equal(second.emptyWindows, 2);
+ assert.match(second.reason, /^2 windows of 3 months in a row held no posts, back to 2026-04-05, with nothing archived and none found/);
+ // Without it, the same two windows are not the end: a year of them is.
+ const year = stepOlderWindow(first.next, { hadPosts: false });
+ assert.equal(year.complete, false);
+});
+
test("a window with posts resets the empty count", () => {
const step = stepOlderWindow(
{ since: "2020-01-01", until: "2020-04-01", emptyWindows: 3 },
diff --git a/common/social/xGalleryDlFetcher.test.ts b/common/social/xGalleryDlFetcher.test.ts
@@ -176,10 +176,23 @@ test("stream: a checkpoint carries the PREVIOUS cursor; the final cursor is the
test("stream: debug noise is dropped; info lines are logged and rate limits noticed", () => {
const logged: string[] = [];
const s = newStream({ onLog: (l) => logged.push(l) });
- s.pushStderr("[urllib3.connectionpool][debug] https://x.com:443 \"GET /i/api HTTP/1.1\" 200");
- s.pushStderr("[twitter][info] Waiting for 14 minutes until 15:11:22 (rate limit)");
+ assert.equal(
+ s.pushStderr("[urllib3.connectionpool][debug] https://x.com:443 \"GET /i/api HTTP/1.1\" 200"),
+ undefined,
+ );
+ // A wait on the rate limit, and a page's cursor, are each a point between
+ // pages — where a drain may stop the run.
+ assert.equal(
+ s.pushStderr("[twitter][info] Waiting for 14 minutes until 15:11:22 (rate limit)"),
+ "rate-limit",
+ );
+ assert.equal(s.pushStderr("[twitter][debug] Cursor: 1/ABC"), "cursor");
+ assert.equal(s.pushStderr("[twitter][info] something else"), undefined);
assert.equal(s.rateLimited, true);
- assert.deepEqual(logged, ["gallery-dl: [twitter][info] Waiting for 14 minutes until 15:11:22 (rate limit)"]);
+ assert.deepEqual(logged, [
+ "gallery-dl: [twitter][info] Waiting for 14 minutes until 15:11:22 (rate limit)",
+ "gallery-dl: [twitter][info] something else",
+ ]);
assert.match(s.stderrTail(), /rate limit/);
});
