commit 7c0081629cdce9c3a87d048dc78016f45aed9179
parent 709175ead6010f26be56b8e8b971bbf62d51b6fc
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sat, 3 Oct 2026 13:29:57 -0400
common: tests for the older-posts backfill
Window arithmetic (stepping, empty-window completion, floor clamping), the
search argv and query encoding as gallery-dl decodes it, the stream's oldest
id and account creation time, and fetchPosts({ older }) end to end against a
fake gallery-dl that answers search URLs: state kept apart from the timeline
cursor, a normal fetch ignoring it, floor and creation-date stops, a resume
inside a cancelled window, a limit, and a guest run refused.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
3 files changed, 612 insertions(+), 0 deletions(-)
diff --git a/common/controller/fetchOlderPosts.test.ts b/common/controller/fetchOlderPosts.test.ts
@@ -0,0 +1,398 @@
+// fetchPosts({ older: true }) over the gallery-dl X fetcher, end to end
+// against a fake binary that answers X SEARCH URLs from a fixed set of posts:
+// the walk steps back a window at a time, writes through the same shard
+// writer, records its own position apart from the timeline cursor, stops at a
+// floor / the account's creation date / a run of empty windows, and a normal
+// fetch never resumes backwards from it.
+//
+// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test controller/fetchOlderPosts.test.ts
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { chmod, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+const ROOT = await mkdtemp(path.join(os.tmpdir(), "fetcholder-"));
+const BIN = path.join(ROOT, "fake-gallery-dl.mjs");
+const ARGS_LOG = path.join(ROOT, "argv.jsonl");
+process.env.TRANSCRIPTS_DIR = path.join(ROOT, "transcripts");
+process.env.GALLERY_DL_BIN = BIN;
+process.env.FAKE_ARGS_LOG = ARGS_LOG;
+
+// The account's posts, by date. An id is X's: the creation time in its high
+// bits, so ids order as dates do.
+const POST_DATES = [
+ "2021-03-10 12:00:00", // the oldest post already archived
+ "2021-02-01 09:00:00",
+ "2020-12-15 18:30:00",
+ "2020-06-01 08:00:00",
+ "2019-11-20 22:15:00",
+];
+
+// The fake: a timeline URL reads nothing (the timeline is exhausted); a search
+// URL answers its query's from:/since:/until:/max_id: from POST_DATES, newest
+// first, honouring --range. FAKE_ACCOUNT_DATE is the account's creation date
+// carried on each record's `user`. FAKE_MODE=stall prints the window's first
+// post, then sleeps on a "rate limit" until killed.
+await writeFile(
+ BIN,
+ `#!/usr/bin/env node
+import { appendFileSync } from "node:fs";
+const args = process.argv.slice(2);
+appendFileSync(process.env.FAKE_ARGS_LOG, JSON.stringify(args) + "\\n");
+const url = new URL(args[args.length - 1]);
+if (!url.pathname.startsWith("/search")) process.exit(0);
+const q = url.searchParams.get("q");
+const term = (k) => (q.split(" ").find((t) => t.startsWith(k + ":")) || "").slice(k.length + 1) || undefined;
+const from = term("from"), since = term("since"), until = term("until"), maxId = term("max_id");
+const range = args[args.indexOf("--range") + 1];
+const cap = args.includes("--range") ? Number(range.split("-")[1]) : Infinity;
+const dates = ${JSON.stringify(POST_DATES)};
+const idOf = (d) => (BigInt(Date.parse(d.replace(" ", "T") + "Z") - 1288834974657) << 22n);
+const hits = dates
+ .filter((d) => d.slice(0, 10) >= since && d.slice(0, 10) < until)
+ .filter((d) => !maxId || idOf(d) <= BigInt(maxId))
+ .sort().reverse();
+const user = { name: from, nick: "Example" };
+if (process.env.FAKE_ACCOUNT_DATE) user.date = process.env.FAKE_ACCOUNT_DATE;
+const line = (d) => '[2, {"tweet_id": ' + idOf(d) + ', "date": "' + d + '", "content": "post at ' + d +
+ '", "author": ' + JSON.stringify(user) + ', "user": ' + JSON.stringify(user) + '}]\\n';
+if (process.env.FAKE_MODE === "stall" && hits.length) {
+ process.stdout.write(line(hits[0]));
+ process.stderr.write("[twitter][info] Waiting for 14 minutes until 15:11:22 (rate limit)\\n");
+ setTimeout(() => process.exit(0), 60_000);
+} else {
+ hits.slice(0, cap).forEach((d) => process.stdout.write(line(d)));
+ process.exit(0);
+}
+`,
+);
+await chmod(BIN, 0o755);
+
+const HANDLE = "example_user";
+const channelsDir = path.join(ROOT, "transcripts", "channels");
+
+async function makeChannel(slug: string): Promise<string> {
+ const channelRoot = path.join(channelsDir, slug);
+ await mkdir(channelRoot, { recursive: true });
+ await writeFile(
+ path.join(channelRoot, "config.json"),
+ JSON.stringify({
+ handling: "transcribe",
+ sourceKind: "social",
+ postFetcher: "x-gallery-dl",
+ socialHandle: HANDLE,
+ platform: "twitter",
+ name: "Example (X)",
+ url: `https://x.com/${HANDLE}`,
+ }),
+ );
+ return channelRoot;
+}
+
+const { fetchPosts, FULL_AND_OLDER_REFUSAL, olderPostsProblem } = await import("./fetchPosts");
+const {
+ readPostFetchState,
+ readSeenPostIds,
+ writePostFetchState,
+ writePosts,
+ readAllPosts,
+} = await import("../lib/posts-server");
+const { normalizeXTweet } = await import("../social/xNormalize");
+const { getPaths } = await import("../lib/paths");
+
+const LOGIN = { cookiesFromBrowser: "firefox" };
+
+function idOf(d: string): string {
+ return String(
+ BigInt(Date.parse(d.replace(" ", "T") + "Z") - 1288834974657) << 22n,
+ );
+}
+
+// The archive as a normal fetch would have left it: the oldest post, plus a
+// timeline resume point.
+async function seed(channelRoot: string, slug: string): Promise<void> {
+ const post = normalizeXTweet(
+ {
+ tweet_id: idOf(POST_DATES[0]),
+ date: POST_DATES[0],
+ content: "archived",
+ author: { name: HANDLE },
+ user: { name: HANDLE },
+ },
+ slug,
+ );
+ assert.ok(post);
+ await writePosts(channelRoot, [post]);
+ await writePostFetchState(channelRoot, {
+ lastFetchedAt: "2026-01-01T00:00:00.000Z",
+ cursor: "1/TIMELINE-RESUME",
+ });
+}
+
+async function argvLines(): Promise<string[][]> {
+ try {
+ return (await readFile(ARGS_LOG, "utf8"))
+ .trim()
+ .split("\n")
+ .filter(Boolean)
+ .map((l) => JSON.parse(l) as string[]);
+ } catch {
+ return [];
+ }
+}
+
+const queryOf = (argv: string[]) =>
+ new URL(argv[argv.length - 1]).searchParams.get("q") ?? "";
+
+test("an older fetch walks back window by window and stops after four empty windows", async () => {
+ const slug = "older-walk";
+ const channelRoot = await makeChannel(slug);
+ await seed(channelRoot, slug);
+ const before = (await argvLines()).length;
+ const log: string[] = [];
+ const result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ olderWindowPauseMs: 0,
+ onLog: (l) => log.push(l),
+ });
+ assert.equal(result.ok, true, log.join("\n"));
+ assert.equal(result.complete, true);
+ assert.equal(result.written, 4);
+ assert.equal((await readSeenPostIds(channelRoot)).size, 5);
+
+ const runs = (await argvLines()).slice(before);
+ const queries = runs.map(queryOf);
+ // Windows tile backwards from the day after the oldest archived post.
+ assert.deepEqual(queries.slice(0, 3), [
+ `from:${HANDLE} since:2020-12-11 until:2021-03-11 include:nativeretweets`,
+ `from:${HANDLE} since:2020-09-11 until:2020-12-11 include:nativeretweets`,
+ `from:${HANDLE} since:2020-06-11 until:2020-09-11 include:nativeretweets`,
+ ]);
+ // Ten windows: three with posts, the run of four empty ones at the end, and
+ // the empty ones between that a later post reset.
+ assert.equal(runs.length, 10, queries.join("\n"));
+ assert.match(queries[9], /since:2018-09-11 until:2018-12-11/);
+
+ const state = await readPostFetchState(channelRoot);
+ assert.equal(state?.older?.complete, true);
+ assert.match(state?.older?.completeReason ?? "", /4 windows of 3 months in a row held no posts/);
+ // The timeline's own state is exactly as the last normal fetch left it.
+ assert.equal(state?.cursor, "1/TIMELINE-RESUME");
+ assert.equal(state?.lastFetchedAt, "2026-01-01T00:00:00.000Z");
+ // Not a sync: the channel's sync stamp is not touched.
+ const config = JSON.parse(await readFile(path.join(channelRoot, "config.json"), "utf8"));
+ assert.equal(config.lastSyncedAt, undefined);
+
+ // Re-running says so, and searches nothing.
+ const again: string[] = [];
+ const rerun = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ olderWindowPauseMs: 0,
+ onLog: (l) => again.push(l),
+ });
+ assert.equal(rerun.ok, true);
+ assert.equal(rerun.written, 0);
+ assert.equal((await argvLines()).length, before + 10);
+ assert.ok(again.some((l) => /already complete/.test(l)), again.join("\n"));
+});
+
+test("a normal fetch ignores the older walk's state, resumes the TIMELINE, and keeps the older state", async () => {
+ const slug = "older-walk";
+ const channelRoot = path.join(channelsDir, slug);
+ const olderBefore = (await readPostFetchState(channelRoot))?.older;
+ assert.ok(olderBefore);
+ const result = await fetchPosts({ paths: getPaths(), slug, settings: LOGIN });
+ assert.equal(result.ok, true);
+ const argv = (await argvLines()).at(-1)!;
+ assert.equal(argv[argv.length - 1], `https://x.com/${HANDLE}/timeline`);
+ assert.ok(argv.includes("extractor.twitter.cursor=1/TIMELINE-RESUME"));
+ assert.equal(argv.some((a) => a.includes("/search")), false);
+ const state = await readPostFetchState(channelRoot);
+ assert.deepEqual(state?.older, olderBefore);
+});
+
+test("a normal fetch with a half-walked older state still reads new posts from the top", async () => {
+ const slug = "older-half";
+ const channelRoot = await makeChannel(slug);
+ await seed(channelRoot, slug);
+ await writePostFetchState(channelRoot, {
+ older: { since: "2019-01-01", until: "2019-04-01", maxId: "123", emptyWindows: 1 },
+ });
+ await fetchPosts({ paths: getPaths(), slug, settings: LOGIN });
+ const argv = (await argvLines()).at(-1)!;
+ assert.equal(argv[argv.length - 1], `https://x.com/${HANDLE}/timeline`);
+ assert.equal(argv.some((a) => a.startsWith("extractor.twitter.cursor=")), false);
+ assert.deepEqual((await readPostFetchState(channelRoot))?.older, {
+ since: "2019-01-01",
+ until: "2019-04-01",
+ maxId: "123",
+ emptyWindows: 1,
+ });
+});
+
+test("a floor date ends the walk at that date, and posts below it are not fetched", async () => {
+ const slug = "older-floor";
+ const channelRoot = await makeChannel(slug);
+ await seed(channelRoot, slug);
+ const result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ floor: "2020-07-01",
+ olderWindowPauseMs: 0,
+ });
+ assert.equal(result.ok, true);
+ assert.equal(result.complete, true);
+ assert.equal(result.written, 2);
+ const queries = (await argvLines()).map(queryOf);
+ assert.match(queries.at(-1)!, /since:2020-07-01 until:2020-09-11/);
+ const state = await readPostFetchState(channelRoot);
+ assert.equal(state?.older?.floor, "2020-07-01");
+ assert.match(state?.older?.completeReason ?? "", /reached 2020-07-01/);
+ const dates = (await readAllPosts(channelRoot)).map((p) => p.createdAt.slice(0, 10));
+ assert.equal(dates.includes("2020-06-01"), false);
+});
+
+test("the account's creation date, once a post carries it, is the floor", async () => {
+ const slug = "older-created";
+ const channelRoot = await makeChannel(slug);
+ await seed(channelRoot, slug);
+ process.env.FAKE_ACCOUNT_DATE = "2019-10-05 10:00:00";
+ try {
+ const result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ olderWindowPauseMs: 0,
+ });
+ assert.equal(result.complete, true);
+ assert.equal(result.written, 4);
+ } finally {
+ delete process.env.FAKE_ACCOUNT_DATE;
+ }
+ const state = await readPostFetchState(channelRoot);
+ assert.equal(state?.older?.accountCreatedAt, "2019-10-05T10:00:00.000Z");
+ assert.match(state?.older?.completeReason ?? "", /reached 2019-10-05/);
+ // It stopped at the window holding the creation date, not four windows on.
+ assert.match((await argvLines()).map(queryOf).at(-1)!, /since:2019-10-05 until:2019-12-11/);
+});
+
+test("a cancelled window keeps what it read and resumes INSIDE the window, below the oldest post read", async () => {
+ const slug = "older-cancel";
+ const channelRoot = await makeChannel(slug);
+ await seed(channelRoot, slug);
+ const controller = new AbortController();
+ process.env.FAKE_MODE = "stall";
+ try {
+ const result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ olderWindowPauseMs: 0,
+ signal: controller.signal,
+ onLog: (l) => {
+ if (/rate limit/.test(l)) controller.abort();
+ },
+ });
+ assert.equal(result.ok, true);
+ assert.equal(result.complete, false);
+ } finally {
+ delete process.env.FAKE_MODE;
+ }
+ // The stall printed the window's newest post: the archived one.
+ const state = await readPostFetchState(channelRoot);
+ assert.deepEqual(
+ { since: state?.older?.since, until: state?.older?.until, maxId: state?.older?.maxId },
+ { since: "2020-12-11", until: "2021-03-11", maxId: idOf(POST_DATES[0]) },
+ );
+ assert.equal(state?.cursor, "1/TIMELINE-RESUME");
+
+ const result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ olderWindowPauseMs: 0,
+ });
+ assert.equal(result.complete, true);
+ assert.equal(result.written, 4);
+ const resumed = (await argvLines()).map(queryOf).find((q) => q.includes("max_id:"));
+ assert.equal(
+ resumed,
+ `from:${HANDLE} since:2020-12-11 until:2021-03-11 include:nativeretweets max_id:${idOf(POST_DATES[0])}`,
+ );
+});
+
+test("a limit stops the walk part-way with a resume point", async () => {
+ const slug = "older-limit";
+ const channelRoot = await makeChannel(slug);
+ await seed(channelRoot, slug);
+ const result = await fetchPosts({
+ paths: getPaths(),
+ slug,
+ settings: LOGIN,
+ older: true,
+ limit: 2,
+ olderWindowPauseMs: 0,
+ });
+ assert.equal(result.complete, false);
+ assert.equal(result.written, 1);
+ const argv = (await argvLines()).at(-1)!;
+ assert.equal(argv[argv.indexOf("--range") + 1], "1-2");
+ const older = (await readPostFetchState(channelRoot))?.older;
+ assert.equal(older?.maxId, idOf(POST_DATES[1]));
+ assert.equal(older?.complete, undefined);
+});
+
+test("with no login an older fetch needs cookies and spawns nothing", async () => {
+ const slug = "older-guest";
+ const channelRoot = await makeChannel(slug);
+ await seed(channelRoot, slug);
+ const before = (await argvLines()).length;
+ const result = await fetchPosts({ paths: getPaths(), slug, settings: {}, older: true });
+ assert.equal(result.ok, false);
+ assert.equal(result.needsCookies, true);
+ assert.match(result.error ?? "", /X search needs a logged-in session/);
+ assert.equal((await argvLines()).length, before);
+ const state = await readPostFetchState(channelRoot);
+ assert.equal(state?.needsCookies, true);
+ assert.equal(state?.cursor, "1/TIMELINE-RESUME");
+ assert.equal(state?.older?.complete, undefined);
+});
+
+test("full and older together are refused, and so is a fetcher with no older walk", async () => {
+ const both = await fetchPosts({
+ paths: getPaths(),
+ slug: "older-walk",
+ settings: LOGIN,
+ full: true,
+ older: true,
+ });
+ assert.equal(both.ok, false);
+ assert.equal(both.error, FULL_AND_OLDER_REFUSAL);
+
+ const badFloor = await fetchPosts({
+ paths: getPaths(),
+ slug: "older-walk",
+ settings: LOGIN,
+ older: true,
+ floor: "2020-02-30",
+ });
+ assert.equal(badFloor.ok, false);
+ assert.match(badFloor.error ?? "", /not a date/);
+
+ assert.equal(olderPostsProblem({ label: "X", fetchOlder: async () => ({ posts: [], complete: true, position: { since: "", until: "", emptyWindows: 0 } }) }), null);
+ assert.equal(olderPostsProblem({ label: "Bluesky" }), "Bluesky cannot fetch older posts.");
+ assert.equal(olderPostsProblem(undefined), "This channel has no post fetcher.");
+});
diff --git a/common/social/olderBackfill.test.ts b/common/social/olderBackfill.test.ts
@@ -0,0 +1,116 @@
+// The older-posts backfill's window arithmetic (olderBackfill.ts).
+//
+// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test social/olderBackfill.test.ts
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ addUtcDays,
+ addUtcMonths,
+ describeOlderBackfill,
+ firstOlderWindow,
+ isUtcDay,
+ olderFloorDay,
+ stepOlderWindow,
+} from "./olderBackfill";
+
+test("day arithmetic is UTC and clamps a month to its last day", () => {
+ assert.equal(addUtcDays("2020-12-31", 1), "2021-01-01");
+ assert.equal(addUtcDays("2020-03-01", -1), "2020-02-29");
+ assert.equal(addUtcMonths("2024-05-31", -3), "2024-02-29");
+ assert.equal(addUtcMonths("2021-01-15", -3), "2020-10-15");
+ assert.equal(isUtcDay("2020-02-29"), true);
+ assert.equal(isUtcDay("2021-02-29"), false);
+ assert.equal(isUtcDay("2021-2-1"), false);
+});
+
+test("the first window ends the day after the oldest archived post", () => {
+ assert.deepEqual(firstOlderWindow({ oldestArchivedAt: "2021-03-10T23:59:00.000Z" }), {
+ since: "2020-12-11",
+ until: "2021-03-11",
+ emptyWindows: 0,
+ });
+ // Nothing archived: it ends tomorrow.
+ assert.equal(
+ firstOlderWindow({ now: new Date("2026-10-03T12:00:00Z") }).until,
+ "2026-10-04",
+ );
+ // A floor inside the first window clamps its start.
+ assert.equal(
+ firstOlderWindow({ oldestArchivedAt: "2021-03-10T00:00:00Z", floor: "2021-01-01" }).since,
+ "2021-01-01",
+ );
+});
+
+test("windows step back with no gap, count empty ones, and end after a run of them", () => {
+ let pos = firstOlderWindow({ oldestArchivedAt: "2021-03-10T00:00:00Z" });
+ const seen: string[] = [];
+ const held = new Set(["2020-12-11"]); // only the first window holds posts
+ for (let i = 0; i < 10; i++) {
+ seen.push(`${pos.since}..${pos.until}`);
+ const step = stepOlderWindow(pos, { hadPosts: held.has(pos.since) });
+ if (step.complete) {
+ assert.equal(step.emptyWindows, 4);
+ assert.match(step.reason, /4 windows of 3 months in a row held no posts, back to 2019-12-11/);
+ break;
+ }
+ assert.equal(step.next.until, pos.since);
+ assert.equal(step.next.maxId, undefined);
+ pos = step.next;
+ }
+ assert.deepEqual(seen, [
+ "2020-12-11..2021-03-11",
+ "2020-09-11..2020-12-11",
+ "2020-06-11..2020-09-11",
+ "2020-03-11..2020-06-11",
+ "2019-12-11..2020-03-11",
+ ]);
+});
+
+test("a window with posts resets the empty count", () => {
+ const step = stepOlderWindow(
+ { since: "2020-01-01", until: "2020-04-01", emptyWindows: 3 },
+ { hadPosts: true },
+ );
+ assert.equal(step.complete, false);
+ if (!step.complete) assert.equal(step.next.emptyWindows, 0);
+});
+
+test("the floor clamps the last window and ends the walk after it", () => {
+ const step = stepOlderWindow(
+ { since: "2020-04-01", until: "2020-07-01", emptyWindows: 0 },
+ { hadPosts: true, floor: "2020-02-15" },
+ );
+ assert.equal(step.complete, false);
+ if (step.complete) return;
+ assert.deepEqual(step.next, { since: "2020-02-15", until: "2020-04-01", emptyWindows: 0 });
+ const last = stepOlderWindow(step.next, { hadPosts: false, floor: "2020-02-15" });
+ assert.equal(last.complete, true);
+ if (last.complete) assert.match(last.reason, /reached 2020-02-15/);
+});
+
+test("the floor is the later of the given date and the account's creation day", () => {
+ assert.equal(olderFloorDay(undefined, undefined), undefined);
+ assert.equal(olderFloorDay("2015-01-01", undefined), "2015-01-01");
+ assert.equal(olderFloorDay(undefined, "2012-06-30T22:00:00.000Z"), "2012-06-30");
+ assert.equal(olderFloorDay("2015-01-01", "2012-06-30T22:00:00.000Z"), "2015-01-01");
+ assert.equal(olderFloorDay("2010-01-01", "2012-06-30T22:00:00.000Z"), "2012-06-30");
+});
+
+test("the channel page's one line", () => {
+ assert.equal(describeOlderBackfill(undefined), "not started");
+ assert.equal(
+ describeOlderBackfill({ since: "2019-01-01", until: "2019-04-01", emptyWindows: 0 }),
+ "walked back to 2019-04-01; next 2019-01-01 – 2019-04-01",
+ );
+ assert.equal(
+ describeOlderBackfill({
+ since: "2019-01-01",
+ until: "2019-04-01",
+ emptyWindows: 4,
+ complete: true,
+ completeReason: "reached 2019-01-01, the earliest date to walk to",
+ }),
+ "complete — reached 2019-01-01, the earliest date to walk to",
+ );
+});
diff --git a/common/social/xGalleryDlFetcher.test.ts b/common/social/xGalleryDlFetcher.test.ts
@@ -209,3 +209,101 @@ test("stream: a gallery-dl that ignores output.jsonl (one pretty array at exit)
test("restart-from-top cursor is the timeline extractor's first state", () => {
assert.equal(RESTART_FROM_TOP_CURSOR, "1/");
});
+
+// ---------------------------------------------------------------------------
+// Search: the older-posts backfill's argv
+// ---------------------------------------------------------------------------
+
+import {
+ buildGalleryDlSearchArgs,
+ xSearchHandle,
+ xSearchQuery,
+ xSearchUrl,
+} from "./xGalleryDlFetcher";
+
+// What gallery-dl's search extractor does with the URL's q: "+" to a space,
+// THEN unquote (extractor/twitter.py, TwitterSearchExtractor.tweets).
+function galleryDlQueryOf(url: string): string {
+ const m = /\/search\/?\?(?:[^&#]+&)*q=([^&#]+)/.exec(url);
+ assert.ok(m, url);
+ return decodeURIComponent(m[1].replace(/\+/g, " "));
+}
+
+test("search query: from/since/until, reposts included, max_id only when resuming", () => {
+ const window = { since: "2020-12-11", until: "2021-03-11" };
+ assert.equal(
+ xSearchQuery("example_user", window),
+ "from:example_user since:2020-12-11 until:2021-03-11 include:nativeretweets",
+ );
+ assert.equal(
+ xSearchQuery("@example_user", { ...window, maxId: "1899000000000000001" }),
+ "from:example_user since:2020-12-11 until:2021-03-11 include:nativeretweets max_id:1899000000000000001",
+ );
+ assert.throws(() => xSearchQuery("example_user", { ...window, maxId: "12 OR x" }), /not a post id/);
+});
+
+test("search handle: anything search would read as more than a name is refused", () => {
+ assert.equal(xSearchHandle(" @Example_User1 "), "Example_User1");
+ for (const bad of ["example user", "example:user", 'ex"ample', "example.user", "", "a-b"]) {
+ assert.throws(() => xSearchHandle(bad), /not an X handle/, bad);
+ }
+});
+
+test("search URL: the query survives gallery-dl's decoding exactly, spaces and colons included", () => {
+ const query = xSearchQuery("example_user", {
+ since: "2020-12-11",
+ until: "2021-03-11",
+ maxId: "1899000000000000001",
+ });
+ const url = xSearchUrl(query);
+ assert.match(url, /^https:\/\/x\.com\/search\?q=from%3Aexample_user%20since%3A2020-12-11%20/);
+ assert.ok(url.endsWith("&f=live"));
+ assert.equal(url.includes(" "), false);
+ assert.equal(galleryDlQueryOf(url), query);
+ // An encoded "+" would stay a "+", not become a space.
+ assert.equal(galleryDlQueryOf(xSearchUrl("a+b c")), "a+b c");
+});
+
+test("search argv: the timeline's flags and login, latest-first max_id paging, no cursor", () => {
+ const argv = buildGalleryDlSearchArgs({
+ handle: "example_user",
+ position: { since: "2020-12-11", until: "2021-03-11" },
+ cookieFile: JAR,
+ limit: 50,
+ });
+ assert.equal(flagValue(argv, "--cookies"), JAR);
+ assert.equal(flagValue(argv, "--range"), "1-50");
+ for (const opt of [
+ "output.jsonl=true",
+ "extractor.twitter.text-tweets=true",
+ "extractor.twitter.retweets=true",
+ "extractor.twitter.replies=true",
+ "extractor.twitter.search-results=latest",
+ "extractor.twitter.search-pagination=max_id",
+ ]) {
+ assert.ok(argv.includes(opt), opt);
+ }
+ assert.equal(argv.some((a) => a.startsWith("extractor.twitter.cursor=")), false);
+ assert.equal(argv.includes("--no-download"), true);
+ assert.equal(
+ galleryDlQueryOf(argv[argv.length - 1]),
+ "from:example_user since:2020-12-11 until:2021-03-11 include:nativeretweets",
+ );
+});
+
+test("stream: tracks the oldest post id read (any post, archived or not) and the account's creation time", () => {
+ const known = String(1899000000000000000n + 1n);
+ const s = newStream({ seenIds: new Set([`${known}`]), stopAtKnown: false, accountHandle: "FakeTester" });
+ s.pushStdout(jsonl(5));
+ s.pushStdout(jsonl(1)); // archived already: still the oldest read
+ s.pushStdout(jsonl(3));
+ assert.equal(s.oldestId, known);
+ assert.equal(s.accountCreatedAt, undefined);
+ s.pushStdout(
+ JSON.stringify([2, { ...tweet(2), user: { name: "faketester", date: "2009-03-10 18:00:00" } }])
+ .replace(/"tweet_id":"(\d+)"/, '"tweet_id":$1'),
+ );
+ assert.equal(s.accountCreatedAt, "2009-03-10T18:00:00.000Z");
+ assert.equal(s.takePending().length, 3);
+ assert.equal(s.pendingCount(), 0);
+});
