commit dd55e7f7c97c335a28f9802b20744058e2ac88cd
parent 978cfaf326111895a5eff2cfbd1c3ba67f703e84
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 6 Aug 2026 17:55:12 -0400
X ingest: target the timeline sub-extractor, and treat cookies as optional
Three fixes, all found by running a real gallery-dl 1.32.9 against live X.
1. A bare profile URL returns NO posts. gallery-dl's `twitter:user` extractor
treats `https://x.com/<handle>` as a DISPATCH entry: with --dump-json it
emits one type-6 record pointing at `/<handle>/timeline` and yields zero
tweets. The configured thequartering-X channel had synced with
`lastFetchedCount: 0` for exactly this reason. buildGalleryDlArgs now
normalizes to the timeline sub-extractor (timelineUrlFor), leaving explicit
sub-routes — /with_replies, /media, /status/<id> — untouched.
2. Cookies are genuinely optional. gallery-dl reads X timelines on a guest
token with no account at all: verified 500 tweets spanning ~7.5 months for
a public account, cursor-paginated, with no credentials and no config. So a
channel with no cookies configured now runs as a guest rather than being
treated as blocked. Credentials remain the path for a deep backfill, since
guest quota is much smaller.
3. Rate limiting was invisible. gallery-dl handles X throttling by BLOCKING
("Waiting for 7 minutes until ... (rate limit)"), which from the outside
looks like a hung job. That is now detected and logged — and, importantly,
a throttled run no longer reports `complete`, because doing so would let
the controller advance the watermark past posts it never saw.
Verified end-to-end against the real channel: 3 posts written where the same
code previously wrote 0, ids byte-exact against the raw gallery-dl output.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Diffstat:
2 files changed, 78 insertions(+), 3 deletions(-)
diff --git a/common/social/xGalleryDlFetcher.ts b/common/social/xGalleryDlFetcher.ts
@@ -59,6 +59,25 @@ function looksLikeAuthFailure(text: string): boolean {
// The argv for a text-only timeline read. Exported so a test can assert the
// flags without spawning anything.
+// gallery-dl's `twitter:user` extractor treats a bare profile URL as a
+// DISPATCH entry: with --dump-json it emits a single type-6 record pointing at
+// `/<handle>/timeline` and yields no tweets at all (verified against
+// gallery-dl 1.32.9 — a real channel synced with lastFetchedCount: 0 because
+// of this). Targeting the timeline sub-extractor directly is what actually
+// returns posts, so normalize here rather than making every caller remember.
+export function timelineUrlFor(accountUrl: string): string {
+ const trimmed = accountUrl.trim().replace(/\/+$/, "");
+ // Already a sub-route (/timeline, /with_replies, /media, /status/...) — leave it.
+ try {
+ const u = new URL(trimmed);
+ const segments = u.pathname.split("/").filter(Boolean);
+ if (segments.length !== 1) return trimmed;
+ return `${u.origin}/${segments[0]}/timeline`;
+ } catch {
+ return trimmed;
+ }
+}
+
export function buildGalleryDlArgs(opts: {
accountUrl: string;
cookies?: string;
@@ -90,6 +109,13 @@ export function buildGalleryDlArgs(opts: {
"-o",
"extractor.twitter.cards=false",
];
+ // Cookies are OPTIONAL. gallery-dl reads X timelines on a guest token with no
+ // account at all (verified: 500 tweets over ~7.5 months for a public
+ // account). Guest access is rate-limited far more aggressively than an
+ // authenticated session, though — gallery-dl will block for minutes on
+ // "Waiting for N minutes (rate limit)" — so credentials remain the path for
+ // a deep backfill. When none are configured we simply run as a guest rather
+ // than failing.
if (opts.cookieFile) {
args.push("--cookies", opts.cookieFile);
} else if (opts.cookies) {
@@ -99,7 +125,7 @@ export function buildGalleryDlArgs(opts: {
if (opts.limit && opts.limit > 0) {
args.push("--range", `1-${Math.floor(opts.limit)}`);
}
- args.push(opts.accountUrl);
+ args.push(timelineUrlFor(opts.accountUrl));
return args;
}
@@ -287,6 +313,20 @@ export const xGalleryDlFetcher: SocialFetcher = {
throw new Error(`gallery-dl exited ${exitCode}: ${tail || "(no output)"}`);
}
+ // gallery-dl handles X rate limits by BLOCKING ("Waiting for 7 minutes
+ // until ... (rate limit)"), which otherwise looks like a hung job. Surface
+ // it so the log explains the stall. Guest (cookie-less) runs hit this far
+ // sooner than authenticated ones.
+ const rateLimited = /Waiting for .*rate limit|429 Too Many Requests/i.test(stderr);
+ if (rateLimited) {
+ onLog?.(
+ "[rate-limit] X throttled this run — gallery-dl paused between pages. " +
+ (cookieFile || cookies
+ ? "Consider a longer sync interval."
+ : "This is a guest (no-credential) run; guest quota is much smaller."),
+ );
+ }
+
const records = parseGalleryDlOutput(stdout);
onLog?.(`gallery-dl returned ${records.length} record(s)`);
const all = normalizeXTweets(records, channelSlug);
@@ -317,8 +357,11 @@ export const xGalleryDlFetcher: SocialFetcher = {
// gallery-dl has no resumable cursor: it either walked the timeline it was
// given or it didn't. `complete` is therefore true whenever it exited
// cleanly — except when a --range cap may have cut it short.
+ // A capped OR rate-limited run stopped early, so it must not report
+ // `complete` — that would let the controller advance the watermark past
+ // posts it never saw.
const capped = Boolean(limit && all.length >= limit);
- return { posts, complete: !capped };
+ return { posts, complete: !capped && !rateLimited };
},
};
diff --git a/common/social/xNormalize.test.ts b/common/social/xNormalize.test.ts
@@ -13,6 +13,7 @@ import {
buildGalleryDlArgs,
parseGalleryDlOutput,
quoteBigIntegers,
+ timelineUrlFor,
} from "./xGalleryDlFetcher";
import { isXCookie, toNetscapeCookieFile } from "./xSessionBroker";
import {
@@ -178,7 +179,9 @@ test("gallery-dl argv enables text-tweets and downloads nothing", () => {
["--cookies-from-browser", "firefox"],
);
assert.match(joined, /--range 1-50/);
- assert.equal(args[args.length - 1], "https://x.com/someaccount");
+ // Normalized to the timeline sub-extractor — a bare profile URL yields no
+ // tweets at all (see timelineUrlFor).
+ assert.equal(args[args.length - 1], "https://x.com/someaccount/timeline");
});
test("gallery-dl argv omits cookies when none are resolved", () => {
@@ -433,3 +436,32 @@ test("quoteBigIntegers only touches long integer JSON values", () => {
{ id: "2085320225776427457", n: 7 },
);
});
+
+// ─── profile URL must target the timeline sub-extractor ───
+
+test("a bare profile URL is normalized to /timeline", () => {
+ // gallery-dl yields only a type-6 DISPATCH record for a bare profile URL and
+ // no tweets at all — this is what made a real channel sync 0 posts.
+ assert.equal(timelineUrlFor("https://x.com/TheQuartering"), "https://x.com/TheQuartering/timeline");
+ assert.equal(timelineUrlFor("https://x.com/TheQuartering/"), "https://x.com/TheQuartering/timeline");
+ assert.equal(timelineUrlFor("https://twitter.com/nasa"), "https://twitter.com/nasa/timeline");
+});
+
+test("an explicit sub-route is left alone", () => {
+ for (const u of [
+ "https://x.com/nasa/timeline",
+ "https://x.com/nasa/with_replies",
+ "https://x.com/nasa/media",
+ "https://x.com/NASA/status/2085320225776427457",
+ ]) {
+ assert.equal(timelineUrlFor(u), u.replace(/\/+$/, ""));
+ }
+});
+
+test("buildGalleryDlArgs emits the timeline URL, and cookies stay optional", () => {
+ const guest = buildGalleryDlArgs({ accountUrl: "https://x.com/TheQuartering" });
+ assert.equal(guest[guest.length - 1], "https://x.com/TheQuartering/timeline");
+ // No credentials configured must NOT be a failure — guest reads work.
+ assert.ok(!guest.includes("--cookies"));
+ assert.ok(!guest.includes("--cookies-from-browser"));
+});
