commit 978cfaf326111895a5eff2cfbd1c3ba67f703e84
parent 73de8e3ea2a41949f3e85b67b9f080a980a4ac06
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 6 Aug 2026 17:23:51 -0400
Fix 64-bit tweet id corruption in gallery-dl output parsing
gallery-dl emits `tweet_id` as an UNQUOTED JSON number
(`"tweet_id": 2085320225776427457`). X ids exceed 2^53, so JSON.parse
silently rounds them — 2085320225776427457 became ...427500.
That corrupted the post id, its slug, its permalink (404) and, worst, the
posts-archive dedupe key that makes re-runs incremental: a rounded id never
matches the real one, so every sync would re-add the same posts, and two
distinct tweets could collide onto one id.
xIdOf() already preferred strings, but the damage happened earlier, at
JSON.parse — by the time the normalizer saw the record the digits were gone.
Quote long integer literals before parsing. The lookahead preserves the
trailing delimiter so adjacent ids in an array both match, and requiring a
`:`/`[`/`,` prefix leaves digits inside string values alone.
Found by running a real unauthenticated `gallery-dl 1.32.9 --dump-json`
against a public profile. The previous test asserted "ids stay strings" but
fed a STRING tweet_id — not what gallery-dl actually emits — so it passed
while the real path was broken. The new test uses the verbatim real shape,
and the fix is verified against a captured 2.4 MB run: 500/500 ids appear
byte-identical in the raw output.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Diffstat:
2 files changed, 64 insertions(+), 3 deletions(-)
diff --git a/common/social/xGalleryDlFetcher.ts b/common/social/xGalleryDlFetcher.ts
@@ -103,6 +103,21 @@ export function buildGalleryDlArgs(opts: {
return args;
}
+// X ids are 64-bit and gallery-dl emits them as UNQUOTED JSON NUMBERS
+// (`"tweet_id": 2085320225776427457`). That exceeds 2^53, so a plain
+// JSON.parse silently rounds it — 2085320225776427457 becomes
+// ...427500 — which would corrupt the post id, its slug, its permalink AND
+// the archive dedupe key that makes re-runs incremental. Verified against
+// real gallery-dl 1.32.9 output.
+//
+// So quote every long integer literal BEFORE parsing. The lookahead leaves the
+// trailing delimiter in place so adjacent numbers in an array both match, and
+// requiring a `:`/`[`/`,` prefix means digits inside string values are
+// untouched (they are already inside quotes).
+export function quoteBigIntegers(json: string): string {
+ return json.replace(/([:[,]\s*)(-?\d{16,})(?=\s*[,}\]])/g, '$1"$2"');
+}
+
// gallery-dl's --dump-json emits either one object per line, or a top-level
// array. Tolerate both, plus interleaved non-JSON log noise.
export function parseGalleryDlOutput(stdout: string): XTweetRaw[] {
@@ -111,7 +126,7 @@ export function parseGalleryDlOutput(stdout: string): XTweetRaw[] {
// Whole-payload array first.
if (trimmed.startsWith("[")) {
try {
- const parsed = JSON.parse(trimmed) as unknown;
+ const parsed = JSON.parse(quoteBigIntegers(trimmed)) as unknown;
if (Array.isArray(parsed)) return flattenRecords(parsed);
} catch {
// fall through to line mode
@@ -122,7 +137,7 @@ export function parseGalleryDlOutput(stdout: string): XTweetRaw[] {
const t = line.trim();
if (!t || (!t.startsWith("{") && !t.startsWith("["))) continue;
try {
- out.push(...flattenRecords([JSON.parse(t) as unknown]));
+ out.push(...flattenRecords([JSON.parse(quoteBigIntegers(t)) as unknown]));
} catch {
// Not JSON (a progress line) — ignore.
}
diff --git a/common/social/xNormalize.test.ts b/common/social/xNormalize.test.ts
@@ -9,7 +9,11 @@ import {
xText,
} from "./xNormalize";
import { parsePost } from "../lib/posts";
-import { buildGalleryDlArgs, parseGalleryDlOutput } from "./xGalleryDlFetcher";
+import {
+ buildGalleryDlArgs,
+ parseGalleryDlOutput,
+ quoteBigIntegers,
+} from "./xGalleryDlFetcher";
import { isXCookie, toNetscapeCookieFile } from "./xSessionBroker";
import {
collectTweetsFromGraphQL,
@@ -387,3 +391,45 @@ test("the fallback never claims a URL by detection", () => {
assert.equal(xPlaywrightFetcher.detect("https://x.com/someaccount"), false);
assert.equal(xPlaywrightFetcher.platform, "twitter");
});
+
+// ─── 64-bit id precision (regression) ───
+//
+// Captured from a real `gallery-dl 1.32.9 --dump-json` run: it emits tweet_id
+// as an UNQUOTED JSON number. The earlier test here fed a *string* id, so it
+// passed while the real path silently corrupted every id.
+
+test("gallery-dl's unquoted 64-bit tweet_id survives parsing intact", () => {
+ // Verbatim shape from real output (id > 2^53).
+ const line = '{"tweet_id": 2085320225776427457, "date": "2026-08-06 11:00:59", "content": "hi", "author": {"name": "NASA"}}';
+
+ // A plain JSON.parse loses precision — this is the bug being guarded.
+ assert.equal(String(JSON.parse(line).tweet_id), "2085320225776427500");
+
+ const [rec] = parseGalleryDlOutput(line);
+ assert.equal(rec.tweet_id, "2085320225776427457", "exact id preserved");
+
+ const post = normalizeXTweet(rec, "xchan")!;
+ assert.equal(post.id, "2085320225776427457");
+ assert.equal(post.slug, "xchan/2085320225776427457");
+ // A corrupted id would 404 and, worse, break the archive dedupe key.
+ assert.match(post.url, /status\/2085320225776427457$/);
+});
+
+test("quoteBigIntegers only touches long integer JSON values", () => {
+ // Adjacent long ints in an array must BOTH be quoted (delimiter reuse).
+ assert.equal(
+ quoteBigIntegers('{"a":[2085320225776427457,2085116299223425392]}'),
+ '{"a":["2085320225776427457","2085116299223425392"]}',
+ );
+ // Short numbers, floats and digits inside strings are left alone.
+ assert.equal(quoteBigIntegers('{"n":42,"f":1.5}'), '{"n":42,"f":1.5}');
+ assert.equal(
+ quoteBigIntegers('{"s":"2085320225776427457"}'),
+ '{"s":"2085320225776427457"}',
+ );
+ // Still valid JSON afterwards.
+ assert.deepEqual(
+ JSON.parse(quoteBigIntegers('{"id":2085320225776427457,"n":7}')),
+ { id: "2085320225776427457", n: 7 },
+ );
+});
