// The SHARED X/Twitter → Post normalizer. // // Both X fetchers produce the same underlying payload: gallery-dl parses X's // GraphQL timeline responses and emits one metadata object per tweet, and the // Playwright fallback re-issues that same GraphQL call from inside an // authenticated page. So the fallback is a TRANSPORT swap, not a rewrite — // this module is the single place tweet shape is understood. // // Deliberately tolerant: X's payload has drifted repeatedly (legacy vs. // `__typename`-tagged results, `full_text` vs `text`, note-tweets for long // posts), and gallery-dl flattens some of it. Every field is probed through a // list of known aliases and a missing one degrades rather than throwing, so a // partial payload still yields a usable Post. import { postPermalink, uploadDateFromCreatedAt, type Post, type PostRef, } from "../lib/posts"; // A gallery-dl `--dump-json` / metadata-postprocessor record for one tweet, or // an entry lifted out of a raw GraphQL timeline response. Everything optional. export type XTweetRaw = Record; function str(v: unknown): string | undefined { return typeof v === "string" && v !== "" ? v : undefined; } function num(v: unknown): number | undefined { if (typeof v === "number" && Number.isFinite(v)) return v; // X sometimes serializes ids and counts as strings. if (typeof v === "string" && /^\d+$/.test(v)) return Number(v); return undefined; } function obj(v: unknown): Record | undefined { return v && typeof v === "object" && !Array.isArray(v) ? (v as Record) : undefined; } // A reference to another tweet, as a string id, or undefined when absent. // gallery-dl writes an unset reference as 0; a 64-bit id arrives as a string // (`quoteBigIntegers`), a short one as a number. function refId(v: unknown): string | undefined { if (typeof v === "string") return /^\d+$/.test(v) && !/^0+$/.test(v) ? v : undefined; if (typeof v === "number" && Number.isSafeInteger(v) && v > 0) return String(v); return undefined; } // First defined value among a record's alias keys. function pick(rec: XTweetRaw, keys: string[]): unknown { for (const k of keys) { const v = rec[k]; if (v !== undefined && v !== null && v !== "") return v; } return undefined; } // X ids are 64-bit and MUST stay strings — JSON numbers lose precision above // 2^53, which silently corrupts a tweet id. export function xIdOf(rec: XTweetRaw): string | undefined { const raw = pick(rec, ["tweet_id", "id_str", "rest_id", "id", "conversation_id"]); if (typeof raw === "string" && raw !== "") return raw; if (typeof raw === "number" && Number.isFinite(raw)) return String(raw); return undefined; } // gallery-dl emits `date` as "YYYY-MM-DD HH:MM:SS" (UTC); raw X emits // `created_at` in the legacy "Wed Oct 10 20:19:24 +0000 2018" form. Normalize // both to ISO-8601. export function xCreatedAt(rec: XTweetRaw): string | undefined { const raw = pick(rec, ["date", "created_at", "createdAt"]); if (typeof raw !== "string" || !raw) return undefined; // Already ISO. if (/^\d{4}-\d{2}-\d{2}T/.test(raw)) { const ms = Date.parse(raw); return Number.isFinite(ms) ? new Date(ms).toISOString() : undefined; } // gallery-dl's "YYYY-MM-DD HH:MM:SS" is UTC but has no zone marker; adding // one avoids it being read as local time (which would shift the date). if (/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(raw)) { const ms = Date.parse(raw.replace(" ", "T") + "Z"); return Number.isFinite(ms) ? new Date(ms).toISOString() : undefined; } const ms = Date.parse(raw); return Number.isFinite(ms) ? new Date(ms).toISOString() : undefined; } // The author handle. gallery-dl nests it under `author` (the tweet's author) // and `user` (the timeline's owner — different for a retweet). export function xAuthor(rec: XTweetRaw): { handle: string; name?: string } { const author = obj(rec.author) ?? obj(rec.user); const handle = str(author?.name) ?? str(author?.screen_name) ?? str(rec.author_name) ?? str(rec.screen_name) ?? ""; const name = str(author?.nick) ?? str(author?.displayName) ?? str(author?.nickname); return { handle, ...(name ? { name } : {}) }; } // The tweet body. `content` is gallery-dl's field; raw X uses `full_text`, and // a long post puts its untruncated body under a note-tweet. export function xText(rec: XTweetRaw): string { const note = obj(obj(obj(rec.note_tweet)?.note_tweet_results)?.result); const noteText = str(note?.text); if (noteText) return noteText; return ( str(pick(rec, ["content", "full_text", "text"])) ?? "" ); } // Expanded outbound URLs. gallery-dl flattens entities; raw X nests them. export function xLinks(rec: XTweetRaw): string[] { const out = new Set(); const entities = obj(rec.entities); const urls = entities?.urls ?? (obj(rec.legacy)?.entities as Record | undefined)?.urls; if (Array.isArray(urls)) { for (const u of urls) { const e = obj(u); const href = str(e?.expanded_url) ?? str(e?.url); // Skip t.co self-links to the quoted tweet — the quote ref covers it. if (href && !/^https?:\/\/t\.co\//.test(href)) out.add(href); } } return [...out]; } function refFrom( id: string | undefined, handle: string | undefined, ): PostRef | undefined { if (!id) return undefined; const ref: PostRef = { platform: "twitter", id }; if (handle) { ref.author = handle; ref.url = postPermalink("twitter", handle, id); } else { ref.url = `https://x.com/i/status/${id}`; } return ref; } // Normalize one tweet record into a Post. Returns null when the record has no // resolvable id or timestamp — a shape we don't understand is skipped rather // than stored as a corrupt entry. export function normalizeXTweet( rec: XTweetRaw, channelSlug: string, ): Post | null { const id = xIdOf(rec); if (!id) return null; const createdAt = xCreatedAt(rec); if (!createdAt) return null; const text = xText(rec); // gallery-dl flattens a tweet's references into `reply_id`, `retweet_id` and // `quote_id` (plus `reply_to`, the replied-to handle), each 0 when unset. const replyToId = refId(pick(rec, ["in_reply_to_status_id_str", "in_reply_to_tweet_id", "in_reply_to_status_id"])) ?? refId(rec.reply_id); const replyToHandle = str( pick(rec, ["in_reply_to_screen_name", "in_reply_to_user", "reply_to"]), ); // gallery-dl's `quote_id` is the reverse link: it is set on a quoted tweet // (yielded only with its `quoted` option) and names the tweet QUOTING it, so // it is never read as this tweet's quoted id. const quotedRec = obj(rec.quoted) ?? obj(rec.quoted_status); const quotedId = quotedRec ? xIdOf(quotedRec) : refId(rec.quoted_status_id_str); const quotedHandle = quotedRec ? xAuthor(quotedRec).handle : undefined; // A gallery-dl retweet is one flat record: `tweet_id` is the retweet itself, // `retweet_id` the original, `author` the ORIGINAL's author and `user` the // timeline's owner — who is the one who posted the retweet. const retweetRec = obj(rec.retweeted_status) ?? obj(rec.retweet); const flatRetweetId = retweetRec ? undefined : refId(rec.retweet_id); const retweetId = retweetRec ? xIdOf(retweetRec) : flatRetweetId; const retweetHandle = retweetRec ? xAuthor(retweetRec).handle : flatRetweetId ? xAuthor(rec).handle || undefined : undefined; const isRepost = Boolean(retweetId) || rec.retweeted === true; const owner = obj(rec.user); const { handle, name } = flatRetweetId && owner ? xAuthor({ author: owner }) : xAuthor(rec); // X's conversation_id IS the thread root, which is exactly our threadId. const threadId = str(pick(rec, ["conversation_id", "conversation_id_str"])) ?? id; const post: Post = { id, slug: `${channelSlug}/${id}`, channelSlug, author: handle, createdAt, uploadDate: uploadDateFromCreatedAt(createdAt), text, url: postPermalink("twitter", handle, id), platform: "twitter", isReply: Boolean(replyToId), isRepost, links: xLinks(rec), threadId, }; if (name) post.authorName = name; const lang = str(rec.lang); if (lang && lang !== "und") post.lang = lang; const replyTo = refFrom(replyToId, replyToHandle); if (replyTo) post.replyTo = replyTo; const quoted = refFrom(quotedId, quotedHandle); if (quoted) post.quoted = quoted; const repostOf = refFrom(retweetId, retweetHandle); if (repostOf) post.repostOf = repostOf; // Media is counted, not archived (v1). const media = rec.media ?? obj(rec.extended_entities)?.media; if (Array.isArray(media) && media.length > 0) post.mediaCount = media.length; else { const count = num(rec.count); if (count && count > 0) post.mediaCount = count; } const engagement: NonNullable = {}; const likes = num(pick(rec, ["favorite_count", "like_count"])); const reposts = num(pick(rec, ["retweet_count"])); const replies = num(pick(rec, ["reply_count"])); const quotes = num(pick(rec, ["quote_count"])); if (likes !== undefined) engagement.likes = likes; if (reposts !== undefined) engagement.reposts = reposts; if (replies !== undefined) engagement.replies = replies; if (quotes !== undefined) engagement.quotes = quotes; if (Object.keys(engagement).length > 0) post.engagement = engagement; return post; } // Normalize a batch, dropping unparseable records and deduping by id (a // timeline page can repeat a tweet across a cursor boundary). export function normalizeXTweets( records: ReadonlyArray, channelSlug: string, ): Post[] { const byId = new Map(); for (const rec of records) { const post = normalizeXTweet(rec, channelSlug); if (post && !byId.has(post.id)) byId.set(post.id, post); } return [...byId.values()]; }