// The X session broker: a persistent Playwright profile that holds a logged-in // X session and exports fresh cookies on demand. // // This is the highest-value half of the Playwright work, and it pays for itself // immediately: gallery-dl's worst flaw is that X cookies expire in days, and a // browser profile that stays logged in removes that problem WITHOUT changing // the fetch path. The user logs in by hand once — 2FA and captcha included, // which is precisely why this is a headed browser and not an automated login — // and the profile then re-exports cookies whenever gallery-dl needs them. // // Deliberately isolated in its own module (and its own browser context) so the // e2e harness — which is also Playwright — and the production fetcher never // share state. Note the mild recursion: Playwright drives the test suite AND is // a production dependency here; keeping the launch in one place is what keeps // that untangled. // // Playwright is already a dependency of both editor and export with browsers // installed, so this adds NO new dependency. It will NOT run in the minimal // Docker build container — post fetching is an editor-host concern, never a // build-time one. // // Release 16 slice XL: the Connect window is the operator's own browser // without the automation signals (xBrowser.ts), and the profile is one of two // login sources — the other is the operator's everyday browser, read directly // (`social.x.cookieSource`, xCookieSource.ts / xBrowserLogin.ts). import path from "node:path"; import { mkdir, readFile, rm, stat } from "node:fs/promises"; import { execa } from "execa"; import { writeFileAtomic } from "../lib/jsonFile-server"; import type { Paths } from "../lib/paths"; import { importPlaywright, type BrowserContextLike, type ChromiumLike, } from "./playwrightRuntime"; import { buildXBrowserLaunchOptions, describeXBrowser, findXBrowser, readXBrowserRecord, recordedXBrowser, writeXBrowserRecord, type XBrowserChoice, } from "./xBrowser"; // Where the persistent browser profile lives. One profile per instance: a // single X identity is all the archive needs. export function xProfileDir(paths: Paths): string { return path.join(paths.transcriptsDir, ".x-session", "profile"); } // The exported cookie jar, in Netscape cookies.txt format — the format // gallery-dl (and yt-dlp) read via `--cookies `. export function xCookieFile(paths: Paths): string { return path.join(paths.transcriptsDir, ".x-session", "cookies.txt"); } export type XSessionStatus = { // A profile directory exists on disk. hasProfile: boolean; // A cookie jar has been exported. hasCookies: boolean; // When the cookie jar was last written. cookiesUpdatedAt?: string; // True when the exported jar carries the session cookie X actually // authenticates with. A jar without it is present-but-useless. looksAuthenticated: boolean; }; // X's session cookie. Its presence is the only cheap local signal that a // profile is actually logged in. const AUTH_COOKIE = "auth_token"; export async function readXSessionStatus(paths: Paths): Promise { const profileDir = xProfileDir(paths); const cookieFile = xCookieFile(paths); const hasProfile = await exists(profileDir); let hasCookies = false; let cookiesUpdatedAt: string | undefined; let looksAuthenticated = false; try { const st = await stat(cookieFile); hasCookies = true; cookiesUpdatedAt = new Date(st.mtimeMs).toISOString(); const text = await readFile(cookieFile, "utf8"); looksAuthenticated = text.includes(AUTH_COOKIE); } catch { /* no jar yet */ } return { hasProfile, hasCookies, cookiesUpdatedAt, looksAuthenticated }; } async function exists(p: string): Promise { try { await stat(p); return true; } catch { return false; } } // A Playwright cookie, structurally typed so this module does not import // @playwright/test at the top level (it must stay importable on a host that // never installs browsers). type BrowserCookie = { name: string; value: string; domain: string; path: string; expires: number; // seconds since epoch; -1 for a session cookie httpOnly: boolean; secure: boolean; }; // Serialize cookies to the Netscape cookies.txt format gallery-dl reads. // Exported for testing — the format is fiddly (leading dot = include // subdomains, TRUE/FALSE literals, tab separators) and getting it wrong fails // silently as "not logged in". export function toNetscapeCookieFile(cookies: ReadonlyArray): string { const lines = [ "# Netscape HTTP Cookie File", "# Written by the Archilyzer X session broker. Do not edit by hand.", "", ]; for (const c of cookies) { const includeSubdomains = c.domain.startsWith(".") ? "TRUE" : "FALSE"; const expires = c.expires && c.expires > 0 ? Math.floor(c.expires) : 0; lines.push( [ c.domain, includeSubdomains, c.path || "/", c.secure ? "TRUE" : "FALSE", String(expires), c.name, c.value, ].join("\t"), ); } return lines.join("\n") + "\n"; } // Only X's own cookies are exported — the profile may hold unrelated ones and // handing those to a subprocess would leak them for no benefit. export function isXCookie(c: BrowserCookie): boolean { const d = c.domain.replace(/^\./, "").toLowerCase(); return d === "x.com" || d === "twitter.com" || d.endsWith(".x.com") || d.endsWith(".twitter.com"); } // ` --version` ("Chromium 153.0.8010.47 Arch Linux"), for the log and the // profile's record. Best effort: a browser that does not answer in 5 s is // simply not versioned. async function browserVersion(b: XBrowserChoice): Promise { if (b.kind !== "system") return undefined; try { const res = await execa(b.executablePath, ["--version"], { reject: false, timeout: 5_000 }); const line = `${res.stdout ?? ""}`.trim().split("\n")[0]; return res.exitCode === 0 && line ? line : undefined; } catch { return undefined; } } function firstLine(err: unknown): string { return ((err as Error)?.message ?? String(err)).split("\n")[0]; } // Launch a HEADED browser against the persistent profile so a human can log in // (password, 2FA, captcha — all of it). Resolves once the caller closes the // window, then exports the cookie jar. // // The window is the operator's own browser without the automation signals // (xBrowser.ts says which, and why Google's sign-in refused the old one). The // executable is recorded in the profile, so a refresh that cannot open the // profile with the bundled build can use the one that wrote it. // // Playwright is imported dynamically: this module must stay importable on a // host with no browsers installed (the Docker build image), where only the // status/read helpers above are ever called. export async function connectXAccount( paths: Paths, opts: { onLog?: (line: string) => void; timeoutMs?: number; env?: Record; } = {}, ): Promise { const log = opts.onLog ?? (() => {}); const profileDir = xProfileDir(paths); await mkdir(profileDir, { recursive: true }); const browser = findXBrowser({ env: opts.env }); const version = await browserVersion(browser); const label = describeXBrowser(browser, version); const { chromium } = await importPlaywright(); log(`Opening ${label}. Log in to X, then close the window.`); let context: BrowserContextLike; try { context = await chromium.launchPersistentContext( profileDir, buildXBrowserLaunchOptions({ browser, headless: false, sandbox: true }), ); } catch (err) { // A host that cannot start Chromium's sandbox (no unprivileged user // namespaces, an AppArmor rule) still gets its window — with the // unsupported-flag bar a system Chrome draws for `--no-sandbox`. log(`The sandboxed launch failed (${firstLine(err)}); opening without the sandbox.`); context = await chromium.launchPersistentContext( profileDir, buildXBrowserLaunchOptions({ browser, headless: false, sandbox: false }), ); } await writeXBrowserRecord(profileDir, browser, version).catch((err) => { log(`Could not record the browser in the profile: ${firstLine(err)}`); }); try { const page = context.pages()[0] ?? (await context.newPage()); await page.goto("https://x.com/login", { waitUntil: "domcontentloaded" }); // Wait for the human. The window closing is the "done" signal — there is no // reliable DOM marker for "logged in" that survives X's redesigns. await new Promise((resolve) => { let settled = false; const finish = () => { if (settled) return; settled = true; resolve(); }; context.on("close", finish); if (opts.timeoutMs) setTimeout(finish, opts.timeoutMs); }); const cookies = (await context.cookies()) as BrowserCookie[]; await writeCookieJar(paths, cookies.filter(isXCookie), log); } finally { await context.close().catch(() => {}); } return { ...(await readXSessionStatus(paths)), browser: label }; } // THE PROFILE, OPENED HEADLESS — the Playwright fallback fetcher (profile // source) comes through here, and the refresh below reads the profile the same // way. The bundled build first (it never logs in, and its headless shell needs // no display), without the automation signals; when it cannot open the profile // and the profile records a system executable, that one, headless. // // `chromium` is injectable so the choice of browser is testable without one. type ProfileLaunchOpts = { onLog?: (line: string) => void; chromium?: ChromiumLike }; const BUNDLED: XBrowserChoice = { kind: "bundled" }; function openProfile( chromium: ChromiumLike, profileDir: string, browser: XBrowserChoice, ): Promise { return chromium.launchPersistentContext( profileDir, buildXBrowserLaunchOptions({ browser, headless: true }), ); } export async function launchXProfile( paths: Paths, opts: ProfileLaunchOpts = {}, ): Promise { const log = opts.onLog ?? (() => {}); const profileDir = xProfileDir(paths); const chromium = opts.chromium ?? (await importPlaywright()).chromium; try { const context = await openProfile(chromium, profileDir, BUNDLED); log("Opened the X session profile with Playwright's bundled Chromium (headless)."); return context; } catch (err) { const record = await readXBrowserRecord(profileDir); const recorded = recordedXBrowser(record); if (!recorded) throw err; log( `Playwright's bundled Chromium could not open the profile (${firstLine(err)}); ` + `using ${describeXBrowser(recorded, record?.version)}, headless.`, ); return openProfile(chromium, profileDir, recorded); } } // One headless pass: open the profile, touch x.com/home (X rotates/renews the // session cookies on a visit, which is the whole point of keeping a live // profile), read X's cookies, close. async function exportProfileCookies( chromium: ChromiumLike, profileDir: string, browser: XBrowserChoice, log: (line: string) => void, ): Promise { const context = await openProfile(chromium, profileDir, browser); try { const page = context.pages()[0] ?? (await context.newPage()); await page .goto("https://x.com/home", { waitUntil: "domcontentloaded", timeout: 45_000 }) .catch(() => { log("Could not load x.com/home; exporting whatever the profile holds."); }); return ((await context.cookies()) as BrowserCookie[]).filter(isXCookie); } finally { await context.close().catch(() => {}); } } const hasAuthCookie = (cookies: ReadonlyArray) => cookies.some((c) => c.name === AUTH_COOKIE); // Re-export cookies from the stored profile WITHOUT any human interaction — // this is the call gallery-dl's cookie refresh actually uses. Runs headless. // // A LOGGED-IN JAR IS NEVER REPLACED BY ONE WITHOUT A LOGIN. A profile written // by a newer system browser can open in the bundled build without an error and // without its cookies (a store that build cannot read). So when the bundled // read comes back with no auth_token while the jar holds one, the profile is // read again with the browser that wrote it (the record); and when no read // finds the login, the jar is kept and the refresh refuses, saying so. export async function refreshXCookies( paths: Paths, opts: ProfileLaunchOpts = {}, ): Promise { const log = opts.onLog ?? (() => {}); const profileDir = xProfileDir(paths); if (!(await exists(profileDir))) { throw new Error( "No X session profile yet — run the headed 'Connect X account' flow first.", ); } const chromium = opts.chromium ?? (await importPlaywright()).chromium; const before = await readXSessionStatus(paths); const record = await readXBrowserRecord(profileDir); const recorded = recordedXBrowser(record); const recordedLabel = recorded ? describeXBrowser(recorded, record?.version) : ""; let cookies: BrowserCookie[]; let readWithRecorded = false; try { cookies = await exportProfileCookies(chromium, profileDir, BUNDLED, log); log("Read the X session profile with Playwright's bundled Chromium (headless)."); } catch (err) { if (!recorded) throw err; log( `Playwright's bundled Chromium could not open the profile (${firstLine(err)}); ` + `using ${recordedLabel}, headless.`, ); cookies = await exportProfileCookies(chromium, profileDir, recorded, log); readWithRecorded = true; } if (!hasAuthCookie(cookies) && before.looksAuthenticated && recorded && !readWithRecorded) { log( "Playwright's bundled Chromium read no X login from a profile whose exported jar holds " + `one; reading it again with ${recordedLabel}, headless.`, ); cookies = await exportProfileCookies(chromium, profileDir, recorded, log); } if (!hasAuthCookie(cookies) && before.looksAuthenticated) { throw new Error( "The profile gave no X login (no auth_token), so the logged-in cookie jar" + (before.cookiesUpdatedAt ? ` exported ${before.cookiesUpdatedAt}` : "") + " was kept, not replaced. If the X session has ended, Connect again (or Forget session).", ); } await writeCookieJar(paths, cookies, log); return readXSessionStatus(paths); } async function writeCookieJar( paths: Paths, cookies: BrowserCookie[], log: (line: string) => void, ): Promise { const file = xCookieFile(paths); // mode is applied as the temp is CREATED, before the rename: the jar is // never world-readable, not even for an instant. await writeFileAtomic(file, toNetscapeCookieFile(cookies), { mkdir: true, mode: 0o600, }); const authed = cookies.some((c) => c.name === AUTH_COOKIE); log( `Exported ${cookies.length} X cookie(s) to ${file}` + (authed ? "" : " — WARNING: no auth_token, the session is not logged in"), ); } // Forget the stored session entirely (profile + exported jar). export async function clearXSession(paths: Paths): Promise { await rm(path.dirname(xCookieFile(paths)), { recursive: true, force: true }); }