// THE OPERATOR'S BROWSER LOGIN, read for the X paths that cannot read it // themselves (release 16 slice XL). // // gallery-dl reads a browser's cookie store on its own (`--cookies-from-browser // `, xGalleryDlFetcher.ts) — every browser it supports, Chromium's // encrypted store included. Two things here need the same login without // gallery-dl: the Playwright fallback fetcher (it hands the cookies to a fresh // headless context) and the Settings section's "Check" (is an X login visible // at all, and when was it last used). Both go through `xCookiesFromBrowser`. // // FIREFOX ONLY. Its `cookies.sqlite` is plain SQLite with plain values. The // Chromium family encrypts each value with a key from the desktop keyring // (libsecret / KWallet) and is left to gallery-dl and yt-dlp, which carry that // decryption; for such a spec the readers here say so instead of guessing. // // READ-ONLY, ALWAYS. The browser's profile is never opened in place and never // written: `cookies.sqlite` and its `-wal` (Firefox writes ahead, so a fresh // login may live only in the WAL) are copied into a private temp dir, read // there, and the dir is removed. Only X's own rows are selected, so no other // site's cookies leave SQLite. The store is found much as gallery-dl finds it // (a profile path or name from the spec, else the most recently modified // `cookies.sqlite` under Firefox's profile roots) — not exactly: see // findFirefoxCookieDb. The CONTAINER is read exactly as gallery-dl reads it // (firefoxContainerScope), and so is the domain a login counts on (x.com). import { copyFile, mkdir, mkdtemp, readdir, readFile, rm, stat } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import type { Paths } from "../lib/paths"; import { loadSqlite } from "./nodeSqlite"; import { readXSessionStatus } from "./xSessionBroker"; import { parseBrowserSpec, resolveXCookieSource, SELF_READ_BROWSERS, xCookieSourceView, type BrowserSpec, type ResolvedXCookieSource, type XCookieSource, type XCookieSourceView, } from "./xCookieSource"; // X's session cookie: the one whose presence means "logged in". export const X_AUTH_COOKIE = "auth_token"; // A cookie as read from a browser's store, in the shape Playwright's // `addCookies` and the broker's cookies.txt writer both take. export type XBrowserCookie = { name: string; value: string; domain: string; path: string; // Seconds since the epoch; -1 for a session cookie. expires: number; httpOnly: boolean; secure: boolean; sameSite?: "Strict" | "Lax" | "None"; // When the browser last sent it, ms since the epoch (Firefox's lastAccessed). lastAccessedMs?: number; }; // --- Finding the store ------------------------------------------------------- // Where Firefox keeps its profiles: the XDG location newer releases use, the // classic one, Snap, Flatpak, macOS. gallery-dl's list differs slightly (it // honours $XDG_CONFIG_HOME and has a second Flatpak root), and for a profile // NAME it takes the first root holding `/cookies.sqlite` where this takes // the newest; on a stock Linux Firefox the two pick the same store. export function firefoxProfileRoots(home: string): string[] { return [ path.join(home, ".config", "mozilla", "firefox"), path.join(home, ".mozilla", "firefox"), path.join(home, "snap", "firefox", "common", ".mozilla", "firefox"), path.join(home, ".var", "app", "org.mozilla.firefox", ".mozilla", "firefox"), path.join(home, "Library", "Application Support", "Firefox", "Profiles"), ]; } const COOKIE_DB = "cookies.sqlite"; // The newest cookies.sqlite at most `depth` directories below `dir`. A profile // keeps it at its own top level, so two levels covers a root of profiles and a // profile given directly, without walking a profile's storage tree. async function newestCookieDb( dir: string, depth: number, ): Promise<{ file: string; mtimeMs: number } | undefined> { let entries; try { entries = await readdir(dir, { withFileTypes: true }); } catch { return undefined; } let best: { file: string; mtimeMs: number } | undefined; for (const e of entries) { const p = path.join(dir, e.name); if (e.name === COOKIE_DB && e.isFile()) { const st = await stat(p).catch(() => null); if (st && (!best || st.mtimeMs > best.mtimeMs)) best = { file: p, mtimeMs: st.mtimeMs }; } else if (depth > 0 && (e.isDirectory() || e.isSymbolicLink())) { const hit = await newestCookieDb(p, depth - 1); if (hit && (!best || hit.mtimeMs > best.mtimeMs)) best = hit; } } return best; } function expandHome(p: string, home: string): string { return p === "~" ? home : p.startsWith("~/") ? path.join(home, p.slice(2)) : p; } export async function findFirefoxCookieDb( spec: BrowserSpec, home: string, ): Promise<{ file?: string; searched: string[] }> { const profile = spec.profile ? expandHome(spec.profile, home) : undefined; const searched = profile ? profile.includes("/") || profile.includes(path.sep) ? [profile] : firefoxProfileRoots(home).map((r) => path.join(r, profile)) : firefoxProfileRoots(home); let best: { file: string; mtimeMs: number } | undefined; for (const root of searched) { const hit = await newestCookieDb(root, 2); if (hit && (!best || hit.mtimeMs > best.mtimeMs)) best = hit; } return { file: best?.file, searched }; } // --- Reading it ---------------------------------------------------------------- // WHICH CONTAINER, as gallery-dl reads it (gallery-dl/cookies.py, // `_firefox_cookies_database`; 1.32.9) — gallery-dl is the fetcher this login // feeds, so the readers here see what it sees: // no `::CONTAINER`, or `::none` — only cookies that belong to no container // (gallery-dl's default; yt-dlp's differs); // `::all` — every container, no filter; // `::NAME` — the container containers.json names so, // matched as gallery-dl matches it (its // `name`, else its `l10nId`, else `l10nID`; // case-sensitive). export type ContainerScope = { kind: "none" } | { kind: "all" } | { kind: "id"; id: number }; function extr(text: string, begin: string, end: string): string { const i = text.indexOf(begin); if (i < 0) return ""; const from = i + begin.length; const j = text.indexOf(end, from); return j < 0 ? "" : text.slice(from, j); } export async function firefoxContainerScope( dbFile: string, container: string | undefined, ): Promise { if (!container || container === "none") return { kind: "none" }; if (container === "all") return { kind: "all" }; let identities: Array> = []; try { const raw = JSON.parse( await readFile(path.join(path.dirname(dbFile), "containers.json"), "utf8"), ) as { identities?: unknown }; if (Array.isArray(raw.identities)) identities = raw.identities as Array>; } catch { /* no containers.json: no container can match */ } const str = (v: unknown) => (typeof v === "string" && v ? v : undefined); const hit = identities.find((c) => { const name = str(c.name); if (name) return name === container; const l10nId = str(c.l10nId); if (l10nId) { return ( (l10nId.startsWith("user-context-") && l10nId.slice(13) === container) || l10nId.slice(l10nId.lastIndexOf("-") + 1) === container ); } const l10nID = str(c.l10nID); return l10nID ? extr(l10nID, "userContext", ".label") === container : false; }); if (typeof hit?.userContextId !== "number") { throw new Error(`No Firefox container named "${container}" in containers.json`); } return { kind: "id", id: hit.userContextId }; } // gallery-dl's SQL, in JS: `NOT INSTR(originAttributes,'userContextId=')` for // no container, `LIKE '%userContextId=' OR LIKE '%userContextId=&%'` // for one. export function inContainerScope(originAttributes: string, scope: ContainerScope): boolean { if (scope.kind === "all") return true; if (scope.kind === "none") return !originAttributes.includes("userContextId="); const tag = `userContextId=${scope.id}`; return originAttributes.endsWith(tag) || originAttributes.includes(`${tag}&`); } const X_HOSTS = "(host = 'x.com' OR host = '.x.com' OR host LIKE '%.x.com' OR " + "host = 'twitter.com' OR host = '.twitter.com' OR host LIKE '%.twitter.com')"; // X's own domain. gallery-dl's twitter extractor looks its `auth_token` up on // `.x.com` (extractor/twitter.py), so a login counts only there; twitter.com's // is the old domain's, read but not counted. export function isXComHost(host: string): boolean { const h = host.replace(/^\./, "").toLowerCase(); return h === "x.com" || h.endsWith(".x.com"); } export function isXComAuth(c: { name: string; domain: string }): boolean { return c.name === X_AUTH_COOKIE && isXComHost(c.domain); } export function isOldDomainAuth(c: { name: string; domain: string }): boolean { if (c.name !== X_AUTH_COOKIE) return false; const h = c.domain.replace(/^\./, "").toLowerCase(); return h === "twitter.com" || h.endsWith(".twitter.com"); } function num(v: unknown): number | undefined { if (typeof v === "number" && Number.isFinite(v)) return v; if (typeof v === "bigint") return Number(v); return undefined; } // Firefox has stored `expiry` in seconds; a value too large for seconds is read // as milliseconds, so a store that changes unit still reads right. export function firefoxExpirySeconds(v: unknown): number { const n = num(v); if (!n || n <= 0) return -1; return Math.floor(n > 1e11 ? n / 1000 : n); } // `lastAccessed` is PRTime: microseconds since the epoch. function firefoxTimeMs(v: unknown): number | undefined { const n = num(v); if (!n || n <= 0) return undefined; return Math.floor(n > 1e14 ? n / 1000 : n); } function firefoxSameSite(v: unknown, secure: boolean): XBrowserCookie["sameSite"] { const n = num(v); if (n === 2) return "Strict"; if (n === 1) return "Lax"; // SameSite=None is only kept by a browser on a secure cookie. if (n === 0) return secure ? "None" : "Lax"; return undefined; } type StoreRow = { cookie: XBrowserCookie; originAttributes: string }; function readRows( DatabaseSync: Awaited>["DatabaseSync"], file: string, nowS: number, ): StoreRow[] { const db = new DatabaseSync(file); try { const rows = db.prepare(`SELECT * FROM moz_cookies WHERE ${X_HOSTS}`).all() as Array< Record >; const out: StoreRow[] = []; for (const r of rows) { if (typeof r.name !== "string" || typeof r.host !== "string") continue; const secure = num(r.isSecure) === 1; const expires = firefoxExpirySeconds(r.expiry); if (expires > 0 && expires < nowS) continue; const cookie: XBrowserCookie = { name: r.name, value: typeof r.value === "string" ? r.value : String(r.value ?? ""), domain: r.host, path: typeof r.path === "string" && r.path ? r.path : "/", expires, httpOnly: num(r.isHttpOnly) === 1, secure, }; const sameSite = firefoxSameSite(r.sameSite, secure); if (sameSite) cookie.sameSite = sameSite; const last = firefoxTimeMs(r.lastAccessed); if (last) cookie.lastAccessedMs = last; out.push({ cookie, originAttributes: typeof r.originAttributes === "string" ? r.originAttributes : "", }); } return out; } finally { db.close(); } } // ONE READ OF THE STORE, two views of X's rows: // `cookies` — with the WAL: what Firefox holds now. The Playwright // fallback uses these (a fresh login is in the WAL until // Firefox checkpoints). // `mainFile` — the main file alone: what gallery-dl sees. It opens // cookies.sqlite `mode=ro&immutable=1`, which ignores the WAL // (and its fallback copies the main file only). // Both in the container scope; `otherScopeAuth` says whether an x.com // auth_token exists OUTSIDE it (in a container the spec did not name). export type FirefoxXRead = { cookies: XBrowserCookie[]; mainFile: XBrowserCookie[]; otherScopeAuth: boolean; }; export async function readFirefoxXStore( dbFile: string, opts: { container?: string; tmpRoot?: string; now?: number } = {}, ): Promise { const { DatabaseSync } = await loadSqlite(); const scope = await firefoxContainerScope(dbFile, opts.container); // mkdtemp makes the dir 0700: the copies are readable by this user only. const tmp = await mkdtemp(path.join(opts.tmpRoot ?? os.tmpdir(), "archilyzer-xcookies-")); try { // The db and its WAL back to back (the shortest window for a checkpoint // between them), then a second copy of the db alone, taken from the first. const withWal = path.join(tmp, "wal", COOKIE_DB); const mainOnly = path.join(tmp, "main", COOKIE_DB); await mkdir(path.dirname(withWal)); await mkdir(path.dirname(mainOnly)); await copyFile(dbFile, withWal); await copyFile(`${dbFile}-wal`, `${withWal}-wal`).catch((err: NodeJS.ErrnoException) => { if (err.code !== "ENOENT") throw err; }); await copyFile(withWal, mainOnly); const nowS = (opts.now ?? Date.now()) / 1000; const walRows = readRows(DatabaseSync, withWal, nowS); const mainRows = readRows(DatabaseSync, mainOnly, nowS); const scoped = (rows: StoreRow[]) => rows.filter((r) => inContainerScope(r.originAttributes, scope)).map((r) => r.cookie); return { cookies: scoped(walRows), mainFile: scoped(mainRows), otherScopeAuth: walRows.some( (r) => !inContainerScope(r.originAttributes, scope) && isXComAuth(r.cookie), ), }; } finally { await rm(tmp, { recursive: true, force: true }); } } // X's cookies as Firefox holds them now (the WAL included), in the spec's // container scope. export async function readFirefoxXCookies( dbFile: string, opts: { container?: string; tmpRoot?: string; now?: number } = {}, ): Promise { return (await readFirefoxXStore(dbFile, opts)).cookies; } export type BrowserCookieRead = | ({ ok: true; browser: string; store: string; // The spec's container, as given (undefined = outside every container). container?: string; } & FirefoxXRead) | { ok: false; // no-spec: cookiesFromBrowser is empty. unsupported: a browser this repo // does not read itself (gallery-dl still does). not-found: no store in // the places searched. unreadable: the store or the spec could not be read. reason: "no-spec" | "unsupported" | "not-found" | "unreadable"; message: string; }; // THE SHARED READ: the X cookies of the browser `spec` names, fresh on every // call. `spec` is the resolved one — a channel's own cookiesFromBrowser over // the global — which is why this takes the spec and not the settings. export async function xCookiesFromBrowser( spec: string | undefined, opts: { home?: string; tmpRoot?: string; now?: number } = {}, ): Promise { const trimmed = spec?.trim(); if (!trimmed) { return { ok: false, reason: "no-spec", message: "cookiesFromBrowser is empty, so there is no browser to read an X login from.", }; } const parsed = parseBrowserSpec(trimmed); if (!parsed) { return { ok: false, reason: "unreadable", message: `Could not read the browser spec "${trimmed}".` }; } if (!SELF_READ_BROWSERS.includes(parsed.browser)) { return { ok: false, reason: "unsupported", message: `${parsed.browser} keeps its cookies encrypted with the desktop keyring; only gallery-dl ` + "and yt-dlp read them (gallery-dl does, on every X fetch). This check and the " + "Playwright fallback read Firefox only.", }; } const home = opts.home ?? os.homedir(); const found = await findFirefoxCookieDb(parsed, home); if (!found.file) { return { ok: false, reason: "not-found", message: `No Firefox cookie store found (looked in ${found.searched.join(", ")}).`, }; } try { const read = await readFirefoxXStore(found.file, { container: parsed.container, tmpRoot: opts.tmpRoot, now: opts.now, }); return { ok: true, browser: parsed.browser, store: found.file, ...(parsed.container ? { container: parsed.container } : {}), ...read, }; } catch (err) { return { ok: false, reason: "unreadable", message: `Could not read ${found.file}: ${(err as Error).message}`, }; } } // --- The source in use, and whether it holds a login -------------------------- // The slice of SiteSettings this module reads. Structural, so settings.ts is // not imported here. export type XLoginSettings = { cookiesFromBrowser?: string; social?: { x?: { cookieSource?: XCookieSource } }; }; // The source for this host now: reads whether the broker's profile is // connected. `browserSpec` defaults to the global cookiesFromBrowser; the fetch // controller passes a channel's own when it has one. export async function resolveXCookieSourceFor( paths: Paths, settings: XLoginSettings, browserSpec: string | undefined = settings.cookiesFromBrowser, ): Promise { const profile = await readXSessionStatus(paths); return resolveXCookieSource({ stored: settings.social?.x?.cookieSource, browserSpec, profileConnected: profile.looksAuthenticated, }); } export type XLoginStatus = XCookieSourceView & { // Whether an auth_token cookie for x.com is visible in the source: null when // the source could not be read (no spec, an unsupported browser, no store). authTokenVisible: boolean | null; // When the login was last seen: for the browser source, when the browser // last sent its auth_token (the store's lastAccessed); for the profile, when // the jar was last exported. lastSeenAt?: string; // Browser source only. True when the x.com auth_token is in Firefox's // write-ahead log alone: gallery-dl, which reads the main file, does not see // it until Firefox checkpoints. walOnly?: boolean; // Browser source only. An auth_token for twitter.com (the old domain) is // present; it is reported, never counted — gallery-dl looks on x.com. oldDomainCookie?: boolean; // Browser source only. An x.com auth_token sits in a Firefox container the // spec does not read. otherContainerLogin?: boolean; checkedAt: string; // A sentence or three for the Settings section. summary: string; }; function when(iso: string): string { return iso.replace("T", " ").replace(/\.\d+Z$/, " UTC"); } export async function readXLoginStatus( paths: Paths, settings: XLoginSettings, opts: { home?: string; tmpRoot?: string; now?: number } = {}, ): Promise { const checkedAt = new Date(opts.now ?? Date.now()).toISOString(); const resolved = await resolveXCookieSourceFor(paths, settings); const base = { ...xCookieSourceView(resolved), checkedAt }; if (resolved.source === "profile") { const profile = await readXSessionStatus(paths); if (profile.looksAuthenticated) { return { ...base, authTokenVisible: true, lastSeenAt: profile.cookiesUpdatedAt, summary: "The connected profile holds an X login" + (profile.cookiesUpdatedAt ? ` (cookies exported ${when(profile.cookiesUpdatedAt)}).` : "."), }; } return { ...base, authTokenVisible: false, lastSeenAt: profile.cookiesUpdatedAt, summary: profile.hasProfile ? "The profile is not logged in to X (no auth_token in its exported cookies). Connect again, or choose the browser login." : "No profile is connected. Connect an X account, or choose the browser login.", }; } const read = await xCookiesFromBrowser(resolved.browserSpec, opts); if (!read.ok) { return { ...base, authTokenVisible: null, summary: read.message }; } // Where the read looked, as gallery-dl looks (see firefoxContainerScope). const scope = !read.container || read.container === "none" ? `${read.browser}, outside its containers (where gallery-dl reads)` : read.container === "all" ? `${read.browser}, in any container` : `${read.browser}'s container "${read.container}"`; const newest = (list: XBrowserCookie[]) => list.filter(isXComAuth).sort((a, b) => (b.lastAccessedMs ?? 0) - (a.lastAccessedMs ?? 0))[0]; const auth = newest(read.cookies); const oldDomainCookie = read.cookies.some(isOldDomainAuth); const oldDomainLine = oldDomainCookie ? " An old-domain cookie is present too (an auth_token for twitter.com); gallery-dl does not use it." : ""; if (!auth) { return { ...base, authTokenVisible: false, oldDomainCookie, otherContainerLogin: read.otherScopeAuth, summary: `No X login in ${scope}: no auth_token cookie for x.com.` + (read.otherScopeAuth ? ` One is in another Firefox container; gallery-dl reads it only when cookiesFromBrowser ` + `names that container (${read.browser}::) or ${read.browser}::all.` : " Log in to x.com in that browser.") + oldDomainLine, }; } const lastSeenAt = auth.lastAccessedMs ? new Date(auth.lastAccessedMs).toISOString() : undefined; const walOnly = !read.mainFile.some(isXComAuth); return { ...base, authTokenVisible: true, lastSeenAt, walOnly, oldDomainCookie, otherContainerLogin: read.otherScopeAuth, summary: `An X login is visible in ${scope}` + (lastSeenAt ? `; its auth_token was last used ${when(lastSeenAt)}.` : ".") + (walOnly ? " Seen in Firefox's write-ahead log only; gallery-dl will see it after Firefox checkpoints (closing Firefox does it)." : "") + oldDomainLine, }; }