commit a39ab7add1b689bd21661054dba653888314a142
parent c9776bb5d55a59c8d8bedb5966aef61ea17007ea
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 1 Oct 2026 12:53:31 -0400
Merge r16/x-login (release 16 slice XL) — connecting an X account works: the Connect window is the operator's own browser without automation signals, and the fetchers can use the operator's browser login directly (social.x.cookieSource, gallery-dl's container semantics, a Check that says what gallery-dl will see); the refresh never replaces a logged-in jar with an empty one; reviewed SHIP
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
31 files changed, 2991 insertions(+), 88 deletions(-)
diff --git a/ENVIRONMENT.md b/ENVIRONMENT.md
@@ -70,6 +70,7 @@ Tokens, credentials and knobs a running process reads. Most configuration is not
| `OLLAMA_DIGEST_MODEL` | `qwen2.5:7b` | The ollama model the local digest lane asks for when settings name none. | common/lib/digestApps.ts |
| `CLAUDE_DIGEST_MODEL` | the CLI's default | The model the metered digest lane asks `claude` for when settings name none. | common/lib/digestApps.ts |
| `NITTER_INSTANCES` | a built-in list | Comma-separated Nitter instances for the X fallback fetcher, in order of preference. | common/social/xNitterFetcher.ts |
+| `ARCHILYZER_X_BROWSER` | the first of `chromium`, `google-chrome`, `google-chrome-stable`, `chrome` on PATH, else Playwright's bundled Chromium | The Chromium-family browser /settings' "Connect X account" opens (a path, or a name looked up on PATH). It is launched without the automation signals, in the X session profile; a value that is not an executable refuses the connect rather than opening another browser. | common/social/xBrowser.ts |
| `UMTOOL_URL` | unset (no link) | umtool's front door; when set, the video page links to it. | editor/app/channels/[slug]/videos/[id]/page.tsx |
| `TRANSCRIPT_SITE_URL` | — | MCP server: one published archive to read over HTTP. | mcp/src/sources.ts |
| `TRANSCRIPT_HUB_URL` | — | MCP server: a hub, federating every archive it lists. | mcp/src/sources.ts |
diff --git a/SETTINGS.md b/SETTINGS.md
@@ -19,6 +19,7 @@ A copied example PINS every default it spells — including each lane's `autoQue
| [`workers`](#workers) | `[]` |
| [`cookiesFromBrowser`](#cookiesfrombrowser) | `""` |
| [`cookieMode`](#cookiemode) | `"when-required"` |
+| [`social`](#social) | object — see below |
| [`sleepBetweenDownloadsSeconds`](#sleepbetweendownloadsseconds) | `10` |
| [`downloadFormat`](#downloadformat) | `"auto"` |
| [`minFreeDiskGB`](#minfreediskgb) | `5` |
@@ -157,6 +158,30 @@ How yt-dlp invocations use the configured cookies (see common/lib/cookiePolicy.t
Default: `"when-required"`
+## `social`
+
+Per-platform settings of the social-post fetchers. Today one key: where the X fetchers' login comes from (`social.x.cookieSource`, chosen in the X account session section of /settings). See common/social/xCookieSource.ts.
+
+#### `social`
+
+| Key | Default | Description |
+|---|---|---|
+| `x` | `{}` | X / Twitter. See `social.x` below. |
+
+#### `social.x`
+
+| Key | Default | Description |
+|---|---|---|
+| `cookieSource` | absent | Where the X fetchers' login comes from. `"browser"`: the operator's everyday browser, named by `cookiesFromBrowser` — gallery-dl is handed `--cookies-from-browser <spec>` and reads it on every run, and the Playwright fallback reads the same store (Firefox only; common/social/xBrowserLogin.ts), so the login lasts as long as the browser's. `"profile"`: the session broker's persistent profile ("Connect X account" on /settings) and the cookie jar it exports. ABSENT (the default) is resolved at read time, never stored: `"browser"` when `cookiesFromBrowser` is set and no profile is connected (no exported jar carrying an auth_token), else `"profile"`. The source, not `cookieMode`, governs the X fetchers. |
+
+Default:
+
+```json
+{
+ "x": {}
+}
+```
+
## `sleepBetweenDownloadsSeconds`
Pause (seconds) inserted between per-video yt-dlp invocations in managed batch downloads. yt-dlp's own `-t sleep` only paces requests within one invocation, so without this the managed loop hammers the source IP back-to-back. 0 disables. Per-channel override available.
diff --git a/common/controller/fetchPosts.ts b/common/controller/fetchPosts.ts
@@ -41,13 +41,17 @@ import "../social/xGalleryDlFetcher";
// only ever used when a channel opts into it via postFetcher: "x-playwright".
import "../social/xPlaywrightFetcher";
import "../social/xNitterFetcher";
+import {
+ resolveXCookieSourceFor,
+ type XLoginSettings,
+} from "../social/xBrowserLogin";
export type FetchPostsOptions = {
paths: Paths;
slug: string;
- // Global settings, for the cookie policy. Structural so settings.ts need not
- // be imported here.
- settings: CookiePolicyInputs;
+ // Global settings, for the cookie policy and the X login source. Structural
+ // so settings.ts need not be imported here.
+ settings: CookiePolicyInputs & XLoginSettings;
// Ignore the stored watermark and re-walk the account's full history. New
// posts are still deduped against the archive, so this is a safe repair
// operation rather than a duplicate-maker.
@@ -122,7 +126,15 @@ export async function fetchPosts(
opts.full || resumeCursor
? undefined
: ((await latestPostCreatedAt(channelRoot)) ?? undefined);
- const cookies = alwaysCookies(resolveCookiePolicy(settings, config));
+ const policy = resolveCookiePolicy(settings, config);
+ const cookies = alwaysCookies(policy);
+ // An X fetcher's login source (social.x.cookieSource, xCookieSource.ts),
+ // resolved against THIS channel's browser spec — its own cookiesFromBrowser
+ // over the global one, whatever the cookieMode.
+ const xLogin =
+ fetcher.platform === "twitter"
+ ? await resolveXCookieSourceFor(paths, settings, policy.cookies)
+ : undefined;
log(
`Fetching posts for ${slug} via ${fetcher.label} (@${handle})` +
@@ -154,6 +166,8 @@ export async function fetchPosts(
cursor: resumeCursor,
seenIds,
cookies,
+ cookieSource: xLogin?.source,
+ browserCookies: xLogin?.browserSpec,
limit: opts.limit,
signal: effectiveSignal,
onLog: log,
diff --git a/common/lib/envVars.ts b/common/lib/envVars.ts
@@ -106,6 +106,7 @@ const DECLARED: EnvVarDecl[] = [
{ name: "OLLAMA_DIGEST_MODEL", audience: "runtime", default: "`qwen2.5:7b`", readBy: "common/lib/digestApps.ts", doc: "The ollama model the local digest lane asks for when settings name none." },
{ name: "CLAUDE_DIGEST_MODEL", audience: "runtime", default: "the CLI's default", readBy: "common/lib/digestApps.ts", doc: "The model the metered digest lane asks `claude` for when settings name none." },
{ name: "NITTER_INSTANCES", audience: "runtime", default: "a built-in list", readBy: "common/social/xNitterFetcher.ts", doc: "Comma-separated Nitter instances for the X fallback fetcher, in order of preference." },
+ { name: "ARCHILYZER_X_BROWSER", audience: "runtime", default: "the first of `chromium`, `google-chrome`, `google-chrome-stable`, `chrome` on PATH, else Playwright's bundled Chromium", readBy: "common/social/xBrowser.ts", doc: "The Chromium-family browser /settings' \"Connect X account\" opens (a path, or a name looked up on PATH). It is launched without the automation signals, in the X session profile; a value that is not an executable refuses the connect rather than opening another browser." },
{ name: "UMTOOL_URL", audience: "runtime", default: "unset (no link)", readBy: "editor/app/channels/[slug]/videos/[id]/page.tsx", doc: "umtool's front door; when set, the video page links to it." },
{ name: "TRANSCRIPT_SITE_URL", audience: "runtime", default: "—", readBy: "mcp/src/sources.ts", doc: "MCP server: one published archive to read over HTTP." },
{ name: "TRANSCRIPT_HUB_URL", audience: "runtime", default: "—", readBy: "mcp/src/sources.ts", doc: "MCP server: a hub, federating every archive it lists." },
diff --git a/common/lib/settingsDocs.test.ts b/common/lib/settingsDocs.test.ts
@@ -33,7 +33,7 @@ test("the example parses back to the defaults", async () => {
assert.deepEqual(parsed, defaultSiteSettings());
});
-// EVERY FIELD, NOT ONLY THE 31 TOP-LEVEL ONES. The *_FIELD_DOCS records are
+// EVERY FIELD, NOT ONLY THE 32 TOP-LEVEL ONES. The *_FIELD_DOCS records are
// complete by type (FieldDocs<T> requires one entry per key); these two check
// the wiring — that every object-valued block has a key table, and that a
// block's table names every key its default actually carries.
diff --git a/common/lib/settingsDocs.ts b/common/lib/settingsDocs.ts
@@ -19,6 +19,8 @@ import {
DIGEST_SETTINGS_FIELD_DOCS,
SAVED_VIDEO_BACKUP_SETTINGS_FIELD_DOCS,
SOCIAL_LINK_FIELD_DOCS,
+ SOCIAL_SETTINGS_FIELD_DOCS,
+ X_SOCIAL_SETTINGS_FIELD_DOCS,
SYNC_SCHEDULER_SETTINGS_FIELD_DOCS,
defaultSiteSettings,
siteSettingsSchema,
@@ -160,6 +162,16 @@ export function blockTables(d: SiteSettings): Partial<Record<keyof SiteSettings,
docs: CHANNEL_AUTO_PAUSE_FIELD_DOCS,
},
],
+ social: [
+ { path: "social", docs: SOCIAL_SETTINGS_FIELD_DOCS, defaults: fromObject(d.social) },
+ {
+ path: "social.x",
+ docs: X_SOCIAL_SETTINGS_FIELD_DOCS,
+ // Absent from the default block: the source is resolved at read time
+ // and only a chosen one is written.
+ defaults: fromObject(d.social.x),
+ },
+ ],
socialLinks: [{ path: "socialLinks[]", docs: SOCIAL_LINK_FIELD_DOCS }],
savedVideoBackup: [
{
diff --git a/common/lib/settingsSchema.test.ts b/common/lib/settingsSchema.test.ts
@@ -35,6 +35,7 @@ import type {
ReportDebouncePreset,
SavedVideoBackupSettings,
SocialLink,
+ SocialSettings,
SyncSchedulerSettings,
} from "./settingsSchema";
@@ -62,6 +63,8 @@ type PreSchemaSiteSettings = {
workers: Worker[];
cookiesFromBrowser: string;
cookieMode: CookieMode;
+ // Release 16 slice XL — the one key it adds.
+ social: SocialSettings;
sleepBetweenDownloadsSeconds: number;
downloadFormat: DownloadFormatPreset;
minFreeDiskGB: number;
@@ -92,7 +95,7 @@ type PreSchemaSiteSettings = {
type Same<A, B> = [A] extends [B] ? ([B] extends [A] ? true : false) : false;
const shapeUnchanged: Same<SiteSettings, PreSchemaSiteSettings> = true;
-test("SiteSettings keeps its 31 fields, in file order", () => {
+test("SiteSettings keeps its 32 fields, in file order", () => {
assert.equal(shapeUnchanged, true);
assert.deepEqual(Object.keys(siteSettingsSchema.shape), [
"adminTitle",
@@ -102,6 +105,7 @@ test("SiteSettings keeps its 31 fields, in file order", () => {
"workers",
"cookiesFromBrowser",
"cookieMode",
+ "social",
"sleepBetweenDownloadsSeconds",
"downloadFormat",
"minFreeDiskGB",
@@ -154,6 +158,22 @@ test("defaults() and defaultSiteSettings() are the schema's answer for an empty
assert.deepEqual(d.archiveStorage, { bucket: "", publicBaseUrl: "" });
assert.equal(d.skipLiveDownloads, true);
assert.equal(d.inlineTranscribeOnFallback, false);
+ // The X login source is resolved at read time, so the default stores none.
+ assert.deepEqual(d.social, { x: {} });
+});
+
+test("social.x.cookieSource keeps a known source and drops anything else", () => {
+ const parse = (social: unknown) => siteSettingsSchema.parse({ social }).social;
+ assert.deepEqual(parse({ x: { cookieSource: "browser" } }), { x: { cookieSource: "browser" } });
+ assert.deepEqual(parse({ x: { cookieSource: "profile" } }), { x: { cookieSource: "profile" } });
+ // An unknown source reads as absent — the read-time default — never a throw.
+ assert.deepEqual(parse({ x: { cookieSource: "chrome" } }), { x: {} });
+ assert.deepEqual(parse({ x: { cookieSource: "browser", extra: 1 }, bsky: {} }), {
+ x: { cookieSource: "browser" },
+ });
+ for (const bad of [null, "browser", [], 3, { x: "browser" }, { x: [] }]) {
+ assert.deepEqual(parse(bad), { x: {} }, JSON.stringify(bad));
+ }
});
// ── THE CLAMP BOUNDARIES ─────────────────────────────────────────────────────
diff --git a/common/lib/settingsSchema.ts b/common/lib/settingsSchema.ts
@@ -89,8 +89,34 @@ import {
type DigestTimestampMode,
} from "./digest";
import type { FieldDocs } from "./fieldDocs";
+import {
+ sanitizeSocial,
+ type SocialSettings,
+ type XSocialSettings,
+} from "../social/xCookieSource";
export type { Worker } from "./workers";
+export type { SocialSettings, XSocialSettings } from "../social/xCookieSource";
+
+// The `social` block (release 16 slice XL). The types and the coercion live
+// with the source they choose, in social/xCookieSource.ts (pure, client-safe);
+// the documentation lives here with every other block's.
+export const SOCIAL_SETTINGS_FIELD_DOCS: FieldDocs<SocialSettings> = {
+ x: "X / Twitter. See `social.x` below.",
+};
+
+export const X_SOCIAL_SETTINGS_FIELD_DOCS: FieldDocs<XSocialSettings> = {
+ cookieSource:
+ "Where the X fetchers' login comes from. `\"browser\"`: the operator's everyday browser, " +
+ "named by `cookiesFromBrowser` — gallery-dl is handed `--cookies-from-browser <spec>` and " +
+ "reads it on every run, and the Playwright fallback reads the same store (Firefox only; " +
+ "common/social/xBrowserLogin.ts), so the login lasts as long as the browser's. " +
+ "`\"profile\"`: the session broker's persistent profile (\"Connect X account\" on /settings) " +
+ "and the cookie jar it exports. ABSENT (the default) is resolved at read time, never " +
+ "stored: `\"browser\"` when `cookiesFromBrowser` is set and no profile is connected (no " +
+ "exported jar carrying an auth_token), else `\"profile\"`. The source, not `cookieMode`, " +
+ "governs the X fetchers.",
+};
export type { AutoQueueSettings } from "./autoQueueTypes";
export type { ChannelPriority } from "./channelPriority";
@@ -1450,6 +1476,9 @@ export const siteSettingsSchema = z.object({
cookieMode: settingsField((v): CookieMode => (isCookieMode(v) ? v : DEFAULT_COOKIE_MODE)).describe(
"How yt-dlp invocations use the configured cookies (see common/lib/cookiePolicy.ts): \"always\" passes them on every invocation, \"when-required\" (default; the historical behavior) only to retry an auth/age failure, \"defer\" never in normal runs — auth-gated videos are excluded from batches and collected into the per-channel \"Needs cookies\" bucket for a manual cookie run. Per-channel override available (ChannelConfig.cookieMode).",
),
+ social: settingsField((v): SocialSettings => sanitizeSocial(v)).describe(
+ "Per-platform settings of the social-post fetchers. Today one key: where the X fetchers' login comes from (`social.x.cookieSource`, chosen in the X account session section of /settings). See common/social/xCookieSource.ts.",
+ ),
sleepBetweenDownloadsSeconds: settingsField((v): number => clampSleepBetweenDownloadsSeconds(v)).describe(
"Pause (seconds) inserted between per-video yt-dlp invocations in managed batch downloads. yt-dlp's own `-t sleep` only paces requests within one invocation, so without this the managed loop hammers the source IP back-to-back. 0 disables. Per-channel override available.",
),
diff --git a/common/social/__fixtures__/firefoxCookieStore.ts b/common/social/__fixtures__/firefoxCookieStore.ts
@@ -0,0 +1,90 @@
+// A FIREFOX COOKIE STORE FOR TESTS — `cookies.sqlite` with Firefox's
+// `moz_cookies` table, written through node:sqlite. Used by
+// social/xBrowserLogin.test.ts and the editor's x-session e2e, so neither
+// reads a real browser profile. Test-only: nothing outside a test imports it.
+
+import { mkdir } from "node:fs/promises";
+import path from "node:path";
+import { loadSqlite, type SqliteDb } from "../nodeSqlite";
+
+export type FixtureCookie = {
+ host: string;
+ name: string;
+ value: string;
+ path?: string;
+ // Seconds since the epoch, as Firefox has stored it.
+ expiry?: number;
+ // PRTime: microseconds since the epoch.
+ lastAccessed?: number;
+ isSecure?: 0 | 1;
+ isHttpOnly?: 0 | 1;
+ sameSite?: number;
+ originAttributes?: string;
+};
+
+// Firefox's table as recent releases create it.
+const SCHEMA = `CREATE TABLE moz_cookies (
+ id INTEGER PRIMARY KEY,
+ originAttributes TEXT NOT NULL DEFAULT '',
+ name TEXT,
+ value TEXT,
+ host TEXT,
+ path TEXT,
+ expiry INTEGER,
+ lastAccessed INTEGER,
+ creationTime INTEGER,
+ isSecure INTEGER,
+ isHttpOnly INTEGER,
+ inBrowserElement INTEGER DEFAULT 0,
+ sameSite INTEGER DEFAULT 0,
+ schemeMap INTEGER DEFAULT 0,
+ isPartitionedAttributeSet INTEGER DEFAULT 0,
+ CONSTRAINT moz_uniqueid UNIQUE (name, host, path, originAttributes)
+)`;
+
+export function insertFixtureCookies(db: SqliteDb, cookies: ReadonlyArray<FixtureCookie>): void {
+ const insert = db.prepare(
+ "INSERT INTO moz_cookies (originAttributes, name, value, host, path, expiry, " +
+ "lastAccessed, creationTime, isSecure, isHttpOnly, sameSite) " +
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+ );
+ const nowUs = Date.now() * 1000;
+ for (const c of cookies) {
+ insert.run(
+ c.originAttributes ?? "",
+ c.name,
+ c.value,
+ c.host,
+ c.path ?? "/",
+ c.expiry ?? Math.floor(Date.now() / 1000) + 86_400 * 365,
+ c.lastAccessed ?? nowUs,
+ c.lastAccessed ?? nowUs,
+ c.isSecure ?? 1,
+ c.isHttpOnly ?? 1,
+ c.sameSite ?? 0,
+ );
+ }
+}
+
+// Write `<profileDir>/cookies.sqlite` holding `cookies`. With `keepOpen`, the
+// store is left open in WAL mode with the rows still in `cookies.sqlite-wal`
+// — what a running Firefox looks like — and the caller closes it.
+export async function writeFirefoxCookieStore(
+ profileDir: string,
+ cookies: ReadonlyArray<FixtureCookie>,
+ opts: { keepOpen?: boolean } = {},
+): Promise<{ file: string; db?: SqliteDb }> {
+ await mkdir(profileDir, { recursive: true });
+ const file = path.join(profileDir, "cookies.sqlite");
+ const { DatabaseSync } = await loadSqlite();
+ const db = new DatabaseSync(file);
+ db.exec(SCHEMA);
+ if (opts.keepOpen) {
+ db.exec("PRAGMA journal_mode=WAL");
+ db.exec("PRAGMA wal_autocheckpoint=0");
+ }
+ insertFixtureCookies(db, cookies);
+ if (opts.keepOpen) return { file, db };
+ db.close();
+ return { file };
+}
diff --git a/common/social/fetchers.ts b/common/social/fetchers.ts
@@ -14,6 +14,7 @@
// in the client bundle.
import type { Post, PostAvailability, PostPlatform } from "../lib/posts";
+import type { XCookieSource } from "./xCookieSource";
export type PostFetchInput = {
// The account's canonical URL as configured on the channel.
@@ -37,6 +38,17 @@ export type PostFetchInput = {
// Resolved cookie spec from resolveCookiePolicy(), when the fetcher needs
// credentials. Bluesky ignores it entirely.
cookies?: string;
+ // WHERE AN X FETCHER'S LOGIN COMES FROM this run — `social.x.cookieSource`,
+ // resolved by the fetch controller (xCookieSource.ts): "browser" reads the
+ // operator's everyday browser named by `browserCookies`; "profile" uses the
+ // session broker's connected profile. Unset (a caller that does not resolve
+ // it): the profile's jar when it holds a login, else `cookies`, as before the
+ // choice existed. Platforms without a login ignore both.
+ cookieSource?: XCookieSource;
+ // The browser spec the "browser" source reads: the channel's
+ // cookiesFromBrowser over the global one, REGARDLESS of cookieMode — the
+ // source, not yt-dlp's mode, governs the X fetchers.
+ browserCookies?: string;
// Soft cap on how many posts to return in one run. Undefined = no cap
// beyond the watermark/seen-id stop conditions.
limit?: number;
@@ -73,7 +85,8 @@ export type SocialFetcherProbe = {
// Which config fields this fetcher surfaces in the channel form. Mirrors
// TranscriptionApp.fields.
export type SocialFetcherFields = {
- // Needs a browser cookie spec (cookiesFromBrowser / cookieMode).
+ // Needs a browser cookie spec (cookiesFromBrowser / cookieMode; for X, the
+ // login source `social.x.cookieSource`).
cookies?: boolean;
// Needs an external binary whose path is configurable.
binPath?: boolean;
diff --git a/common/social/nodeSqlite.ts b/common/social/nodeSqlite.ts
@@ -0,0 +1,36 @@
+// node:sqlite, loaded at run time — for reading a browser's cookie store
+// (xBrowserLogin.ts) and for the tests' fixture stores.
+//
+// The specifier is assembled so no bundler tries to resolve it (the same reason
+// playwrightRuntime.ts assembles its own): `node:sqlite` exists only with the
+// prefix, and Node 22.13+ loads it without a flag (with an ExperimentalWarning,
+// once per process). Structural types, because the repo's @types/node predates
+// the module. No imports, so a test harness can load it on its own.
+
+type SqliteStatement = {
+ all: (...params: unknown[]) => unknown[];
+ run: (...params: unknown[]) => unknown;
+};
+
+export type SqliteDb = {
+ prepare: (sql: string) => SqliteStatement;
+ exec: (sql: string) => void;
+ close: () => void;
+};
+
+export type SqliteModule = {
+ DatabaseSync: new (file: string, opts?: Record<string, unknown>) => SqliteDb;
+};
+
+const SQLITE = ["node", "sqlite"].join(":");
+
+export async function loadSqlite(): Promise<SqliteModule> {
+ const mod = (await import(/* webpackIgnore: true */ SQLITE)) as Partial<SqliteModule> & {
+ default?: SqliteModule;
+ };
+ const m = mod.DatabaseSync ? (mod as SqliteModule) : mod.default;
+ if (!m?.DatabaseSync) {
+ throw new Error("node:sqlite is not available in this Node (22.13 or later loads it without a flag).");
+ }
+ return m;
+}
diff --git a/common/social/playwrightRuntime.ts b/common/social/playwrightRuntime.ts
@@ -47,6 +47,9 @@ export type BrowserContextLike = {
newPage: () => Promise<PageLike>;
pages: () => PageLike[];
cookies: () => Promise<unknown[]>;
+ // The X fallback fetcher's browser-login path hands the operator's browser
+ // cookies to a fresh context (xBrowserLogin.ts).
+ addCookies: (cookies: ReadonlyArray<Record<string, unknown>>) => Promise<void>;
close: () => Promise<void>;
on: (event: string, cb: () => void) => void;
};
diff --git a/common/social/xBrowser.test.ts b/common/social/xBrowser.test.ts
@@ -0,0 +1,162 @@
+// The Connect window's browser and its launch options (release 16 slice XL).
+// Pure: nothing here launches a browser.
+//
+// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test social/xBrowser.test.ts
+
+import { test, after } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync } from "node:fs";
+import { rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import {
+ buildXBrowserLaunchOptions,
+ describeXBrowser,
+ findXBrowser,
+ readXBrowserRecord,
+ recordedXBrowser,
+ writeXBrowserRecord,
+ X_BROWSER_ENV,
+} from "./xBrowser";
+
+const TMP = mkdtempSync(path.join(os.tmpdir(), "xl-xbrowser-"));
+after(() => rm(TMP, { recursive: true, force: true }));
+
+// A fake filesystem of executables.
+const only = (...files: string[]) => (p: string) => files.includes(p);
+
+test("the system browser: the first of chromium, google-chrome, google-chrome-stable, chrome on PATH", () => {
+ const env = { PATH: "/opt/bin:/usr/bin" };
+ assert.deepEqual(
+ findXBrowser({ env, isExecutable: only("/usr/bin/google-chrome-stable", "/usr/bin/chromium") }),
+ { kind: "system", executablePath: "/usr/bin/chromium", from: "path" },
+ );
+ assert.deepEqual(
+ findXBrowser({ env, isExecutable: only("/usr/bin/chrome", "/opt/bin/google-chrome") }),
+ { kind: "system", executablePath: "/opt/bin/google-chrome", from: "path" },
+ );
+});
+
+test("no system browser: Playwright's bundled build", () => {
+ assert.deepEqual(findXBrowser({ env: { PATH: "/usr/bin" }, isExecutable: only() }), {
+ kind: "bundled",
+ });
+ assert.deepEqual(findXBrowser({ env: {}, isExecutable: only("/usr/bin/chromium") }), {
+ kind: "bundled",
+ });
+});
+
+test("ARCHILYZER_X_BROWSER wins, as a path or a name on PATH", () => {
+ const isExecutable = only("/srv/brave", "/usr/bin/chromium", "/usr/bin/vivaldi");
+ assert.deepEqual(
+ findXBrowser({ env: { [X_BROWSER_ENV]: "/srv/brave", PATH: "/usr/bin" }, isExecutable }),
+ { kind: "system", executablePath: "/srv/brave", from: "env" },
+ );
+ assert.deepEqual(
+ findXBrowser({ env: { [X_BROWSER_ENV]: "vivaldi", PATH: "/usr/bin" }, isExecutable }),
+ { kind: "system", executablePath: "/usr/bin/vivaldi", from: "env" },
+ );
+ // A blank override is no override.
+ assert.deepEqual(
+ findXBrowser({ env: { [X_BROWSER_ENV]: " ", PATH: "/usr/bin" }, isExecutable }),
+ { kind: "system", executablePath: "/usr/bin/chromium", from: "path" },
+ );
+});
+
+test("an ARCHILYZER_X_BROWSER that is not an executable refuses — it never opens another browser", () => {
+ const isExecutable = only("/usr/bin/chromium");
+ assert.throws(
+ () => findXBrowser({ env: { [X_BROWSER_ENV]: "/nope/chrome", PATH: "/usr/bin" }, isExecutable }),
+ /ARCHILYZER_X_BROWSER names "\/nope\/chrome", which is not an executable file\./,
+ );
+ assert.throws(
+ () => findXBrowser({ env: { [X_BROWSER_ENV]: "edge", PATH: "/usr/bin" }, isExecutable }),
+ /not an executable file on PATH/,
+ );
+});
+
+test("the Connect window: the system browser, headed, sandboxed, without the automation signals", () => {
+ const opts = buildXBrowserLaunchOptions({
+ browser: { kind: "system", executablePath: "/usr/bin/chromium", from: "path" },
+ headless: false,
+ sandbox: true,
+ });
+ assert.deepEqual(opts, {
+ headless: false,
+ executablePath: "/usr/bin/chromium",
+ ignoreDefaultArgs: ["--enable-automation"],
+ args: ["--disable-blink-features=AutomationControlled", "--test-type"],
+ chromiumSandbox: true,
+ viewport: { width: 1280, height: 900 },
+ });
+});
+
+test("the bundled build: no executablePath, the same signals dropped", () => {
+ const headed = buildXBrowserLaunchOptions({ browser: { kind: "bundled" }, headless: false, sandbox: true });
+ assert.equal("executablePath" in headed, false);
+ assert.deepEqual(headed.ignoreDefaultArgs, ["--enable-automation"]);
+ assert.deepEqual(headed.args, ["--disable-blink-features=AutomationControlled", "--test-type"]);
+ assert.equal(headed.chromiumSandbox, true);
+
+ // The headless refresh: no viewport, Playwright's default sandbox setting.
+ const headless = buildXBrowserLaunchOptions({ browser: { kind: "bundled" }, headless: true });
+ assert.deepEqual(headless, {
+ headless: true,
+ ignoreDefaultArgs: ["--enable-automation"],
+ args: ["--disable-blink-features=AutomationControlled", "--test-type"],
+ });
+});
+
+test("ignoreDefaultArgs names ONLY --enable-automation (never `true`: it would drop --password-store=basic)", () => {
+ for (const browser of [
+ { kind: "bundled" as const },
+ { kind: "system" as const, executablePath: "/usr/bin/chromium", from: "path" as const },
+ ]) {
+ for (const headless of [true, false]) {
+ const o = buildXBrowserLaunchOptions({ browser, headless, sandbox: !headless });
+ assert.ok(Array.isArray(o.ignoreDefaultArgs));
+ assert.deepEqual(o.ignoreDefaultArgs, ["--enable-automation"]);
+ }
+ }
+});
+
+test("describeXBrowser says which browser, and how it was found", () => {
+ assert.match(describeXBrowser({ kind: "bundled" }), /bundled Chromium/);
+ assert.equal(
+ describeXBrowser(
+ { kind: "system", executablePath: "/usr/bin/chromium", from: "path" },
+ "Chromium 153.0.8010.47 Arch Linux",
+ ),
+ "Chromium 153.0.8010.47 Arch Linux at /usr/bin/chromium (found on PATH)",
+ );
+ assert.match(
+ describeXBrowser({ kind: "system", executablePath: "/srv/brave", from: "env" }),
+ /^\/srv\/brave \(from ARCHILYZER_X_BROWSER\)$/,
+ );
+});
+
+test("the profile records who wrote it, and the refresh's fallback reads it back", async () => {
+ const profile = path.join(TMP, "profile");
+ assert.equal(await readXBrowserRecord(profile), null);
+
+ await writeXBrowserRecord(
+ profile,
+ { kind: "system", executablePath: "/usr/bin/chromium", from: "path" },
+ "Chromium 153",
+ );
+ const rec = await readXBrowserRecord(profile);
+ assert.equal(rec?.executablePath, "/usr/bin/chromium");
+ assert.equal(rec?.version, "Chromium 153");
+ assert.deepEqual(recordedXBrowser(rec, only("/usr/bin/chromium")), {
+ kind: "system",
+ executablePath: "/usr/bin/chromium",
+ from: "recorded",
+ });
+ // The recorded executable is gone: no fallback.
+ assert.equal(recordedXBrowser(rec, only()), undefined);
+
+ await writeXBrowserRecord(profile, { kind: "bundled" });
+ const bundled = await readXBrowserRecord(profile);
+ assert.equal(bundled?.executablePath, null);
+ assert.equal(recordedXBrowser(bundled, only("/usr/bin/chromium")), undefined);
+});
diff --git a/common/social/xBrowser.ts b/common/social/xBrowser.ts
@@ -0,0 +1,242 @@
+// THE BROWSER THAT OPENS THE X SESSION PROFILE (release 16 slice XL).
+//
+// Why this exists: the Connect window used to be Playwright's bundled Chromium
+// launched with its automation signals on — `--enable-automation` (which is
+// what draws "Chrome is being controlled by automated test software") and
+// `navigator.webdriver === true`, which Playwright's debugging pipe turns on by
+// itself. Google's sign-in refuses such a browser ("this browser or app may not
+// be secure") and X's own login form stalled in it (2026-10-01).
+//
+// So the Connect window is the operator's own browser when one is installed —
+// ARCHILYZER_X_BROWSER, else the first of chromium / google-chrome /
+// google-chrome-stable / chrome on PATH, else the bundled build — and every
+// launch here drops the signals:
+// - `ignoreDefaultArgs: ["--enable-automation"]` — no automation bar. ONLY
+// that one: `ignoreDefaultArgs: true` would also drop Playwright's
+// `--password-store=basic`, and a system Chromium would then encrypt the
+// profile's cookies with the desktop keyring's key, which the bundled
+// headless build that refreshes the jar cannot read.
+// - `--disable-blink-features=AutomationControlled` — navigator.webdriver is
+// false. Measured: without it, it stays true even with
+// `--enable-automation` gone (the pipe sets it).
+// - `--test-type` and, for the headed window, the sandbox on — a system
+// Chrome draws "You are using an unsupported command-line flag" for the
+// blink flag above and for Playwright's default `--no-sandbox`. The
+// sandbox removes the second. `--test-type` is Chromium's internal
+// test-harness switch: it makes the browser skip its startup bars, the
+// bad-flags one among them, and sets no automation signal (measured:
+// navigator.webdriver, window.chrome, the user agent and the plugins are
+// the same with and without it). ChromeDriver passes
+// `--test-type=webdriver`; the bare switch is used here. It is not the
+// documented route — that is the `CommandLineFlagSecurityWarningsEnabled`
+// policy, which is machine-wide, needs root, and would silence the
+// operator's everyday browser too. The bundled build draws neither bar.
+// None of this hides the debugging pipe itself; Google's sign-in may still
+// refuse an embedded browser, which the Settings section says.
+//
+// THE PROFILE REMEMBERS WHO WROTE IT. A connect records its executable in the
+// profile dir (`archilyzer-browser.json`). The headless refresh keeps the
+// bundled build — it never logs in, and its headless shell needs no display —
+// and falls back to the recorded executable when the bundled build cannot open
+// the profile (a profile written by a newer system browser). Measured
+// 2026-10-01: a profile written by system Chromium 153 opens in the bundled
+// headless shell 147 with its cookies readable, so the fallback is a guard,
+// not the common path.
+
+import { accessSync, constants, statSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { writeFileAtomic } from "../lib/jsonFile-server";
+
+export const X_BROWSER_ENV = "ARCHILYZER_X_BROWSER";
+
+// Looked up on PATH in this order when ARCHILYZER_X_BROWSER is unset.
+export const SYSTEM_CHROMIUM_NAMES: readonly string[] = [
+ "chromium",
+ "google-chrome",
+ "google-chrome-stable",
+ "chrome",
+];
+
+export type XBrowserChoice =
+ | {
+ kind: "system";
+ executablePath: string;
+ // How it was found: the env override, PATH, or the profile's record.
+ from: "env" | "path" | "recorded";
+ }
+ | { kind: "bundled" };
+
+export function isExecutableFile(p: string): boolean {
+ try {
+ if (!statSync(p).isFile()) return false;
+ accessSync(p, constants.X_OK);
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+function onPath(
+ name: string,
+ pathEnv: string | undefined,
+ isExecutable: (p: string) => boolean,
+): string | undefined {
+ for (const dir of (pathEnv ?? "").split(path.delimiter)) {
+ if (!dir) continue;
+ const candidate = path.join(dir, name);
+ if (isExecutable(candidate)) return candidate;
+ }
+ return undefined;
+}
+
+// The Connect window's browser. Throws only when ARCHILYZER_X_BROWSER names
+// something that is not an executable: an explicit setting that is wrong is
+// said, not silently replaced by a different browser.
+export function findXBrowser(
+ opts: {
+ env?: Record<string, string | undefined>;
+ isExecutable?: (p: string) => boolean;
+ } = {},
+): XBrowserChoice {
+ const env = opts.env ?? process.env;
+ const isExecutable = opts.isExecutable ?? isExecutableFile;
+ const override = env.ARCHILYZER_X_BROWSER?.trim();
+ if (override) {
+ const resolved = override.includes("/")
+ ? isExecutable(override)
+ ? override
+ : undefined
+ : onPath(override, env.PATH, isExecutable);
+ if (!resolved) {
+ throw new Error(
+ `${X_BROWSER_ENV} names "${override}", which is not an executable file` +
+ (override.includes("/") ? "." : " on PATH."),
+ );
+ }
+ return { kind: "system", executablePath: resolved, from: "env" };
+ }
+ for (const name of SYSTEM_CHROMIUM_NAMES) {
+ const hit = onPath(name, env.PATH, isExecutable);
+ if (hit) return { kind: "system", executablePath: hit, from: "path" };
+ }
+ return { kind: "bundled" };
+}
+
+// THE LAUNCH OPTIONS, pure. `headless: false` is the Connect window (the
+// operator logs in); `headless: true` is the refresh and the Playwright
+// fallback fetcher. `sandbox` turns Chromium's sandbox on — the headed window
+// asks for it (no unsupported-flag bar) and retries without it when the host
+// cannot start one; a headless launch keeps Playwright's default.
+export type XBrowserLaunchOptions = {
+ headless: boolean;
+ executablePath?: string;
+ ignoreDefaultArgs: string[];
+ args: string[];
+ chromiumSandbox?: boolean;
+ viewport?: { width: number; height: number };
+};
+
+export const X_BROWSER_ARGS: readonly string[] = [
+ "--disable-blink-features=AutomationControlled",
+ "--test-type",
+];
+
+export function buildXBrowserLaunchOptions(opts: {
+ browser: XBrowserChoice;
+ headless: boolean;
+ sandbox?: boolean;
+}): XBrowserLaunchOptions {
+ const out: XBrowserLaunchOptions = {
+ headless: opts.headless,
+ ignoreDefaultArgs: ["--enable-automation"],
+ args: [...X_BROWSER_ARGS],
+ };
+ if (opts.browser.kind === "system") {
+ out.executablePath = opts.browser.executablePath;
+ }
+ if (opts.sandbox) out.chromiumSandbox = true;
+ if (!opts.headless) out.viewport = { width: 1280, height: 900 };
+ return out;
+}
+
+export function describeXBrowser(b: XBrowserChoice, version?: string): string {
+ if (b.kind === "bundled") {
+ return "Playwright's bundled Chromium (no system Chromium or Chrome found)";
+ }
+ const how =
+ b.from === "env"
+ ? `from ${X_BROWSER_ENV}`
+ : b.from === "path"
+ ? "found on PATH"
+ : "the browser that created the profile";
+ return `${version ? `${version} at ` : ""}${b.executablePath} (${how})`;
+}
+
+// --- The profile's record of who wrote it ---------------------------------
+
+export const X_BROWSER_RECORD = "archilyzer-browser.json";
+
+export type XBrowserRecord = {
+ // The system executable that opened the Connect window; null = the bundled
+ // build.
+ executablePath: string | null;
+ // `<exe> --version`, when it answered.
+ version?: string;
+ recordedAt: string;
+};
+
+export function xBrowserRecordFile(profileDir: string): string {
+ return path.join(profileDir, X_BROWSER_RECORD);
+}
+
+export async function readXBrowserRecord(
+ profileDir: string,
+): Promise<XBrowserRecord | null> {
+ try {
+ const raw = JSON.parse(
+ await readFile(xBrowserRecordFile(profileDir), "utf8"),
+ ) as Record<string, unknown>;
+ if (
+ raw.executablePath !== null &&
+ typeof raw.executablePath !== "string"
+ ) {
+ return null;
+ }
+ return {
+ executablePath: raw.executablePath as string | null,
+ ...(typeof raw.version === "string" ? { version: raw.version } : {}),
+ recordedAt: typeof raw.recordedAt === "string" ? raw.recordedAt : "",
+ };
+ } catch {
+ return null;
+ }
+}
+
+export async function writeXBrowserRecord(
+ profileDir: string,
+ browser: XBrowserChoice,
+ version?: string,
+): Promise<void> {
+ const record: XBrowserRecord = {
+ executablePath: browser.kind === "system" ? browser.executablePath : null,
+ ...(version ? { version } : {}),
+ recordedAt: new Date().toISOString(),
+ };
+ await writeFileAtomic(
+ xBrowserRecordFile(profileDir),
+ JSON.stringify(record, null, 2) + "\n",
+ { mkdir: true },
+ );
+}
+
+// The browser the record names, for the refresh's fallback — undefined when it
+// names the bundled build, nothing, or an executable that is gone.
+export function recordedXBrowser(
+ record: XBrowserRecord | null,
+ isExecutable: (p: string) => boolean = isExecutableFile,
+): XBrowserChoice | undefined {
+ if (!record?.executablePath) return undefined;
+ if (!isExecutable(record.executablePath)) return undefined;
+ return { kind: "system", executablePath: record.executablePath, from: "recorded" };
+}
diff --git a/common/social/xBrowserLogin.test.ts b/common/social/xBrowserLogin.test.ts
@@ -0,0 +1,357 @@
+// Reading the operator's browser login for X, and the status the Settings
+// section's "Check" shows (release 16 slice XL). Every store here is a fixture
+// written through node:sqlite (__fixtures__/firefoxCookieStore.ts) under a temp
+// HOME — no test reads a real browser profile.
+//
+// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test social/xBrowserLogin.test.ts
+
+import { test, after } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync } from "node:fs";
+import { mkdir, readdir, rm, stat, utimes, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import {
+ firefoxExpirySeconds,
+ isOldDomainAuth,
+ isXComAuth,
+ readFirefoxXCookies,
+ readFirefoxXStore,
+ readXLoginStatus,
+ xCookiesFromBrowser,
+} from "./xBrowserLogin";
+import { xCookieFile, xProfileDir } from "./xSessionBroker";
+import {
+ insertFixtureCookies,
+ writeFirefoxCookieStore,
+ type FixtureCookie,
+} from "./__fixtures__/firefoxCookieStore";
+
+const TMP = mkdtempSync(path.join(os.tmpdir(), "xl-xbrowserlogin-"));
+after(() => rm(TMP, { recursive: true, force: true }));
+
+let n = 0;
+const fresh = (label: string) => path.join(TMP, `${label}-${++n}`);
+
+const NOW_S = Math.floor(Date.now() / 1000);
+const YEAR = 86_400 * 365;
+const LAST_USED_US = Date.UTC(2026, 9, 1, 8, 30, 0) * 1000; // 2026-10-01 08:30 UTC
+
+const X_LOGIN: FixtureCookie[] = [
+ { host: ".x.com", name: "auth_token", value: "fixture-auth", lastAccessed: LAST_USED_US },
+ { host: ".x.com", name: "ct0", value: "fixture-csrf", isHttpOnly: 0, sameSite: 1 },
+ { host: "x.com", name: "lang", value: "en", isSecure: 0, isHttpOnly: 0, sameSite: 0 },
+ { host: ".twitter.com", name: "guest_id", value: "v1%3A1", sameSite: 2 },
+ // Not X: never selected.
+ { host: ".example.com", name: "session", value: "not-x" },
+ { host: "notx.com", name: "auth_token", value: "lookalike" },
+ // Expired: skipped.
+ { host: ".x.com", name: "old", value: "gone", expiry: NOW_S - 60 },
+];
+
+test("reads X's cookies only, mapped to the shape Playwright and cookies.txt take", async () => {
+ const dir = fresh("profile");
+ const { file } = await writeFirefoxCookieStore(dir, X_LOGIN);
+ const cookies = await readFirefoxXCookies(file, { tmpRoot: TMP });
+ const byName = new Map(cookies.map((c) => [c.name, c]));
+ assert.deepEqual([...byName.keys()].sort(), ["auth_token", "ct0", "guest_id", "lang"]);
+ const auth = byName.get("auth_token")!;
+ assert.equal(auth.value, "fixture-auth");
+ assert.equal(auth.domain, ".x.com");
+ assert.equal(auth.path, "/");
+ assert.equal(auth.secure, true);
+ assert.equal(auth.httpOnly, true);
+ assert.equal(auth.sameSite, "None");
+ assert.equal(auth.lastAccessedMs, LAST_USED_US / 1000);
+ assert.ok(auth.expires > NOW_S);
+ assert.equal(byName.get("ct0")!.sameSite, "Lax");
+ assert.equal(byName.get("ct0")!.httpOnly, false);
+ assert.equal(byName.get("guest_id")!.sameSite, "Strict");
+ // SameSite=None on a cookie that is not secure is not kept by a browser.
+ assert.equal(byName.get("lang")!.sameSite, "Lax");
+ assert.equal(byName.get("lang")!.domain, "x.com");
+});
+
+test("a login still in the WAL of a running Firefox is read", async () => {
+ const dir = fresh("profile-wal");
+ const { file, db } = await writeFirefoxCookieStore(dir, X_LOGIN, { keepOpen: true });
+ try {
+ const listed = await readdir(dir);
+ assert.ok(listed.includes("cookies.sqlite-wal"), "the fixture leaves its rows in the WAL");
+ const cookies = await readFirefoxXCookies(file, { tmpRoot: TMP });
+ assert.ok(cookies.some((c) => c.name === "auth_token" && c.value === "fixture-auth"));
+ } finally {
+ db?.close();
+ }
+});
+
+test("the browser's store is never written: no file in the profile changes, no temp is left", async () => {
+ const dir = fresh("profile-ro");
+ const { file, db } = await writeFirefoxCookieStore(dir, X_LOGIN, { keepOpen: true });
+ try {
+ const before = await Promise.all(
+ (await readdir(dir)).sort().map(async (f) => {
+ const st = await stat(path.join(dir, f));
+ return `${f}:${st.size}:${st.mtimeMs}`;
+ }),
+ );
+ const tmpRoot = fresh("tmproot");
+ await mkdir(tmpRoot);
+ await readFirefoxXCookies(file, { tmpRoot });
+ const afterList = await Promise.all(
+ (await readdir(dir)).sort().map(async (f) => {
+ const st = await stat(path.join(dir, f));
+ return `${f}:${st.size}:${st.mtimeMs}`;
+ }),
+ );
+ assert.deepEqual(afterList, before);
+ assert.deepEqual(await readdir(tmpRoot), [], "the private copy is removed");
+ } finally {
+ db?.close();
+ }
+});
+
+test("containers are read as gallery-dl reads them: none by default, ::all, or one by name", async () => {
+ const dir = fresh("profile-containers");
+ const { file } = await writeFirefoxCookieStore(dir, [
+ { host: ".x.com", name: "auth_token", value: "default" },
+ { host: ".x.com", name: "auth_token", value: "work", originAttributes: "^userContextId=2" },
+ { host: ".x.com", name: "auth_token", value: "work-fpd", originAttributes: "^firstPartyDomain=x.com&userContextId=2&x=1" },
+ { host: ".x.com", name: "auth_token", value: "personal", originAttributes: "^userContextId=1" },
+ { host: ".x.com", name: "auth_token", value: "shopping", originAttributes: "^userContextId=12" },
+ ]);
+ await writeFile(
+ path.join(dir, "containers.json"),
+ JSON.stringify({
+ version: 5,
+ identities: [
+ { userContextId: 1, public: true, l10nID: "userContextPersonal.label" },
+ { userContextId: 2, public: true, name: "Work" },
+ { userContextId: 12, public: true, l10nId: "user-context-shopping" },
+ ],
+ }),
+ );
+ const values = async (container?: string) =>
+ (await readFirefoxXCookies(file, { container, tmpRoot: TMP })).map((c) => c.value).sort();
+ // No container in the spec: ONLY cookies outside every container (gallery-dl's
+ // default; yt-dlp would read them all).
+ assert.deepEqual(await values(), ["default"]);
+ assert.deepEqual(await values("none"), ["default"]);
+ assert.deepEqual(await values("all"), ["default", "personal", "shopping", "work", "work-fpd"]);
+ // By name (`name`), by `l10nID` (userContext<X>.label) and by `l10nId`
+ // (user-context-<x>) — case-sensitive, as gallery-dl matches. Container 2 does
+ // not pick up container 12.
+ assert.deepEqual(await values("Work"), ["work", "work-fpd"]);
+ assert.deepEqual(await values("Personal"), ["personal"]);
+ assert.deepEqual(await values("shopping"), ["shopping"]);
+ await assert.rejects(values("personal"), /No Firefox container named "personal"/);
+ await assert.rejects(values("Banking"), /No Firefox container named "Banking"/);
+});
+
+test("the store read has two views: with the WAL (Firefox now) and the main file alone (gallery-dl's)", async () => {
+ const dir = fresh("profile-views");
+ const { file, db } = await writeFirefoxCookieStore(dir, [
+ { host: ".x.com", name: "guest_id", value: "g" },
+ ]);
+ // The schema and guest_id are in the main file; the login lands in the WAL
+ // of the still-open store, as a fresh login does in a running Firefox.
+ assert.equal(db, undefined);
+ const live = await writeFirefoxCookieStore(fresh("profile-views-wal"), [], { keepOpen: true });
+ try {
+ insertFixtureCookies(live.db!, [{ host: ".x.com", name: "auth_token", value: "fresh" }]);
+ const read = await readFirefoxXStore(live.file, { tmpRoot: TMP });
+ assert.deepEqual(read.cookies.map((c) => c.value), ["fresh"]);
+ assert.deepEqual(read.mainFile, []);
+ } finally {
+ live.db?.close();
+ }
+ const settled = await readFirefoxXStore(file, { tmpRoot: TMP });
+ assert.deepEqual(settled.cookies.map((c) => c.name), ["guest_id"]);
+ assert.deepEqual(settled.mainFile.map((c) => c.name), ["guest_id"]);
+});
+
+test("a login counts on x.com only; twitter.com's auth_token is the old domain's", () => {
+ assert.equal(isXComAuth({ name: "auth_token", domain: ".x.com" }), true);
+ assert.equal(isXComAuth({ name: "auth_token", domain: "x.com" }), true);
+ assert.equal(isXComAuth({ name: "auth_token", domain: "api.x.com" }), true);
+ assert.equal(isXComAuth({ name: "auth_token", domain: ".twitter.com" }), false);
+ assert.equal(isXComAuth({ name: "auth_token", domain: ".notx.com" }), false);
+ assert.equal(isXComAuth({ name: "ct0", domain: ".x.com" }), false);
+ assert.equal(isOldDomainAuth({ name: "auth_token", domain: ".twitter.com" }), true);
+ assert.equal(isOldDomainAuth({ name: "auth_token", domain: ".x.com" }), false);
+});
+
+test("Firefox's expiry reads as seconds, or as milliseconds when too large for seconds", () => {
+ assert.equal(firefoxExpirySeconds(1_800_000_000), 1_800_000_000);
+ assert.equal(firefoxExpirySeconds(1_800_000_000_123), 1_800_000_000);
+ assert.equal(firefoxExpirySeconds(0), -1);
+ assert.equal(firefoxExpirySeconds(null), -1);
+});
+
+test("discovery: the most recently used store under Firefox's roots, as gallery-dl picks it", async () => {
+ const home = fresh("home");
+ const root = path.join(home, ".mozilla", "firefox");
+ const older = await writeFirefoxCookieStore(path.join(root, "aaa.default"), [
+ { host: ".x.com", name: "auth_token", value: "older-profile" },
+ ]);
+ const newer = await writeFirefoxCookieStore(path.join(root, "bbb.default-release"), [
+ { host: ".x.com", name: "auth_token", value: "newer-profile" },
+ ]);
+ const t = Date.now() / 1000;
+ await utimes(older.file, t - 3_600, t - 3_600);
+ await utimes(newer.file, t, t);
+
+ const read = await xCookiesFromBrowser("firefox", { home, tmpRoot: TMP });
+ assert.equal(read.ok, true);
+ assert.ok(read.ok && read.store === newer.file);
+ assert.ok(read.ok && read.cookies.some((c) => c.value === "newer-profile"));
+
+ // A profile NAME is looked up under the roots; a PATH is used as given.
+ const named = await xCookiesFromBrowser("firefox:aaa.default", { home, tmpRoot: TMP });
+ assert.ok(named.ok && named.store === older.file);
+ const byPath = await xCookiesFromBrowser(`firefox:${path.join(root, "aaa.default")}`, {
+ home,
+ tmpRoot: TMP,
+ });
+ assert.ok(byPath.ok && byPath.store === older.file);
+});
+
+test("what xCookiesFromBrowser will not read, it says", async () => {
+ const home = fresh("home-empty");
+ await mkdir(home);
+ const none = await xCookiesFromBrowser("", { home });
+ assert.equal(none.ok, false);
+ assert.ok(!none.ok && none.reason === "no-spec");
+
+ const chromium = await xCookiesFromBrowser("chromium:Default", { home });
+ assert.ok(!chromium.ok && chromium.reason === "unsupported");
+ assert.match(!chromium.ok ? chromium.message : "", /gallery-dl/);
+
+ const missing = await xCookiesFromBrowser("firefox", { home });
+ assert.ok(!missing.ok && missing.reason === "not-found");
+ assert.match(!missing.ok ? missing.message : "", /\.mozilla\/firefox/);
+});
+
+// --- The status ---------------------------------------------------------------
+
+function pathsFor(transcriptsDir: string): Paths {
+ return { transcriptsDir } as Paths;
+}
+
+async function connectProfile(paths: Paths, authed: boolean) {
+ await mkdir(xProfileDir(paths), { recursive: true });
+ await writeFile(
+ xCookieFile(paths),
+ "# Netscape HTTP Cookie File\n" +
+ (authed ? ".x.com\tTRUE\t/\tTRUE\t1900000000\tauth_token\tjar\n" : ".x.com\tTRUE\t/\tTRUE\t1900000000\tct0\tjar\n"),
+ );
+}
+
+test("status — browser source by default: the login is visible, with when the browser last used it", async () => {
+ const home = fresh("home-status");
+ await writeFirefoxCookieStore(path.join(home, ".mozilla", "firefox", "p.default"), X_LOGIN);
+ const paths = pathsFor(fresh("transcripts"));
+ const s = await readXLoginStatus(paths, { cookiesFromBrowser: "firefox" }, { home, tmpRoot: TMP });
+ assert.equal(s.source, "browser");
+ assert.equal(s.chosen, false);
+ assert.equal(s.label, "Browser login (firefox)");
+ assert.equal(s.authTokenVisible, true);
+ assert.equal(s.lastSeenAt, "2026-10-01T08:30:00.000Z");
+ assert.equal(s.walOnly, false);
+ assert.equal(s.oldDomainCookie, false);
+ assert.equal(
+ s.summary,
+ "An X login is visible in firefox, outside its containers (where gallery-dl reads); " +
+ "its auth_token was last used 2026-10-01 08:30:00 UTC.",
+ );
+});
+
+test("status — a login only in Firefox's write-ahead log says gallery-dl will not see it yet", async () => {
+ const home = fresh("home-wal");
+ const live = await writeFirefoxCookieStore(path.join(home, ".mozilla", "firefox", "p.default"), X_LOGIN, {
+ keepOpen: true,
+ });
+ try {
+ const s = await readXLoginStatus(pathsFor(fresh("transcripts")), { cookiesFromBrowser: "firefox" }, { home, tmpRoot: TMP });
+ assert.equal(s.authTokenVisible, true);
+ assert.equal(s.walOnly, true);
+ assert.match(
+ s.summary,
+ /Seen in Firefox's write-ahead log only; gallery-dl will see it after Firefox checkpoints \(closing Firefox does it\)\.$/,
+ );
+ } finally {
+ live.db?.close();
+ }
+});
+
+test("status — twitter.com's auth_token is reported as the old domain's, never counted", async () => {
+ const home = fresh("home-olddomain");
+ await writeFirefoxCookieStore(path.join(home, ".mozilla", "firefox", "p.default"), [
+ { host: ".twitter.com", name: "auth_token", value: "old" },
+ ]);
+ const s = await readXLoginStatus(pathsFor(fresh("transcripts")), { cookiesFromBrowser: "firefox" }, { home, tmpRoot: TMP });
+ assert.equal(s.authTokenVisible, false);
+ assert.equal(s.oldDomainCookie, true);
+ assert.match(s.summary, /^No X login in firefox, outside its containers \(where gallery-dl reads\): no auth_token cookie for x\.com\./);
+ assert.match(s.summary, /An old-domain cookie is present too \(an auth_token for twitter\.com\); gallery-dl does not use it\.$/);
+});
+
+test("status — a login only inside a container is not counted, and the Check says where it is", async () => {
+ const home = fresh("home-container");
+ await writeFirefoxCookieStore(path.join(home, ".mozilla", "firefox", "p.default"), [
+ { host: ".x.com", name: "auth_token", value: "in-a-container", originAttributes: "^userContextId=3" },
+ ]);
+ const paths = pathsFor(fresh("transcripts"));
+ const s = await readXLoginStatus(paths, { cookiesFromBrowser: "firefox" }, { home, tmpRoot: TMP });
+ assert.equal(s.authTokenVisible, false);
+ assert.equal(s.otherContainerLogin, true);
+ assert.match(s.summary, /One is in another Firefox container; gallery-dl reads it only when cookiesFromBrowser names that container \(firefox::<name>\) or firefox::all\./);
+ const all = await readXLoginStatus(paths, { cookiesFromBrowser: "firefox::all" }, { home, tmpRoot: TMP });
+ assert.equal(all.authTokenVisible, true);
+ assert.match(all.summary, /^An X login is visible in firefox, in any container/);
+});
+
+test("status — browser source with no X login in the browser", async () => {
+ const home = fresh("home-nologin");
+ await writeFirefoxCookieStore(path.join(home, ".mozilla", "firefox", "p.default"), [
+ { host: ".x.com", name: "guest_id", value: "g" },
+ ]);
+ const s = await readXLoginStatus(pathsFor(fresh("transcripts")), { cookiesFromBrowser: "firefox" }, { home, tmpRoot: TMP });
+ assert.equal(s.authTokenVisible, false);
+ assert.match(s.summary, /No X login in firefox/);
+});
+
+test("status — a browser that cannot be read here reports null, not false", async () => {
+ const s = await readXLoginStatus(
+ pathsFor(fresh("transcripts")),
+ { cookiesFromBrowser: "chromium", social: { x: { cookieSource: "browser" } } },
+ { home: fresh("home-x") },
+ );
+ assert.equal(s.source, "browser");
+ assert.equal(s.chosen, true);
+ assert.equal(s.authTokenVisible, null);
+});
+
+test("status — the profile: chosen by default when no browser is set, and when one is connected", async () => {
+ const paths = pathsFor(fresh("transcripts"));
+ const none = await readXLoginStatus(paths, { cookiesFromBrowser: "" });
+ assert.equal(none.source, "profile");
+ assert.equal(none.label, "Connected profile");
+ assert.equal(none.authTokenVisible, false);
+ assert.match(none.summary, /No profile is connected/);
+
+ await connectProfile(paths, false);
+ const notLoggedIn = await readXLoginStatus(paths, { cookiesFromBrowser: "" });
+ assert.equal(notLoggedIn.authTokenVisible, false);
+ assert.match(notLoggedIn.summary, /not logged in to X/);
+
+ // A connected profile turns the default to "profile" even with a browser set.
+ await connectProfile(paths, true);
+ const connected = await readXLoginStatus(paths, { cookiesFromBrowser: "firefox" }, { home: fresh("home-y") });
+ assert.equal(connected.source, "profile");
+ assert.equal(connected.chosen, false);
+ assert.equal(connected.authTokenVisible, true);
+ assert.ok(connected.lastSeenAt);
+ assert.match(connected.summary, /^The connected profile holds an X login \(cookies exported /);
+});
diff --git a/common/social/xBrowserLogin.ts b/common/social/xBrowserLogin.ts
@@ -0,0 +1,561 @@
+// THE OPERATOR'S BROWSER LOGIN, read for the X paths that cannot read it
+// themselves (release 16 slice XL).
+//
+// gallery-dl reads a browser's cookie store on its own (`--cookies-from-browser
+// <spec>`, xGalleryDlFetcher.ts) — every browser it supports, Chromium's
+// encrypted store included. Two things here need the same login without
+// gallery-dl: the Playwright fallback fetcher (it hands the cookies to a fresh
+// headless context) and the Settings section's "Check" (is an X login visible
+// at all, and when was it last used). Both go through `xCookiesFromBrowser`.
+//
+// FIREFOX ONLY. Its `cookies.sqlite` is plain SQLite with plain values. The
+// Chromium family encrypts each value with a key from the desktop keyring
+// (libsecret / KWallet) and is left to gallery-dl and yt-dlp, which carry that
+// decryption; for such a spec the readers here say so instead of guessing.
+//
+// READ-ONLY, ALWAYS. The browser's profile is never opened in place and never
+// written: `cookies.sqlite` and its `-wal` (Firefox writes ahead, so a fresh
+// login may live only in the WAL) are copied into a private temp dir, read
+// there, and the dir is removed. Only X's own rows are selected, so no other
+// site's cookies leave SQLite. The store is found much as gallery-dl finds it
+// (a profile path or name from the spec, else the most recently modified
+// `cookies.sqlite` under Firefox's profile roots) — not exactly: see
+// findFirefoxCookieDb. The CONTAINER is read exactly as gallery-dl reads it
+// (firefoxContainerScope), and so is the domain a login counts on (x.com).
+
+import { copyFile, mkdir, mkdtemp, readdir, readFile, rm, stat } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import { loadSqlite } from "./nodeSqlite";
+import { readXSessionStatus } from "./xSessionBroker";
+import {
+ parseBrowserSpec,
+ resolveXCookieSource,
+ SELF_READ_BROWSERS,
+ xCookieSourceView,
+ type BrowserSpec,
+ type ResolvedXCookieSource,
+ type XCookieSource,
+ type XCookieSourceView,
+} from "./xCookieSource";
+
+// X's session cookie: the one whose presence means "logged in".
+export const X_AUTH_COOKIE = "auth_token";
+
+// A cookie as read from a browser's store, in the shape Playwright's
+// `addCookies` and the broker's cookies.txt writer both take.
+export type XBrowserCookie = {
+ name: string;
+ value: string;
+ domain: string;
+ path: string;
+ // Seconds since the epoch; -1 for a session cookie.
+ expires: number;
+ httpOnly: boolean;
+ secure: boolean;
+ sameSite?: "Strict" | "Lax" | "None";
+ // When the browser last sent it, ms since the epoch (Firefox's lastAccessed).
+ lastAccessedMs?: number;
+};
+
+// --- Finding the store -------------------------------------------------------
+
+// Where Firefox keeps its profiles: the XDG location newer releases use, the
+// classic one, Snap, Flatpak, macOS. gallery-dl's list differs slightly (it
+// honours $XDG_CONFIG_HOME and has a second Flatpak root), and for a profile
+// NAME it takes the first root holding `<name>/cookies.sqlite` where this takes
+// the newest; on a stock Linux Firefox the two pick the same store.
+export function firefoxProfileRoots(home: string): string[] {
+ return [
+ path.join(home, ".config", "mozilla", "firefox"),
+ path.join(home, ".mozilla", "firefox"),
+ path.join(home, "snap", "firefox", "common", ".mozilla", "firefox"),
+ path.join(home, ".var", "app", "org.mozilla.firefox", ".mozilla", "firefox"),
+ path.join(home, "Library", "Application Support", "Firefox", "Profiles"),
+ ];
+}
+
+const COOKIE_DB = "cookies.sqlite";
+
+// The newest cookies.sqlite at most `depth` directories below `dir`. A profile
+// keeps it at its own top level, so two levels covers a root of profiles and a
+// profile given directly, without walking a profile's storage tree.
+async function newestCookieDb(
+ dir: string,
+ depth: number,
+): Promise<{ file: string; mtimeMs: number } | undefined> {
+ let entries;
+ try {
+ entries = await readdir(dir, { withFileTypes: true });
+ } catch {
+ return undefined;
+ }
+ let best: { file: string; mtimeMs: number } | undefined;
+ for (const e of entries) {
+ const p = path.join(dir, e.name);
+ if (e.name === COOKIE_DB && e.isFile()) {
+ const st = await stat(p).catch(() => null);
+ if (st && (!best || st.mtimeMs > best.mtimeMs)) best = { file: p, mtimeMs: st.mtimeMs };
+ } else if (depth > 0 && (e.isDirectory() || e.isSymbolicLink())) {
+ const hit = await newestCookieDb(p, depth - 1);
+ if (hit && (!best || hit.mtimeMs > best.mtimeMs)) best = hit;
+ }
+ }
+ return best;
+}
+
+function expandHome(p: string, home: string): string {
+ return p === "~" ? home : p.startsWith("~/") ? path.join(home, p.slice(2)) : p;
+}
+
+export async function findFirefoxCookieDb(
+ spec: BrowserSpec,
+ home: string,
+): Promise<{ file?: string; searched: string[] }> {
+ const profile = spec.profile ? expandHome(spec.profile, home) : undefined;
+ const searched = profile
+ ? profile.includes("/") || profile.includes(path.sep)
+ ? [profile]
+ : firefoxProfileRoots(home).map((r) => path.join(r, profile))
+ : firefoxProfileRoots(home);
+ let best: { file: string; mtimeMs: number } | undefined;
+ for (const root of searched) {
+ const hit = await newestCookieDb(root, 2);
+ if (hit && (!best || hit.mtimeMs > best.mtimeMs)) best = hit;
+ }
+ return { file: best?.file, searched };
+}
+
+// --- Reading it ----------------------------------------------------------------
+
+// WHICH CONTAINER, as gallery-dl reads it (gallery-dl/cookies.py,
+// `_firefox_cookies_database`; 1.32.9) — gallery-dl is the fetcher this login
+// feeds, so the readers here see what it sees:
+// no `::CONTAINER`, or `::none` — only cookies that belong to no container
+// (gallery-dl's default; yt-dlp's differs);
+// `::all` — every container, no filter;
+// `::NAME` — the container containers.json names so,
+// matched as gallery-dl matches it (its
+// `name`, else its `l10nId`, else `l10nID`;
+// case-sensitive).
+export type ContainerScope = { kind: "none" } | { kind: "all" } | { kind: "id"; id: number };
+
+function extr(text: string, begin: string, end: string): string {
+ const i = text.indexOf(begin);
+ if (i < 0) return "";
+ const from = i + begin.length;
+ const j = text.indexOf(end, from);
+ return j < 0 ? "" : text.slice(from, j);
+}
+
+export async function firefoxContainerScope(
+ dbFile: string,
+ container: string | undefined,
+): Promise<ContainerScope> {
+ if (!container || container === "none") return { kind: "none" };
+ if (container === "all") return { kind: "all" };
+ let identities: Array<Record<string, unknown>> = [];
+ try {
+ const raw = JSON.parse(
+ await readFile(path.join(path.dirname(dbFile), "containers.json"), "utf8"),
+ ) as { identities?: unknown };
+ if (Array.isArray(raw.identities)) identities = raw.identities as Array<Record<string, unknown>>;
+ } catch {
+ /* no containers.json: no container can match */
+ }
+ const str = (v: unknown) => (typeof v === "string" && v ? v : undefined);
+ const hit = identities.find((c) => {
+ const name = str(c.name);
+ if (name) return name === container;
+ const l10nId = str(c.l10nId);
+ if (l10nId) {
+ return (
+ (l10nId.startsWith("user-context-") && l10nId.slice(13) === container) ||
+ l10nId.slice(l10nId.lastIndexOf("-") + 1) === container
+ );
+ }
+ const l10nID = str(c.l10nID);
+ return l10nID ? extr(l10nID, "userContext", ".label") === container : false;
+ });
+ if (typeof hit?.userContextId !== "number") {
+ throw new Error(`No Firefox container named "${container}" in containers.json`);
+ }
+ return { kind: "id", id: hit.userContextId };
+}
+
+// gallery-dl's SQL, in JS: `NOT INSTR(originAttributes,'userContextId=')` for
+// no container, `LIKE '%userContextId=<id>' OR LIKE '%userContextId=<id>&%'`
+// for one.
+export function inContainerScope(originAttributes: string, scope: ContainerScope): boolean {
+ if (scope.kind === "all") return true;
+ if (scope.kind === "none") return !originAttributes.includes("userContextId=");
+ const tag = `userContextId=${scope.id}`;
+ return originAttributes.endsWith(tag) || originAttributes.includes(`${tag}&`);
+}
+
+const X_HOSTS =
+ "(host = 'x.com' OR host = '.x.com' OR host LIKE '%.x.com' OR " +
+ "host = 'twitter.com' OR host = '.twitter.com' OR host LIKE '%.twitter.com')";
+
+// X's own domain. gallery-dl's twitter extractor looks its `auth_token` up on
+// `.x.com` (extractor/twitter.py), so a login counts only there; twitter.com's
+// is the old domain's, read but not counted.
+export function isXComHost(host: string): boolean {
+ const h = host.replace(/^\./, "").toLowerCase();
+ return h === "x.com" || h.endsWith(".x.com");
+}
+
+export function isXComAuth(c: { name: string; domain: string }): boolean {
+ return c.name === X_AUTH_COOKIE && isXComHost(c.domain);
+}
+
+export function isOldDomainAuth(c: { name: string; domain: string }): boolean {
+ if (c.name !== X_AUTH_COOKIE) return false;
+ const h = c.domain.replace(/^\./, "").toLowerCase();
+ return h === "twitter.com" || h.endsWith(".twitter.com");
+}
+
+function num(v: unknown): number | undefined {
+ if (typeof v === "number" && Number.isFinite(v)) return v;
+ if (typeof v === "bigint") return Number(v);
+ return undefined;
+}
+
+// Firefox has stored `expiry` in seconds; a value too large for seconds is read
+// as milliseconds, so a store that changes unit still reads right.
+export function firefoxExpirySeconds(v: unknown): number {
+ const n = num(v);
+ if (!n || n <= 0) return -1;
+ return Math.floor(n > 1e11 ? n / 1000 : n);
+}
+
+// `lastAccessed` is PRTime: microseconds since the epoch.
+function firefoxTimeMs(v: unknown): number | undefined {
+ const n = num(v);
+ if (!n || n <= 0) return undefined;
+ return Math.floor(n > 1e14 ? n / 1000 : n);
+}
+
+function firefoxSameSite(v: unknown, secure: boolean): XBrowserCookie["sameSite"] {
+ const n = num(v);
+ if (n === 2) return "Strict";
+ if (n === 1) return "Lax";
+ // SameSite=None is only kept by a browser on a secure cookie.
+ if (n === 0) return secure ? "None" : "Lax";
+ return undefined;
+}
+
+type StoreRow = { cookie: XBrowserCookie; originAttributes: string };
+
+function readRows(
+ DatabaseSync: Awaited<ReturnType<typeof loadSqlite>>["DatabaseSync"],
+ file: string,
+ nowS: number,
+): StoreRow[] {
+ const db = new DatabaseSync(file);
+ try {
+ const rows = db.prepare(`SELECT * FROM moz_cookies WHERE ${X_HOSTS}`).all() as Array<
+ Record<string, unknown>
+ >;
+ const out: StoreRow[] = [];
+ for (const r of rows) {
+ if (typeof r.name !== "string" || typeof r.host !== "string") continue;
+ const secure = num(r.isSecure) === 1;
+ const expires = firefoxExpirySeconds(r.expiry);
+ if (expires > 0 && expires < nowS) continue;
+ const cookie: XBrowserCookie = {
+ name: r.name,
+ value: typeof r.value === "string" ? r.value : String(r.value ?? ""),
+ domain: r.host,
+ path: typeof r.path === "string" && r.path ? r.path : "/",
+ expires,
+ httpOnly: num(r.isHttpOnly) === 1,
+ secure,
+ };
+ const sameSite = firefoxSameSite(r.sameSite, secure);
+ if (sameSite) cookie.sameSite = sameSite;
+ const last = firefoxTimeMs(r.lastAccessed);
+ if (last) cookie.lastAccessedMs = last;
+ out.push({
+ cookie,
+ originAttributes: typeof r.originAttributes === "string" ? r.originAttributes : "",
+ });
+ }
+ return out;
+ } finally {
+ db.close();
+ }
+}
+
+// ONE READ OF THE STORE, two views of X's rows:
+// `cookies` — with the WAL: what Firefox holds now. The Playwright
+// fallback uses these (a fresh login is in the WAL until
+// Firefox checkpoints).
+// `mainFile` — the main file alone: what gallery-dl sees. It opens
+// cookies.sqlite `mode=ro&immutable=1`, which ignores the WAL
+// (and its fallback copies the main file only).
+// Both in the container scope; `otherScopeAuth` says whether an x.com
+// auth_token exists OUTSIDE it (in a container the spec did not name).
+export type FirefoxXRead = {
+ cookies: XBrowserCookie[];
+ mainFile: XBrowserCookie[];
+ otherScopeAuth: boolean;
+};
+
+export async function readFirefoxXStore(
+ dbFile: string,
+ opts: { container?: string; tmpRoot?: string; now?: number } = {},
+): Promise<FirefoxXRead> {
+ const { DatabaseSync } = await loadSqlite();
+ const scope = await firefoxContainerScope(dbFile, opts.container);
+ // mkdtemp makes the dir 0700: the copies are readable by this user only.
+ const tmp = await mkdtemp(path.join(opts.tmpRoot ?? os.tmpdir(), "archilyzer-xcookies-"));
+ try {
+ // The db and its WAL back to back (the shortest window for a checkpoint
+ // between them), then a second copy of the db alone, taken from the first.
+ const withWal = path.join(tmp, "wal", COOKIE_DB);
+ const mainOnly = path.join(tmp, "main", COOKIE_DB);
+ await mkdir(path.dirname(withWal));
+ await mkdir(path.dirname(mainOnly));
+ await copyFile(dbFile, withWal);
+ await copyFile(`${dbFile}-wal`, `${withWal}-wal`).catch((err: NodeJS.ErrnoException) => {
+ if (err.code !== "ENOENT") throw err;
+ });
+ await copyFile(withWal, mainOnly);
+ const nowS = (opts.now ?? Date.now()) / 1000;
+ const walRows = readRows(DatabaseSync, withWal, nowS);
+ const mainRows = readRows(DatabaseSync, mainOnly, nowS);
+ const scoped = (rows: StoreRow[]) =>
+ rows.filter((r) => inContainerScope(r.originAttributes, scope)).map((r) => r.cookie);
+ return {
+ cookies: scoped(walRows),
+ mainFile: scoped(mainRows),
+ otherScopeAuth: walRows.some(
+ (r) => !inContainerScope(r.originAttributes, scope) && isXComAuth(r.cookie),
+ ),
+ };
+ } finally {
+ await rm(tmp, { recursive: true, force: true });
+ }
+}
+
+// X's cookies as Firefox holds them now (the WAL included), in the spec's
+// container scope.
+export async function readFirefoxXCookies(
+ dbFile: string,
+ opts: { container?: string; tmpRoot?: string; now?: number } = {},
+): Promise<XBrowserCookie[]> {
+ return (await readFirefoxXStore(dbFile, opts)).cookies;
+}
+
+export type BrowserCookieRead =
+ | ({
+ ok: true;
+ browser: string;
+ store: string;
+ // The spec's container, as given (undefined = outside every container).
+ container?: string;
+ } & FirefoxXRead)
+ | {
+ ok: false;
+ // no-spec: cookiesFromBrowser is empty. unsupported: a browser this repo
+ // does not read itself (gallery-dl still does). not-found: no store in
+ // the places searched. unreadable: the store or the spec could not be read.
+ reason: "no-spec" | "unsupported" | "not-found" | "unreadable";
+ message: string;
+ };
+
+// THE SHARED READ: the X cookies of the browser `spec` names, fresh on every
+// call. `spec` is the resolved one — a channel's own cookiesFromBrowser over
+// the global — which is why this takes the spec and not the settings.
+export async function xCookiesFromBrowser(
+ spec: string | undefined,
+ opts: { home?: string; tmpRoot?: string; now?: number } = {},
+): Promise<BrowserCookieRead> {
+ const trimmed = spec?.trim();
+ if (!trimmed) {
+ return {
+ ok: false,
+ reason: "no-spec",
+ message: "cookiesFromBrowser is empty, so there is no browser to read an X login from.",
+ };
+ }
+ const parsed = parseBrowserSpec(trimmed);
+ if (!parsed) {
+ return { ok: false, reason: "unreadable", message: `Could not read the browser spec "${trimmed}".` };
+ }
+ if (!SELF_READ_BROWSERS.includes(parsed.browser)) {
+ return {
+ ok: false,
+ reason: "unsupported",
+ message:
+ `${parsed.browser} keeps its cookies encrypted with the desktop keyring; only gallery-dl ` +
+ "and yt-dlp read them (gallery-dl does, on every X fetch). This check and the " +
+ "Playwright fallback read Firefox only.",
+ };
+ }
+ const home = opts.home ?? os.homedir();
+ const found = await findFirefoxCookieDb(parsed, home);
+ if (!found.file) {
+ return {
+ ok: false,
+ reason: "not-found",
+ message: `No Firefox cookie store found (looked in ${found.searched.join(", ")}).`,
+ };
+ }
+ try {
+ const read = await readFirefoxXStore(found.file, {
+ container: parsed.container,
+ tmpRoot: opts.tmpRoot,
+ now: opts.now,
+ });
+ return {
+ ok: true,
+ browser: parsed.browser,
+ store: found.file,
+ ...(parsed.container ? { container: parsed.container } : {}),
+ ...read,
+ };
+ } catch (err) {
+ return {
+ ok: false,
+ reason: "unreadable",
+ message: `Could not read ${found.file}: ${(err as Error).message}`,
+ };
+ }
+}
+
+// --- The source in use, and whether it holds a login --------------------------
+
+// The slice of SiteSettings this module reads. Structural, so settings.ts is
+// not imported here.
+export type XLoginSettings = {
+ cookiesFromBrowser?: string;
+ social?: { x?: { cookieSource?: XCookieSource } };
+};
+
+// The source for this host now: reads whether the broker's profile is
+// connected. `browserSpec` defaults to the global cookiesFromBrowser; the fetch
+// controller passes a channel's own when it has one.
+export async function resolveXCookieSourceFor(
+ paths: Paths,
+ settings: XLoginSettings,
+ browserSpec: string | undefined = settings.cookiesFromBrowser,
+): Promise<ResolvedXCookieSource> {
+ const profile = await readXSessionStatus(paths);
+ return resolveXCookieSource({
+ stored: settings.social?.x?.cookieSource,
+ browserSpec,
+ profileConnected: profile.looksAuthenticated,
+ });
+}
+
+export type XLoginStatus = XCookieSourceView & {
+ // Whether an auth_token cookie for x.com is visible in the source: null when
+ // the source could not be read (no spec, an unsupported browser, no store).
+ authTokenVisible: boolean | null;
+ // When the login was last seen: for the browser source, when the browser
+ // last sent its auth_token (the store's lastAccessed); for the profile, when
+ // the jar was last exported.
+ lastSeenAt?: string;
+ // Browser source only. True when the x.com auth_token is in Firefox's
+ // write-ahead log alone: gallery-dl, which reads the main file, does not see
+ // it until Firefox checkpoints.
+ walOnly?: boolean;
+ // Browser source only. An auth_token for twitter.com (the old domain) is
+ // present; it is reported, never counted — gallery-dl looks on x.com.
+ oldDomainCookie?: boolean;
+ // Browser source only. An x.com auth_token sits in a Firefox container the
+ // spec does not read.
+ otherContainerLogin?: boolean;
+ checkedAt: string;
+ // A sentence or three for the Settings section.
+ summary: string;
+};
+
+function when(iso: string): string {
+ return iso.replace("T", " ").replace(/\.\d+Z$/, " UTC");
+}
+
+export async function readXLoginStatus(
+ paths: Paths,
+ settings: XLoginSettings,
+ opts: { home?: string; tmpRoot?: string; now?: number } = {},
+): Promise<XLoginStatus> {
+ const checkedAt = new Date(opts.now ?? Date.now()).toISOString();
+ const resolved = await resolveXCookieSourceFor(paths, settings);
+ const base = { ...xCookieSourceView(resolved), checkedAt };
+
+ if (resolved.source === "profile") {
+ const profile = await readXSessionStatus(paths);
+ if (profile.looksAuthenticated) {
+ return {
+ ...base,
+ authTokenVisible: true,
+ lastSeenAt: profile.cookiesUpdatedAt,
+ summary:
+ "The connected profile holds an X login" +
+ (profile.cookiesUpdatedAt ? ` (cookies exported ${when(profile.cookiesUpdatedAt)}).` : "."),
+ };
+ }
+ return {
+ ...base,
+ authTokenVisible: false,
+ lastSeenAt: profile.cookiesUpdatedAt,
+ summary: profile.hasProfile
+ ? "The profile is not logged in to X (no auth_token in its exported cookies). Connect again, or choose the browser login."
+ : "No profile is connected. Connect an X account, or choose the browser login.",
+ };
+ }
+
+ const read = await xCookiesFromBrowser(resolved.browserSpec, opts);
+ if (!read.ok) {
+ return { ...base, authTokenVisible: null, summary: read.message };
+ }
+ // Where the read looked, as gallery-dl looks (see firefoxContainerScope).
+ const scope = !read.container || read.container === "none"
+ ? `${read.browser}, outside its containers (where gallery-dl reads)`
+ : read.container === "all"
+ ? `${read.browser}, in any container`
+ : `${read.browser}'s container "${read.container}"`;
+ const newest = (list: XBrowserCookie[]) =>
+ list.filter(isXComAuth).sort((a, b) => (b.lastAccessedMs ?? 0) - (a.lastAccessedMs ?? 0))[0];
+ const auth = newest(read.cookies);
+ const oldDomainCookie = read.cookies.some(isOldDomainAuth);
+ const oldDomainLine = oldDomainCookie
+ ? " An old-domain cookie is present too (an auth_token for twitter.com); gallery-dl does not use it."
+ : "";
+ if (!auth) {
+ return {
+ ...base,
+ authTokenVisible: false,
+ oldDomainCookie,
+ otherContainerLogin: read.otherScopeAuth,
+ summary:
+ `No X login in ${scope}: no auth_token cookie for x.com.` +
+ (read.otherScopeAuth
+ ? ` One is in another Firefox container; gallery-dl reads it only when cookiesFromBrowser ` +
+ `names that container (${read.browser}::<name>) or ${read.browser}::all.`
+ : " Log in to x.com in that browser.") +
+ oldDomainLine,
+ };
+ }
+ const lastSeenAt = auth.lastAccessedMs ? new Date(auth.lastAccessedMs).toISOString() : undefined;
+ const walOnly = !read.mainFile.some(isXComAuth);
+ return {
+ ...base,
+ authTokenVisible: true,
+ lastSeenAt,
+ walOnly,
+ oldDomainCookie,
+ otherContainerLogin: read.otherScopeAuth,
+ summary:
+ `An X login is visible in ${scope}` +
+ (lastSeenAt ? `; its auth_token was last used ${when(lastSeenAt)}.` : ".") +
+ (walOnly
+ ? " Seen in Firefox's write-ahead log only; gallery-dl will see it after Firefox checkpoints (closing Firefox does it)."
+ : "") +
+ oldDomainLine,
+ };
+}
diff --git a/common/social/xCookieSource.test.ts b/common/social/xCookieSource.test.ts
@@ -0,0 +1,102 @@
+// The X login source and its read-time default (release 16 slice XL).
+//
+// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test social/xCookieSource.test.ts
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ isXCookieSource,
+ parseBrowserSpec,
+ resolveXCookieSource,
+ sanitizeSocial,
+ xCookieSourceLabel,
+} from "./xCookieSource";
+
+test("the default: browser when cookiesFromBrowser is set and no profile is connected", () => {
+ assert.deepEqual(resolveXCookieSource({ browserSpec: "firefox", profileConnected: false }), {
+ source: "browser",
+ chosen: false,
+ browserSpec: "firefox",
+ });
+});
+
+test("the default: profile when a profile is connected, even with cookiesFromBrowser set", () => {
+ assert.deepEqual(resolveXCookieSource({ browserSpec: "firefox", profileConnected: true }), {
+ source: "profile",
+ chosen: false,
+ browserSpec: "firefox",
+ });
+});
+
+test("the default: profile when cookiesFromBrowser is empty", () => {
+ for (const browserSpec of [undefined, "", " "]) {
+ assert.deepEqual(resolveXCookieSource({ browserSpec, profileConnected: false }), {
+ source: "profile",
+ chosen: false,
+ browserSpec: undefined,
+ });
+ }
+});
+
+test("a stored choice always wins over the default", () => {
+ assert.deepEqual(
+ resolveXCookieSource({ stored: "profile", browserSpec: "firefox", profileConnected: false }),
+ { source: "profile", chosen: true, browserSpec: "firefox" },
+ );
+ assert.deepEqual(
+ resolveXCookieSource({ stored: "browser", browserSpec: "firefox", profileConnected: true }),
+ { source: "browser", chosen: true, browserSpec: "firefox" },
+ );
+ // Chosen with nothing to read: still the browser, and the label says why it
+ // will not log in.
+ const r = resolveXCookieSource({ stored: "browser", browserSpec: "", profileConnected: true });
+ assert.deepEqual(r, { source: "browser", chosen: true, browserSpec: undefined });
+ assert.equal(xCookieSourceLabel(r), "Browser login (no cookiesFromBrowser set)");
+});
+
+test("the labels", () => {
+ assert.equal(
+ xCookieSourceLabel({ source: "browser", chosen: false, browserSpec: "firefox" }),
+ "Browser login (firefox)",
+ );
+ assert.equal(xCookieSourceLabel({ source: "profile", chosen: true }), "Connected profile");
+});
+
+test("isXCookieSource and sanitizeSocial", () => {
+ assert.equal(isXCookieSource("browser"), true);
+ assert.equal(isXCookieSource("profile"), true);
+ assert.equal(isXCookieSource("firefox"), false);
+ assert.equal(isXCookieSource(undefined), false);
+ assert.deepEqual(sanitizeSocial(undefined), { x: {} });
+ assert.deepEqual(sanitizeSocial({ x: { cookieSource: "profile" } }), { x: { cookieSource: "profile" } });
+ assert.deepEqual(sanitizeSocial({ x: { cookieSource: "auto" } }), { x: {} });
+});
+
+test("parseBrowserSpec reads yt-dlp's and gallery-dl's syntax", () => {
+ assert.deepEqual(parseBrowserSpec("firefox"), { browser: "firefox" });
+ assert.deepEqual(parseBrowserSpec("Firefox"), { browser: "firefox" });
+ assert.deepEqual(parseBrowserSpec("chrome:Default"), { browser: "chrome", profile: "Default" });
+ assert.deepEqual(parseBrowserSpec("firefox:/home/u/.mozilla/firefox/abc.default"), {
+ browser: "firefox",
+ profile: "/home/u/.mozilla/firefox/abc.default",
+ });
+ assert.deepEqual(parseBrowserSpec("firefox::Work"), { browser: "firefox", container: "Work" });
+ assert.deepEqual(parseBrowserSpec("firefox:abc.default::none"), {
+ browser: "firefox",
+ profile: "abc.default",
+ container: "none",
+ });
+ assert.deepEqual(parseBrowserSpec("chromium+gnomekeyring:Profile 1"), {
+ browser: "chromium",
+ keyring: "gnomekeyring",
+ profile: "Profile 1",
+ });
+ // gallery-dl's /DOMAIN.
+ assert.deepEqual(parseBrowserSpec("firefox/.x.com:default"), {
+ browser: "firefox",
+ domain: ".x.com",
+ profile: "default",
+ });
+ assert.equal(parseBrowserSpec(""), null);
+ assert.equal(parseBrowserSpec(":profile"), null);
+});
diff --git a/common/social/xCookieSource.ts b/common/social/xCookieSource.ts
@@ -0,0 +1,131 @@
+// WHERE THE X FETCHERS' LOGIN COMES FROM — one setting, `social.x.cookieSource`
+// (release 16 slice XL).
+//
+// "browser" — the operator's everyday browser, named by `cookiesFromBrowser`
+// (the same spec yt-dlp reads). gallery-dl is handed
+// `--cookies-from-browser <spec>` and reads the browser's store
+// itself on every run; the other X paths read the same store
+// through xCookiesFromBrowser (xBrowserLogin.ts). Nothing expires
+// while the operator stays logged in to x.com there.
+// "profile" — the session broker's persistent profile (xSessionBroker.ts):
+// "Connect X account" once, then the jar it exports.
+//
+// THE DEFAULT IS RESOLVED AT READ TIME, never stored: with no
+// `social.x.cookieSource` in settings.json, the source is "browser" when
+// `cookiesFromBrowser` is set and no profile is connected (no exported jar
+// carrying an auth_token), else "profile". Choosing a source stores it, and a
+// stored choice always wins.
+//
+// Pure and client-safe — no node imports. The settings schema, the fetch
+// controller and the editor's Settings section all read it.
+
+export type XCookieSource = "browser" | "profile";
+
+export const X_COOKIE_SOURCES: readonly XCookieSource[] = ["browser", "profile"];
+
+export function isXCookieSource(v: unknown): v is XCookieSource {
+ return v === "browser" || v === "profile";
+}
+
+// The `social` block of settings.json. Only X has a login to choose today; the
+// block is per platform so a second one does not need a second top-level key.
+export type XSocialSettings = {
+ // Absent = the read-time default above.
+ cookieSource?: XCookieSource;
+};
+
+export type SocialSettings = {
+ x: XSocialSettings;
+};
+
+// Total over `unknown`, as every settings coercion is: anything that is not a
+// known source reads as absent (the default), and unknown keys are dropped.
+export function sanitizeSocial(value: unknown): SocialSettings {
+ const r = (value && typeof value === "object" && !Array.isArray(value)
+ ? value
+ : {}) as Record<string, unknown>;
+ const x = (r.x && typeof r.x === "object" && !Array.isArray(r.x)
+ ? r.x
+ : {}) as Record<string, unknown>;
+ return {
+ x: isXCookieSource(x.cookieSource) ? { cookieSource: x.cookieSource } : {},
+ };
+}
+
+export type ResolvedXCookieSource = {
+ source: XCookieSource;
+ // True when settings.json names the source; false when the read-time default
+ // decided it.
+ chosen: boolean;
+ // The browser spec the "browser" source reads (`cookiesFromBrowser`, a
+ // channel's own when it has one), or undefined when none is set.
+ browserSpec?: string;
+};
+
+export function resolveXCookieSource(input: {
+ stored?: XCookieSource;
+ browserSpec?: string;
+ // The session broker's profile holds a login: its exported jar carries an
+ // auth_token (readXSessionStatus().looksAuthenticated).
+ profileConnected: boolean;
+}): ResolvedXCookieSource {
+ const browserSpec = input.browserSpec?.trim() || undefined;
+ if (isXCookieSource(input.stored)) {
+ return { source: input.stored, chosen: true, browserSpec };
+ }
+ return {
+ source: browserSpec && !input.profileConnected ? "browser" : "profile",
+ chosen: false,
+ browserSpec,
+ };
+}
+
+// A browser cookie spec, as `cookiesFromBrowser` holds it. yt-dlp's syntax is
+// BROWSER[+KEYRING][:PROFILE][::CONTAINER]; gallery-dl's adds /DOMAIN after the
+// browser name. Both programs get the spec verbatim — this parse is only for
+// the readers here, which need the browser, the profile and the container.
+export type BrowserSpec = {
+ browser: string;
+ domain?: string;
+ keyring?: string;
+ profile?: string;
+ container?: string;
+};
+
+const SPEC_RE =
+ /^(?<browser>[^/+:]+)(?:\/(?<domain>[^+:]+))?(?:\s*\+\s*(?<keyring>[^:]+))?(?:\s*:\s*(?!:)(?<profile>.+?))?(?:\s*::\s*(?<container>.+))?$/;
+
+export function parseBrowserSpec(spec: string): BrowserSpec | null {
+ const m = SPEC_RE.exec(spec.trim());
+ if (!m?.groups) return null;
+ const g = m.groups;
+ const out: BrowserSpec = { browser: g.browser.trim().toLowerCase() };
+ if (g.domain) out.domain = g.domain.trim();
+ if (g.keyring) out.keyring = g.keyring.trim();
+ if (g.profile) out.profile = g.profile.trim();
+ if (g.container) out.container = g.container.trim();
+ return out;
+}
+
+// The browsers whose cookie store this repo reads itself (xBrowserLogin.ts).
+// Every other browser gallery-dl and yt-dlp support (the Chromium family keeps
+// its cookies encrypted with a desktop-keyring key) is read by those programs
+// alone; the Playwright fallback and the Settings check say so instead.
+export const SELF_READ_BROWSERS: readonly string[] = ["firefox"];
+
+// The label the Settings section and the logs use for a source.
+export function xCookieSourceLabel(r: ResolvedXCookieSource): string {
+ if (r.source === "browser") {
+ return r.browserSpec
+ ? `Browser login (${r.browserSpec})`
+ : "Browser login (no cookiesFromBrowser set)";
+ }
+ return "Connected profile";
+}
+
+// A resolved source with its label: what the Settings section renders.
+export type XCookieSourceView = ResolvedXCookieSource & { label: string };
+
+export function xCookieSourceView(r: ResolvedXCookieSource): XCookieSourceView {
+ return { ...r, label: xCookieSourceLabel(r) };
+}
diff --git a/common/social/xGalleryDlFetcher.test.ts b/common/social/xGalleryDlFetcher.test.ts
@@ -0,0 +1,86 @@
+// gallery-dl's argv per X login source (release 16 slice XL). The rest of
+// gallery-dl's argv and parsing is covered in xNormalize.test.ts.
+//
+// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test social/xGalleryDlFetcher.test.ts
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { buildGalleryDlArgs, galleryDlCookieChoice } from "./xGalleryDlFetcher";
+
+const ACCOUNT = "https://x.com/someaccount";
+const JAR = "/corpus/.x-session/cookies.txt";
+
+function argvFor(choice: ReturnType<typeof galleryDlCookieChoice>): string[] {
+ return buildGalleryDlArgs({
+ accountUrl: ACCOUNT,
+ cookies: choice.cookies,
+ cookieFile: choice.cookieFile,
+ });
+}
+
+function flagValue(argv: string[], flag: string): string | undefined {
+ const i = argv.indexOf(flag);
+ return i >= 0 ? argv[i + 1] : undefined;
+}
+
+test("browser source: --cookies-from-browser with the spec, never the jar", () => {
+ const argv = argvFor(
+ galleryDlCookieChoice({ source: "browser", browserCookies: "firefox", jarFile: JAR }),
+ );
+ assert.equal(flagValue(argv, "--cookies-from-browser"), "firefox");
+ assert.equal(argv.includes("--cookies"), false);
+ assert.equal(argv[argv.length - 1], "https://x.com/someaccount/timeline");
+});
+
+test("browser source: the spec is passed verbatim (profile, container, gallery-dl's /DOMAIN)", () => {
+ for (const spec of ["firefox:abc.default-release", "firefox::Work", "chromium+gnomekeyring:Default", "firefox/.x.com"]) {
+ const argv = argvFor(galleryDlCookieChoice({ source: "browser", browserCookies: spec }));
+ assert.equal(flagValue(argv, "--cookies-from-browser"), spec);
+ }
+});
+
+test("browser source: passed whatever the cookieMode — the source governs X", () => {
+ // No "always"-mode spec (cookieMode when-required or defer), still passed.
+ const choice = galleryDlCookieChoice({
+ source: "browser",
+ browserCookies: "firefox",
+ alwaysCookies: undefined,
+ });
+ assert.equal(choice.cookies, "firefox");
+ assert.match(choice.note, /the browser \(firefox\)/);
+});
+
+test("browser source with no spec: a guest run that says why", () => {
+ const choice = galleryDlCookieChoice({ source: "browser", browserCookies: " ", jarFile: JAR });
+ const argv = argvFor(choice);
+ assert.equal(argv.includes("--cookies-from-browser"), false);
+ assert.equal(argv.includes("--cookies"), false);
+ assert.match(choice.note, /cookiesFromBrowser is empty/);
+});
+
+test("profile source: the jar, and no --cookies-from-browser", () => {
+ const argv = argvFor(
+ galleryDlCookieChoice({ source: "profile", browserCookies: "firefox", jarFile: JAR, alwaysCookies: "firefox" }),
+ );
+ assert.equal(flagValue(argv, "--cookies"), JAR);
+ assert.equal(argv.includes("--cookies-from-browser"), false);
+});
+
+test("profile source with no logged-in jar: the cookieMode \"always\" spec, else a guest", () => {
+ const always = argvFor(galleryDlCookieChoice({ source: "profile", browserCookies: "firefox", alwaysCookies: "firefox" }));
+ assert.equal(flagValue(always, "--cookies-from-browser"), "firefox");
+
+ const guest = galleryDlCookieChoice({ source: "profile", browserCookies: "firefox" });
+ const argv = argvFor(guest);
+ assert.equal(argv.includes("--cookies-from-browser"), false);
+ assert.equal(argv.includes("--cookies"), false);
+ assert.match(guest.note, /guest/);
+});
+
+test("no source resolved (a caller from before the choice): the jar, else the always-mode spec", () => {
+ assert.equal(flagValue(argvFor(galleryDlCookieChoice({ jarFile: JAR, alwaysCookies: "firefox" })), "--cookies"), JAR);
+ assert.equal(
+ flagValue(argvFor(galleryDlCookieChoice({ alwaysCookies: "firefox" })), "--cookies-from-browser"),
+ "firefox",
+ );
+});
diff --git a/common/social/xGalleryDlFetcher.ts b/common/social/xGalleryDlFetcher.ts
@@ -29,6 +29,7 @@ import { getPaths } from "../lib/paths";
import type { Post } from "../lib/posts";
import { normalizeXTweets, type XTweetRaw } from "./xNormalize";
import { readXSessionStatus, xCookieFile } from "./xSessionBroker";
+import type { XCookieSource } from "./xCookieSource";
import {
registerSocialFetcher,
type PostFetchInput,
@@ -129,6 +130,50 @@ export function buildGalleryDlArgs(opts: {
return args;
}
+// WHICH LOGIN gallery-dl is handed this run (release 16 slice XL), pure so the
+// argv per source is testable:
+// "browser" — `--cookies-from-browser <spec>`, always (gallery-dl reads the
+// operator's browser itself, fresh on every run); never the jar.
+// With no spec, a guest run that says why.
+// "profile" — the broker's jar when it holds a login, else the cookieMode
+// "always" spec, else a guest run.
+// unset — the same as "profile": a caller that resolves no source keeps
+// the behaviour from before the choice existed.
+export function galleryDlCookieChoice(opts: {
+ source?: XCookieSource;
+ // The spec the browser source reads (channel over global, any cookieMode).
+ browserCookies?: string;
+ // The broker's exported jar, when it carries an auth_token.
+ jarFile?: string;
+ // resolveCookiePolicy()'s "always"-mode spec.
+ alwaysCookies?: string;
+}): { cookies?: string; cookieFile?: string; note: string } {
+ if (opts.source === "browser") {
+ const spec = opts.browserCookies?.trim();
+ if (spec) {
+ return { cookies: spec, note: `X login: the browser (${spec}), read by gallery-dl.` };
+ }
+ return {
+ note:
+ "X login: the browser source is chosen but cookiesFromBrowser is empty — " +
+ "running as a guest.",
+ };
+ }
+ if (opts.jarFile) {
+ return {
+ cookieFile: opts.jarFile,
+ note: "X login: the connected profile (the X session broker's exported cookies).",
+ };
+ }
+ if (opts.alwaysCookies) {
+ return {
+ cookies: opts.alwaysCookies,
+ note: `X login: no connected profile; the cookieMode "always" browser (${opts.alwaysCookies}).`,
+ };
+ }
+ return { note: "X login: none (no connected profile) — running as a guest." };
+}
+
// X ids are 64-bit and gallery-dl emits them as UNQUOTED JSON NUMBERS
// (`"tweet_id": 2085320225776427457`). That exceeds 2^53, so a plain
// JSON.parse silently rounds it — 2085320225776427457 becomes
@@ -256,19 +301,25 @@ export const xGalleryDlFetcher: SocialFetcher = {
},
async fetch(input: PostFetchInput): Promise<PostFetchResult> {
- const { accountUrl, channelSlug, since, seenIds, cookies, limit, signal, onLog } =
+ const { accountUrl, channelSlug, since, seenIds, limit, signal, onLog } =
input;
const paths = getPaths();
const bin = paths.galleryDlBin;
- // Prefer the session broker's exported jar when one exists — it is kept
- // fresh by a live browser profile, so it survives the cookie expiry that
- // otherwise breaks gallery-dl within days.
+ // The login source (galleryDlCookieChoice): the operator's browser, or the
+ // session broker's exported jar — kept fresh by a live browser profile, so
+ // it survives the cookie expiry that otherwise breaks gallery-dl within days.
const status = await readXSessionStatus(paths);
- const cookieFile =
- status.hasCookies && status.looksAuthenticated
- ? xCookieFile(paths)
- : undefined;
- if (cookieFile) onLog?.("Using the X session broker's exported cookies.");
+ const choice = galleryDlCookieChoice({
+ source: input.cookieSource,
+ browserCookies: input.browserCookies,
+ jarFile:
+ status.hasCookies && status.looksAuthenticated
+ ? xCookieFile(paths)
+ : undefined,
+ alwaysCookies: input.cookies,
+ });
+ onLog?.(choice.note);
+ const { cookies, cookieFile } = choice;
const args = buildGalleryDlArgs({ accountUrl, cookies, cookieFile, limit });
onLog?.(`Running ${bin} for ${accountUrl}`);
diff --git a/common/social/xPlaywrightFetcher.ts b/common/social/xPlaywrightFetcher.ts
@@ -11,7 +11,8 @@
//
// Known costs, accepted deliberately:
// - Playwright Chromium's JA3/TLS fingerprint matches no real Chrome release
-// (plus navigator.webdriver and CDP artifacts), so X CAN detect it.
+// (plus the HeadlessChrome user agent and CDP artifacts; navigator.webdriver
+// is off since release 16 slice XL, xBrowser.ts), so X CAN detect it.
// - Ban risk on the logged-in account is higher than an offline cookie read.
// - A browser process per fetch is slow and RAM-hungry.
// - It will NOT run in the minimal Docker build container — post fetching
@@ -31,9 +32,15 @@
import type { Post } from "../lib/posts";
import { normalizeXTweets, type XTweetRaw } from "./xNormalize";
-import { xProfileDir } from "./xSessionBroker";
+import { launchXProfile } from "./xSessionBroker";
import { getPaths } from "../lib/paths";
-import { importPlaywright } from "./playwrightRuntime";
+import {
+ importPlaywright,
+ type BrowserContextLike,
+ type BrowserLike,
+} from "./playwrightRuntime";
+import { buildXBrowserLaunchOptions } from "./xBrowser";
+import { isXComAuth, xCookiesFromBrowser } from "./xBrowserLogin";
import {
registerSocialFetcher,
type PostFetchInput,
@@ -151,11 +158,35 @@ export const xPlaywrightFetcher: SocialFetcher = {
const { accountUrl, channelSlug, since, seenIds, limit, signal, onLog } = input;
const paths = getPaths();
- const { chromium } = await importPlaywright();
- onLog?.("Launching the authenticated browser profile (fallback path).");
- const context = await chromium.launchPersistentContext(xProfileDir(paths), {
- headless: true,
- });
+ // The login source (xCookieSource.ts). "browser": a fresh headless
+ // context carrying the operator's browser cookies, read now — the browser
+ // store this repo reads itself is Firefox's (xBrowserLogin.ts); any other
+ // is refused by name rather than run logged out. Otherwise the broker's
+ // persistent profile, as before.
+ let context: BrowserContextLike;
+ let browser: BrowserLike | undefined;
+ if (input.cookieSource === "browser") {
+ const read = await xCookiesFromBrowser(input.browserCookies);
+ if (!read.ok) {
+ throw new Error(`The X login source is the browser, and it cannot be read here: ${read.message}`);
+ }
+ const hasAuth = read.cookies.some(isXComAuth);
+ onLog?.(
+ `Launching a headless browser with ${read.cookies.length} X cookie(s) from ${read.browser} ` +
+ `(fallback path)${hasAuth ? "" : " — WARNING: no auth_token for x.com, the browser is not logged in to X"}.`,
+ );
+ const { chromium } = await importPlaywright();
+ browser = await chromium.launch(
+ buildXBrowserLaunchOptions({ browser: { kind: "bundled" }, headless: true }),
+ );
+ context = await browser.newContext({});
+ await context.addCookies(
+ read.cookies.map(({ lastAccessedMs: _unused, ...c }) => c),
+ );
+ } else {
+ onLog?.("Launching the authenticated browser profile (fallback path).");
+ context = await launchXProfile(paths, { onLog });
+ }
const captured: XTweetRaw[] = [];
try {
@@ -201,6 +232,7 @@ export const xPlaywrightFetcher: SocialFetcher = {
onLog?.(`Captured ${captured.length} tweet record(s) from the timeline.`);
} finally {
await context.close().catch(() => {});
+ await browser?.close().catch(() => {});
}
// From here on it is identical to the gallery-dl path — same normalizer,
diff --git a/common/social/xSessionBroker.test.ts b/common/social/xSessionBroker.test.ts
@@ -0,0 +1,157 @@
+// The headless refresh never replaces a logged-in jar with one without a login
+// (release 16 slice XL, review L4). Playwright is replaced by a fake chromium
+// that answers per executable: nothing here launches a browser or loads x.com.
+//
+// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test social/xSessionBroker.test.ts
+
+import { test, after } from "node:test";
+import assert from "node:assert/strict";
+import { chmodSync, mkdtempSync, writeFileSync } from "node:fs";
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import type { ChromiumLike } from "./playwrightRuntime";
+import { writeXBrowserRecord } from "./xBrowser";
+import { refreshXCookies, xCookieFile, xProfileDir } from "./xSessionBroker";
+
+const TMP = mkdtempSync(path.join(os.tmpdir(), "xl-broker-"));
+after(() => rm(TMP, { recursive: true, force: true }));
+
+// A stand-in for the system browser the profile records: it only has to be an
+// executable file.
+const SYSTEM_EXE = path.join(TMP, "chromium");
+writeFileSync(SYSTEM_EXE, "#!/bin/sh\nexit 0\n");
+chmodSync(SYSTEM_EXE, 0o755);
+
+type Cookie = { name: string; value: string; domain: string; path: string; expires: number; httpOnly: boolean; secure: boolean };
+const cookie = (name: string, value: string): Cookie => ({
+ name,
+ value,
+ domain: ".x.com",
+ path: "/",
+ expires: 1_900_000_000,
+ httpOnly: true,
+ secure: true,
+});
+
+// Answers launchPersistentContext by executable: "bundled" when the options
+// name none. An Error answer throws, as a launch that cannot open the profile.
+function fakeChromium(answers: Record<string, Cookie[] | Error>) {
+ const launches: string[] = [];
+ const chromium = {
+ launch: async () => {
+ throw new Error("not used");
+ },
+ launchPersistentContext: async (_dir: string, opts: Record<string, unknown>) => {
+ const who = typeof opts.executablePath === "string" ? opts.executablePath : "bundled";
+ launches.push(who);
+ const answer = answers[who];
+ if (answer instanceof Error) throw answer;
+ const page = {
+ goto: async () => undefined,
+ waitForSelector: async () => undefined,
+ waitForTimeout: async () => undefined,
+ evaluate: async () => undefined,
+ on: () => undefined,
+ };
+ return {
+ pages: () => [page],
+ newPage: async () => page,
+ cookies: async () => answer ?? [],
+ addCookies: async () => undefined,
+ close: async () => undefined,
+ on: () => undefined,
+ };
+ },
+ } as unknown as ChromiumLike;
+ return { chromium, launches };
+}
+
+let n = 0;
+async function corpus(opts: { jar?: "logged-in" | "logged-out"; recorded?: boolean }) {
+ const paths = { transcriptsDir: path.join(TMP, `corpus-${++n}`) } as Paths;
+ await mkdir(xProfileDir(paths), { recursive: true });
+ if (opts.recorded) {
+ await writeXBrowserRecord(
+ xProfileDir(paths),
+ { kind: "system", executablePath: SYSTEM_EXE, from: "path" },
+ "Chromium 153",
+ );
+ }
+ if (opts.jar) {
+ await writeFile(
+ xCookieFile(paths),
+ "# Netscape HTTP Cookie File\n" +
+ (opts.jar === "logged-in"
+ ? ".x.com\tTRUE\t/\tTRUE\t1900000000\tauth_token\tOLD-LOGIN\n"
+ : ".x.com\tTRUE\t/\tTRUE\t1900000000\tguest_id\tg\n"),
+ );
+ }
+ return paths;
+}
+
+const jarText = (paths: Paths) => readFile(xCookieFile(paths), "utf8");
+
+test("the bundled read has the login: written, the recorded browser never launched", async () => {
+ const paths = await corpus({ jar: "logged-in", recorded: true });
+ const { chromium, launches } = fakeChromium({ bundled: [cookie("auth_token", "FRESH")] });
+ const status = await refreshXCookies(paths, { chromium });
+ assert.deepEqual(launches, ["bundled"]);
+ assert.equal(status.looksAuthenticated, true);
+ assert.match(await jarText(paths), /auth_token\tFRESH/);
+});
+
+test("an EMPTY bundled read of a logged-in profile is read again with the recorded browser", async () => {
+ const paths = await corpus({ jar: "logged-in", recorded: true });
+ const { chromium, launches } = fakeChromium({
+ bundled: [],
+ [SYSTEM_EXE]: [cookie("auth_token", "FROM-SYSTEM"), cookie("ct0", "c")],
+ });
+ const lines: string[] = [];
+ await refreshXCookies(paths, { chromium, onLog: (l) => lines.push(l) });
+ assert.deepEqual(launches, ["bundled", SYSTEM_EXE]);
+ assert.match(await jarText(paths), /auth_token\tFROM-SYSTEM/);
+ assert.ok(lines.some((l) => /read no X login from a profile whose exported jar holds one/.test(l)));
+});
+
+test("no read finds the login: the logged-in jar is KEPT and the refresh refuses", async () => {
+ const paths = await corpus({ jar: "logged-in", recorded: true });
+ const before = await jarText(paths);
+ const { chromium, launches } = fakeChromium({ bundled: [cookie("guest_id", "g")], [SYSTEM_EXE]: [] });
+ await assert.rejects(
+ refreshXCookies(paths, { chromium }),
+ /The profile gave no X login \(no auth_token\), so the logged-in cookie jar exported .* was kept, not replaced\./,
+ );
+ assert.deepEqual(launches, ["bundled", SYSTEM_EXE]);
+ assert.equal(await jarText(paths), before);
+});
+
+test("no record to fall back on: the logged-in jar is kept all the same", async () => {
+ const paths = await corpus({ jar: "logged-in" });
+ const before = await jarText(paths);
+ const { chromium, launches } = fakeChromium({ bundled: [] });
+ await assert.rejects(refreshXCookies(paths, { chromium }), /was kept, not replaced/);
+ assert.deepEqual(launches, ["bundled"]);
+ assert.equal(await jarText(paths), before);
+});
+
+test("a jar without a login is replaced as before (nothing to protect)", async () => {
+ const paths = await corpus({ jar: "logged-out", recorded: true });
+ const { chromium, launches } = fakeChromium({ bundled: [cookie("guest_id", "new")] });
+ const status = await refreshXCookies(paths, { chromium });
+ assert.deepEqual(launches, ["bundled"]);
+ assert.equal(status.looksAuthenticated, false);
+ assert.match(await jarText(paths), /guest_id\tnew/);
+});
+
+test("the bundled build cannot open the profile: the recorded browser reads it", async () => {
+ const paths = await corpus({ jar: "logged-in", recorded: true });
+ const { chromium, launches } = fakeChromium({
+ bundled: new Error("profile is from a newer version"),
+ [SYSTEM_EXE]: [cookie("auth_token", "VIA-RECORD")],
+ });
+ await refreshXCookies(paths, { chromium });
+ assert.deepEqual(launches, ["bundled", SYSTEM_EXE]);
+ assert.match(await jarText(paths), /auth_token\tVIA-RECORD/);
+});
diff --git a/common/social/xSessionBroker.ts b/common/social/xSessionBroker.ts
@@ -18,12 +18,31 @@
// installed, so this adds NO new dependency. It will NOT run in the minimal
// Docker build container — post fetching is an editor-host concern, never a
// build-time one.
+//
+// Release 16 slice XL: the Connect window is the operator's own browser
+// without the automation signals (xBrowser.ts), and the profile is one of two
+// login sources — the other is the operator's everyday browser, read directly
+// (`social.x.cookieSource`, xCookieSource.ts / xBrowserLogin.ts).
import path from "node:path";
import { mkdir, readFile, rm, stat } from "node:fs/promises";
+import { execa } from "execa";
import { writeFileAtomic } from "../lib/jsonFile-server";
import type { Paths } from "../lib/paths";
-import { importPlaywright } from "./playwrightRuntime";
+import {
+ importPlaywright,
+ type BrowserContextLike,
+ type ChromiumLike,
+} from "./playwrightRuntime";
+import {
+ buildXBrowserLaunchOptions,
+ describeXBrowser,
+ findXBrowser,
+ readXBrowserRecord,
+ recordedXBrowser,
+ writeXBrowserRecord,
+ type XBrowserChoice,
+} from "./xBrowser";
// Where the persistent browser profile lives. One profile per instance: a
// single X identity is all the archive needs.
@@ -129,26 +148,71 @@ export function isXCookie(c: BrowserCookie): boolean {
return d === "x.com" || d === "twitter.com" || d.endsWith(".x.com") || d.endsWith(".twitter.com");
}
+// `<exe> --version` ("Chromium 153.0.8010.47 Arch Linux"), for the log and the
+// profile's record. Best effort: a browser that does not answer in 5 s is
+// simply not versioned.
+async function browserVersion(b: XBrowserChoice): Promise<string | undefined> {
+ if (b.kind !== "system") return undefined;
+ try {
+ const res = await execa(b.executablePath, ["--version"], { reject: false, timeout: 5_000 });
+ const line = `${res.stdout ?? ""}`.trim().split("\n")[0];
+ return res.exitCode === 0 && line ? line : undefined;
+ } catch {
+ return undefined;
+ }
+}
+
+function firstLine(err: unknown): string {
+ return ((err as Error)?.message ?? String(err)).split("\n")[0];
+}
+
// Launch a HEADED browser against the persistent profile so a human can log in
// (password, 2FA, captcha — all of it). Resolves once the caller closes the
// window, then exports the cookie jar.
//
+// The window is the operator's own browser without the automation signals
+// (xBrowser.ts says which, and why Google's sign-in refused the old one). The
+// executable is recorded in the profile, so a refresh that cannot open the
+// profile with the bundled build can use the one that wrote it.
+//
// Playwright is imported dynamically: this module must stay importable on a
// host with no browsers installed (the Docker build image), where only the
// status/read helpers above are ever called.
export async function connectXAccount(
paths: Paths,
- opts: { onLog?: (line: string) => void; timeoutMs?: number } = {},
-): Promise<XSessionStatus> {
+ opts: {
+ onLog?: (line: string) => void;
+ timeoutMs?: number;
+ env?: Record<string, string | undefined>;
+ } = {},
+): Promise<XSessionStatus & { browser: string }> {
const log = opts.onLog ?? (() => {});
const profileDir = xProfileDir(paths);
await mkdir(profileDir, { recursive: true });
+ const browser = findXBrowser({ env: opts.env });
+ const version = await browserVersion(browser);
+ const label = describeXBrowser(browser, version);
const { chromium } = await importPlaywright();
- log("Opening a browser window. Log in to X, then close the window.");
- const context = await chromium.launchPersistentContext(profileDir, {
- headless: false,
- viewport: { width: 1280, height: 900 },
+ log(`Opening ${label}. Log in to X, then close the window.`);
+ let context: BrowserContextLike;
+ try {
+ context = await chromium.launchPersistentContext(
+ profileDir,
+ buildXBrowserLaunchOptions({ browser, headless: false, sandbox: true }),
+ );
+ } catch (err) {
+ // A host that cannot start Chromium's sandbox (no unprivileged user
+ // namespaces, an AppArmor rule) still gets its window — with the
+ // unsupported-flag bar a system Chrome draws for `--no-sandbox`.
+ log(`The sandboxed launch failed (${firstLine(err)}); opening without the sandbox.`);
+ context = await chromium.launchPersistentContext(
+ profileDir,
+ buildXBrowserLaunchOptions({ browser, headless: false, sandbox: false }),
+ );
+ }
+ await writeXBrowserRecord(profileDir, browser, version).catch((err) => {
+ log(`Could not record the browser in the profile: ${firstLine(err)}`);
});
try {
const page = context.pages()[0] ?? (await context.newPage());
@@ -170,40 +234,135 @@ export async function connectXAccount(
} finally {
await context.close().catch(() => {});
}
- return readXSessionStatus(paths);
+ return { ...(await readXSessionStatus(paths)), browser: label };
}
-// Re-export cookies from the stored profile WITHOUT any human interaction —
-// this is the call gallery-dl's cookie refresh actually uses. Runs headless.
-export async function refreshXCookies(
+// THE PROFILE, OPENED HEADLESS — the Playwright fallback fetcher (profile
+// source) comes through here, and the refresh below reads the profile the same
+// way. The bundled build first (it never logs in, and its headless shell needs
+// no display), without the automation signals; when it cannot open the profile
+// and the profile records a system executable, that one, headless.
+//
+// `chromium` is injectable so the choice of browser is testable without one.
+type ProfileLaunchOpts = { onLog?: (line: string) => void; chromium?: ChromiumLike };
+
+const BUNDLED: XBrowserChoice = { kind: "bundled" };
+
+function openProfile(
+ chromium: ChromiumLike,
+ profileDir: string,
+ browser: XBrowserChoice,
+): Promise<BrowserContextLike> {
+ return chromium.launchPersistentContext(
+ profileDir,
+ buildXBrowserLaunchOptions({ browser, headless: true }),
+ );
+}
+
+export async function launchXProfile(
paths: Paths,
- opts: { onLog?: (line: string) => void } = {},
-): Promise<XSessionStatus> {
+ opts: ProfileLaunchOpts = {},
+): Promise<BrowserContextLike> {
const log = opts.onLog ?? (() => {});
const profileDir = xProfileDir(paths);
- if (!(await exists(profileDir))) {
- throw new Error(
- "No X session profile yet — run the headed 'Connect X account' flow first.",
+ const chromium = opts.chromium ?? (await importPlaywright()).chromium;
+ try {
+ const context = await openProfile(chromium, profileDir, BUNDLED);
+ log("Opened the X session profile with Playwright's bundled Chromium (headless).");
+ return context;
+ } catch (err) {
+ const record = await readXBrowserRecord(profileDir);
+ const recorded = recordedXBrowser(record);
+ if (!recorded) throw err;
+ log(
+ `Playwright's bundled Chromium could not open the profile (${firstLine(err)}); ` +
+ `using ${describeXBrowser(recorded, record?.version)}, headless.`,
);
+ return openProfile(chromium, profileDir, recorded);
}
- const { chromium } = await importPlaywright();
- const context = await chromium.launchPersistentContext(profileDir, {
- headless: true,
- });
+}
+
+// One headless pass: open the profile, touch x.com/home (X rotates/renews the
+// session cookies on a visit, which is the whole point of keeping a live
+// profile), read X's cookies, close.
+async function exportProfileCookies(
+ chromium: ChromiumLike,
+ profileDir: string,
+ browser: XBrowserChoice,
+ log: (line: string) => void,
+): Promise<BrowserCookie[]> {
+ const context = await openProfile(chromium, profileDir, browser);
try {
- // Touching the site lets X rotate/renew the session cookies before we read
- // them, which is the whole point of keeping a live profile.
const page = context.pages()[0] ?? (await context.newPage());
await page
.goto("https://x.com/home", { waitUntil: "domcontentloaded", timeout: 45_000 })
.catch(() => {
log("Could not load x.com/home; exporting whatever the profile holds.");
});
- const cookies = (await context.cookies()) as BrowserCookie[];
- await writeCookieJar(paths, cookies.filter(isXCookie), log);
+ return ((await context.cookies()) as BrowserCookie[]).filter(isXCookie);
} finally {
await context.close().catch(() => {});
}
+}
+
+const hasAuthCookie = (cookies: ReadonlyArray<BrowserCookie>) =>
+ cookies.some((c) => c.name === AUTH_COOKIE);
+
+// Re-export cookies from the stored profile WITHOUT any human interaction —
+// this is the call gallery-dl's cookie refresh actually uses. Runs headless.
+//
+// A LOGGED-IN JAR IS NEVER REPLACED BY ONE WITHOUT A LOGIN. A profile written
+// by a newer system browser can open in the bundled build without an error and
+// without its cookies (a store that build cannot read). So when the bundled
+// read comes back with no auth_token while the jar holds one, the profile is
+// read again with the browser that wrote it (the record); and when no read
+// finds the login, the jar is kept and the refresh refuses, saying so.
+export async function refreshXCookies(
+ paths: Paths,
+ opts: ProfileLaunchOpts = {},
+): Promise<XSessionStatus> {
+ const log = opts.onLog ?? (() => {});
+ const profileDir = xProfileDir(paths);
+ if (!(await exists(profileDir))) {
+ throw new Error(
+ "No X session profile yet — run the headed 'Connect X account' flow first.",
+ );
+ }
+ const chromium = opts.chromium ?? (await importPlaywright()).chromium;
+ const before = await readXSessionStatus(paths);
+ const record = await readXBrowserRecord(profileDir);
+ const recorded = recordedXBrowser(record);
+ const recordedLabel = recorded ? describeXBrowser(recorded, record?.version) : "";
+
+ let cookies: BrowserCookie[];
+ let readWithRecorded = false;
+ try {
+ cookies = await exportProfileCookies(chromium, profileDir, BUNDLED, log);
+ log("Read the X session profile with Playwright's bundled Chromium (headless).");
+ } catch (err) {
+ if (!recorded) throw err;
+ log(
+ `Playwright's bundled Chromium could not open the profile (${firstLine(err)}); ` +
+ `using ${recordedLabel}, headless.`,
+ );
+ cookies = await exportProfileCookies(chromium, profileDir, recorded, log);
+ readWithRecorded = true;
+ }
+ if (!hasAuthCookie(cookies) && before.looksAuthenticated && recorded && !readWithRecorded) {
+ log(
+ "Playwright's bundled Chromium read no X login from a profile whose exported jar holds " +
+ `one; reading it again with ${recordedLabel}, headless.`,
+ );
+ cookies = await exportProfileCookies(chromium, profileDir, recorded, log);
+ }
+ if (!hasAuthCookie(cookies) && before.looksAuthenticated) {
+ throw new Error(
+ "The profile gave no X login (no auth_token), so the logged-in cookie jar" +
+ (before.cookiesUpdatedAt ? ` exported ${before.cookiesUpdatedAt}` : "") +
+ " was kept, not replaced. If the X session has ended, Connect again (or Forget session).",
+ );
+ }
+ await writeCookieJar(paths, cookies, log);
return readXSessionStatus(paths);
}
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -15,6 +15,7 @@
- **A site's Charts tab gives a sixth series its own colour.** The dashboard's charts coloured their series from five colours and started again at the sixth, so a chart broken down by six or more channels drew the sixth in the first one's colour. The sixth now takes the palette's sixth colour, and each from the seventh on a hue of its own; the first five are unchanged. The published sites' charts get the same change with their next build.
- **A form whose save is refused keeps what you typed.** Every editor form put its plain fields back to the stored values when its save was refused — a site's ID rejected, a page size out of range, a slug already taken — so everything typed had to be typed again. A refused save now leaves every field as you left it, beside the reason: **Settings**; a site's form (new and existing); the hub's config on `/sites`; **Cut release**; a channel's form (new and **Configure**), **Rename** and **Delete**; a video's **Delete directory**; **Drive health timing** on `/storage`; the backup config on `/saved-videos`; the sync operation's controls; the **Digest**, **Diarization**, **Speaker attribution** and **Speaker work lane** settings; and the worker list on `/workers`. A save that succeeds behaves as before, with one difference you may notice: a drop-down, and a checkbox or choice that the page tracks as you change it (a cadence, a worker's **Enabled**, a social link's **Keep in header**, a site membership, a site's accent), now shows what was saved. A form's own drop-downs used to go back to what the page had loaded with until a reload, and a second save from the same page sent that old choice again; the others went back until the page next refreshed itself (every 5 seconds by default).
- **A media move no longer starts over a job that is writing into the channel, holds the channel's writers while it runs, and makes its copy match the source before it verifies — so a transcription or a download during a move cannot fail it.** A move that has waited its turn behind other moves now checks again when it starts: if a job is running on the channel, or an auto-queue lane is working on one of its videos, it stops at once and says which ("a transcription of abc123 is running (Transcribe all, job …) — wait for it or cancel it"), with nothing copied — and a job you have just cancelled counts until it has actually stopped ("is stopping … — wait for it to stop"); **Preview** says the same, and the Storage panel's blocked message now names the job too. While a move's marker stands, the channel is held: every lane skips it, and every job that reads or writes its media (single-video transcriptions, downloads and transcodes and the availability checks now included) refuses to start, including one that was already queued when the move began. The rack shows a **media held** chip in the channel's Tier cell and the Storage panel says "Held: its media is moving"; both go when the move finishes or its marker is cleared. The copy is now followed by a pass that makes the destination copy match the source — files the source no longer has are removed from the copy, never from the source — so a file written or deleted during the copy (a transcriber's scratch folder, say) no longer fails the check, and **Resume move** finishes a move whose copy holds such leftovers. Every file removed from a copy is listed in the move's log, and **Preview** says so when a copy from an earlier attempt is already there. If the source keeps changing, the move stops and lists what differs: extra on the destination, missing there, or changed. A new **Reconcile and resume** button beside **Resume move** lists those differences, makes the copy match and finishes the move, so no file has to be deleted by hand. The saved-video store's move does the same matching and the same check before it starts. Needs a rebuild and restart of the editor.
+- **Connecting an X account opens your own browser, and the X fetchers can use your everyday browser's X login instead.** **Settings → X account session → Connect X account** used to open Playwright's bundled Chromium with its automation signals on (the "controlled by automated test software" bar, `navigator.webdriver`): Google's sign-in refused it and X's own login form stalled in it. It now opens your Chromium or Chrome when one is installed (`ARCHILYZER_X_BROWSER` names another; Playwright's bundled Chromium otherwise), without those signals. Google's sign-in may still refuse an embedded browser; X's password login is the reliable path. A new **Login source** choice (`social.x.cookieSource` in `settings.json`) says where the X fetchers' login comes from: **Browser login** hands gallery-dl `--cookies-from-browser` with your `cookiesFromBrowser` on every fetch, so the login lasts as long as you stay logged in to x.com in that browser and no window is needed; **Connected profile** is the session broker, as before. Left on **Automatic**, it is the browser login when `cookiesFromBrowser` is set and no profile is connected, and the profile otherwise. **Check** says which source is in use, whether an X login is visible in it and when it was last used (the browser's cookies are read from a private copy, never written; this reads Firefox's, and gallery-dl reads Chromium's itself). Needs a rebuild and restart of the editor.
## [0.11.0] - 2026-09-30
- **Transcripts that arrived after a video was first seen are counted.** The stats behind the homepage, the hub and every site's charts were cached per video and refreshed only when the video's metadata changed, so a transcript that came later — a Whisper run days after the download, or a video downloaded after the last index build — never reached them, and a video with YouTube captions alone had no transcription date. Counts and charts were low; the homepage could show a site with 0 transcripts, 0 channels and 0 hours while it served its videos. A stat is now also redone whenever the index re-reads the video, every transcript has a date, and a captioned video is dated by when its captions arrived rather than by a later Normalize run, so its place on "Transcribed over time" can move. **After updating, rebuild and restart the editor before anything else:** until then, **Build stats dataset** runs the old code and would undo the new stats, while a site, hub or homepage build already runs the new code — and the first stats build of any kind re-reads every video once (about 10–30 minutes on a large archive; it can be stopped and picks up where it stopped). Then build the index, the stats, the homepage, the hub, and the sites.
diff --git a/editor/app/settings/components/XSessionSection.tsx b/editor/app/settings/components/XSessionSection.tsx
@@ -1,44 +1,108 @@
"use client";
-// "Connect X account" — the operator-facing half of the X session broker.
+// The X account session — where the X fetchers' login comes from, whether a
+// login is visible there, and the operator-facing half of the session broker.
//
-// Why this exists: gallery-dl is the primary X fetcher and its worst flaw is
-// that X cookies expire within days. A persistent, logged-in browser profile
-// removes that problem without touching the fetch path at all — the fetcher
-// just reads a jar this keeps fresh.
+// Two sources (release 16 slice XL, common/social/xCookieSource.ts):
+// - the operator's everyday browser (`cookiesFromBrowser`): gallery-dl reads
+// its X login on every fetch, so nothing expires while the operator stays
+// logged in to x.com there — no window at all;
+// - a profile connected here once: a persistent, logged-in browser profile
+// the fetcher re-exports fresh cookies from, because X cookies otherwise
+// expire within days.
-import { useState, useTransition } from "react";
+import { useEffect, useState, useTransition } from "react";
import {
+ checkXLoginAction,
clearXSessionAction,
connectXAccountAction,
refreshXCookiesAction,
+ setXCookieSourceAction,
+ type XSessionActionResult,
} from "../xSessionActions";
import type { XSessionStatus } from "yt-dlp-transcript-common/social/xSessionBroker";
+import type { XLoginStatus } from "yt-dlp-transcript-common/social/xBrowserLogin";
+import {
+ resolveXCookieSource,
+ xCookieSourceLabel,
+ type XCookieSourceView,
+} from "yt-dlp-transcript-common/social/xCookieSource";
-export function XSessionSection({ initial }: { initial: XSessionStatus }) {
+export function XSessionSection({
+ initial,
+ initialSource,
+}: {
+ initial: XSessionStatus;
+ initialSource: XCookieSourceView;
+}) {
const [status, setStatus] = useState<XSessionStatus>(initial);
+ const [source, setSource] = useState<XCookieSourceView>(initialSource);
+ const [login, setLogin] = useState<XLoginStatus | null>(null);
const [error, setError] = useState<string | null>(null);
const [note, setNote] = useState<string | null>(null);
+ // Two transitions: the session actions (Connect waits for the window to
+ // close) and the source controls (Check, the source select).
const [pending, startTransition] = useTransition();
+ const [checking, startChecking] = useTransition();
+
+ // A save of cookiesFromBrowser above (or another tab's choice) re-renders the
+ // page with a new resolved source; take it.
+ useEffect(() => {
+ setSource(initialSource);
+ // Keyed on the values, not the object: every server render makes a new one.
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [initialSource.source, initialSource.chosen, initialSource.browserSpec]);
- const run = (
- fn: () => Promise<
- { ok: true; status: XSessionStatus } | { ok: false; error: string }
- >,
- okNote: string,
- ) =>
+ const run = (fn: () => Promise<XSessionActionResult>, okNote: (r: { browser?: string }) => string) =>
startTransition(async () => {
setError(null);
setNote(null);
const res = await fn();
if (res.ok) {
setStatus(res.status);
- setNote(okNote);
+ setSource(res.source);
+ setLogin(null);
+ setNote(okNote(res));
+ } else {
+ setError(res.error);
+ }
+ });
+
+ const check = () =>
+ startChecking(async () => {
+ setError(null);
+ setNote(null);
+ const res = await checkXLoginAction();
+ if (res.ok) {
+ setLogin(res.login);
+ setSource(res.login);
} else {
setError(res.error);
}
});
+ const choose = (choice: string) =>
+ startChecking(async () => {
+ setError(null);
+ setNote(null);
+ const res = await setXCookieSourceAction(choice);
+ if (res.ok) {
+ setSource(res.source);
+ setLogin(null);
+ setNote("Login source saved.");
+ } else {
+ setError(res.error);
+ }
+ });
+
+ // What "Automatic" resolves to right now, for its option's text.
+ const automatic = xCookieSourceLabel(
+ resolveXCookieSource({
+ browserSpec: source.browserSpec,
+ profileConnected: status.looksAuthenticated,
+ }),
+ );
+
return (
<section
data-x-session=""
@@ -62,12 +126,68 @@ export function XSessionSection({ initial }: { initial: XSessionStatus }) {
</div>
<p className="text-xs text-muted-foreground">
- X post archiving needs a logged-in session, and X cookies expire within
- days. Connecting an account once stores a browser profile on this host;
- the fetcher then re-exports fresh cookies from it automatically, so the
- session stops being the thing that breaks.
+ X post archiving needs a logged-in session, from one of two sources.{" "}
+ <strong>Browser login</strong>: your everyday browser, named by{" "}
+ <code>cookiesFromBrowser</code> above — gallery-dl reads its X login on
+ every fetch, so nothing expires while you stay logged in to x.com there,
+ and no window is needed. <strong>Connected profile</strong>: a browser
+ profile connected here once; the fetcher re-exports fresh cookies from
+ it, because X cookies otherwise expire within days.
+ </p>
+
+ <div className="flex flex-wrap items-center gap-2">
+ <label htmlFor="x-cookie-source" className="text-sm">
+ Login source
+ </label>
+ <select
+ id="x-cookie-source"
+ aria-label="x cookie source"
+ value={source.chosen ? source.source : "auto"}
+ onChange={(e) => choose(e.target.value)}
+ disabled={pending || checking}
+ className="rounded-md border border-border bg-background px-2 py-1 text-sm disabled:opacity-50"
+ >
+ <option value="auto">Automatic — now: {automatic}</option>
+ <option value="browser">
+ Browser login{source.browserSpec ? ` (${source.browserSpec})` : ""}
+ </option>
+ <option value="profile">Connected profile</option>
+ </select>
+ <button
+ type="button"
+ onClick={check}
+ disabled={pending || checking}
+ aria-label="check x login"
+ className="rounded-md border border-border px-3 py-1 text-sm hover:bg-muted disabled:opacity-50"
+ >
+ {checking ? "Checking…" : "Check"}
+ </button>
+ </div>
+
+ <p aria-label="x cookie source in use" className="text-xs text-muted-foreground">
+ In use: {source.label}
+ {source.chosen ? "" : " (automatic)"}
</p>
+ {login && (
+ <p
+ aria-label="x login status"
+ className={
+ "text-xs " +
+ (login.authTokenVisible === true && !login.walOnly
+ ? "text-success"
+ : login.authTokenVisible === false
+ ? "text-destructive"
+ : "text-muted-foreground")
+ }
+ >
+ {login.summary}{" "}
+ <span className="text-muted-foreground">
+ (checked {new Date(login.checkedAt).toLocaleTimeString()})
+ </span>
+ </p>
+ )}
+
{status.cookiesUpdatedAt && (
<p className="text-xs text-muted-foreground">
Cookies last exported:{" "}
@@ -79,7 +199,9 @@ export function XSessionSection({ initial }: { initial: XSessionStatus }) {
<button
type="button"
onClick={() =>
- run(connectXAccountAction, "Connected. Cookies exported.")
+ run(connectXAccountAction, (r) =>
+ `Connected${r.browser ? ` with ${r.browser}` : ""}. Cookies exported.`,
+ )
}
disabled={pending}
aria-label="connect x account"
@@ -89,7 +211,7 @@ export function XSessionSection({ initial }: { initial: XSessionStatus }) {
</button>
<button
type="button"
- onClick={() => run(refreshXCookiesAction, "Cookies refreshed.")}
+ onClick={() => run(refreshXCookiesAction, () => "Cookies refreshed.")}
disabled={pending || !status.hasProfile}
aria-label="refresh x cookies"
className="rounded-md border border-border px-3 py-1 text-sm hover:bg-muted disabled:opacity-50"
@@ -98,7 +220,7 @@ export function XSessionSection({ initial }: { initial: XSessionStatus }) {
</button>
<button
type="button"
- onClick={() => run(clearXSessionAction, "Session cleared.")}
+ onClick={() => run(clearXSessionAction, () => "Session cleared.")}
disabled={pending || !status.hasProfile}
aria-label="clear x session"
className="rounded-md border border-border px-3 py-1 text-sm hover:bg-muted disabled:opacity-50"
@@ -109,9 +231,13 @@ export function XSessionSection({ initial }: { initial: XSessionStatus }) {
<p className="text-xs text-muted-foreground">
Connecting opens a browser window <strong>on the machine running the
- editor</strong>; complete the login (2FA and captcha included), then
- close the window. The login is never automated — that is what gets
- accounts flagged.
+ editor</strong>: your own Chromium or Chrome when one is installed (else
+ Playwright's bundled Chromium), without the automation signals.
+ Complete the login (2FA and captcha included), then close the window.
+ Google's sign-in may still refuse an embedded browser (“this
+ browser or app may not be secure”) — <strong>X's own
+ password login is the reliable path</strong>. The login is never
+ automated — that is what gets accounts flagged.
</p>
{note && (
diff --git a/editor/app/settings/page.tsx b/editor/app/settings/page.tsx
@@ -3,6 +3,8 @@ import Link from "next/link";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import { getSettings } from "yt-dlp-transcript-common/lib/settings";
import { readXSessionStatus } from "yt-dlp-transcript-common/social/xSessionBroker";
+import { resolveXCookieSourceFor } from "yt-dlp-transcript-common/social/xBrowserLogin";
+import { xCookieSourceView } from "yt-dlp-transcript-common/social/xCookieSource";
import { SettingsForm } from "./components/SettingsForm";
import { XSessionSection } from "./components/XSessionSection";
@@ -29,8 +31,13 @@ export default async function SettingsPage() {
["galleryDlBin", paths.galleryDlBin],
];
- // The X session broker's current state (see xSessionActions.ts).
+ // The X session broker's current state and the X login source in use (see
+ // xSessionActions.ts). The source is resolved here, not read: with no stored
+ // choice it depends on cookiesFromBrowser and the profile.
const xSession = await readXSessionStatus(paths);
+ const xSource = xCookieSourceView(
+ await resolveXCookieSourceFor(paths, settings),
+ );
return (
<div className="flex flex-col gap-8">
@@ -86,7 +93,7 @@ export default async function SettingsPage() {
</section>
<section className="flex flex-col gap-3 border-t border-border pt-6">
- <XSessionSection initial={xSession} />
+ <XSessionSection initial={xSession} initialSource={xSource} />
</section>
<section className="flex flex-col gap-3 border-t border-border pt-6">
diff --git a/editor/app/settings/xSessionActions.ts b/editor/app/settings/xSessionActions.ts
@@ -1,16 +1,21 @@
"use server";
-// Server actions for the X session broker (common/social/xSessionBroker.ts).
+// Server actions for the X session broker (common/social/xSessionBroker.ts)
+// and the X login source (common/social/xCookieSource.ts, release 16 slice XL).
//
// "Connect X account" opens a HEADED browser on the editor host so the operator
// can log in by hand — password, 2FA and captcha included. That is the whole
// point: automating an X login is what gets accounts flagged, and a human doing
-// it once produces a profile that stays valid.
+// it once produces a profile that stays valid. The window is the operator's own
+// Chromium or Chrome when one is installed, without the automation signals
+// (common/social/xBrowser.ts).
//
// Because it opens a window on the SERVER's display, this is deliberately an
// explicit operator action and never something a fetch triggers on its own.
+import { revalidatePath } from "next/cache";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { getSettings } from "yt-dlp-transcript-common/lib/settings";
import {
clearXSession,
connectXAccount,
@@ -18,28 +23,49 @@ import {
refreshXCookies,
type XSessionStatus,
} from "yt-dlp-transcript-common/social/xSessionBroker";
+import {
+ readXLoginStatus,
+ resolveXCookieSourceFor,
+ type XLoginStatus,
+} from "yt-dlp-transcript-common/social/xBrowserLogin";
+import {
+ isXCookieSource,
+ xCookieSourceView,
+ type XCookieSourceView,
+} from "yt-dlp-transcript-common/social/xCookieSource";
+import { saveSettings } from "./saveSettings";
+// Every session action returns the source in use beside the profile's status:
+// connecting or forgetting a profile can move the read-time default.
export type XSessionActionResult =
- | { ok: true; status: XSessionStatus }
+ | { ok: true; status: XSessionStatus; source: XCookieSourceView; browser?: string }
| { ok: false; error: string };
+async function sourceNow(): Promise<XCookieSourceView> {
+ return xCookieSourceView(await resolveXCookieSourceFor(getPaths(), getSettings()));
+}
+
export async function xSessionStatusAction(): Promise<XSessionStatus> {
return readXSessionStatus(getPaths());
}
export async function connectXAccountAction(): Promise<XSessionActionResult> {
try {
- const status = await connectXAccount(getPaths());
+ const { browser, ...status } = await connectXAccount(getPaths(), {
+ // Which browser opened, and any fallback, in the editor's log.
+ onLog: (line) => console.log(`[x-session] ${line}`),
+ });
if (!status.looksAuthenticated) {
return {
ok: false,
error:
- "The browser closed without a logged-in X session (no auth_token " +
- "cookie was exported). Try again and complete the login before " +
- "closing the window.",
+ `The browser (${browser}) closed without a logged-in X session (no ` +
+ "auth_token cookie was exported). Try again and complete the login " +
+ "before closing the window — X's own password login is the reliable " +
+ "path where Google's sign-in refuses.",
};
}
- return { ok: true, status };
+ return { ok: true, status, source: await sourceNow(), browser };
} catch (e) {
return { ok: false, error: (e as Error).message };
}
@@ -47,7 +73,10 @@ export async function connectXAccountAction(): Promise<XSessionActionResult> {
export async function refreshXCookiesAction(): Promise<XSessionActionResult> {
try {
- return { ok: true, status: await refreshXCookies(getPaths()) };
+ const status = await refreshXCookies(getPaths(), {
+ onLog: (line) => console.log(`[x-session] ${line}`),
+ });
+ return { ok: true, status, source: await sourceNow() };
} catch (e) {
return { ok: false, error: (e as Error).message };
}
@@ -56,8 +85,51 @@ export async function refreshXCookiesAction(): Promise<XSessionActionResult> {
export async function clearXSessionAction(): Promise<XSessionActionResult> {
try {
await clearXSession(getPaths());
- return { ok: true, status: await readXSessionStatus(getPaths()) };
+ return {
+ ok: true,
+ status: await readXSessionStatus(getPaths()),
+ source: await sourceNow(),
+ };
+ } catch (e) {
+ return { ok: false, error: (e as Error).message };
+ }
+}
+
+// THE CHECK: the source in use, whether an X login (an auth_token for x.com)
+// is visible in it, and when it was last seen. Reads the browser's cookie store
+// read-only (a private copy) or the profile's exported jar.
+export type XLoginCheckResult =
+ | { ok: true; login: XLoginStatus }
+ | { ok: false; error: string };
+
+export async function checkXLoginAction(): Promise<XLoginCheckResult> {
+ try {
+ return { ok: true, login: await readXLoginStatus(getPaths(), getSettings()) };
+ } catch (e) {
+ return { ok: false, error: (e as Error).message };
+ }
+}
+
+// THE CHOICE: "browser", "profile", or "auto" — no stored choice, the default
+// resolved at read time. Writes `social.x.cookieSource` through the one
+// settings writer and nothing else.
+export type XSourceChoiceResult =
+ | { ok: true; source: XCookieSourceView }
+ | { ok: false; error: string };
+
+export async function setXCookieSourceAction(
+ choice: string,
+): Promise<XSourceChoiceResult> {
+ if (choice !== "auto" && !isXCookieSource(choice)) {
+ return { ok: false, error: `Unknown X login source "${choice}".` };
+ }
+ try {
+ await saveSettings({
+ social: { x: choice === "auto" ? {} : { cookieSource: choice } },
+ });
} catch (e) {
return { ok: false, error: (e as Error).message };
}
+ revalidatePath("/settings");
+ return { ok: true, source: await sourceNow() };
}
diff --git a/editor/e2e/x-session.spec.ts b/editor/e2e/x-session.spec.ts
@@ -1,9 +1,15 @@
-// The X session broker UI (common/social/xSessionBroker.ts). The headed login
+// The X session broker UI (common/social/xSessionBroker.ts) and the X login
+// source (common/social/xCookieSource.ts, release 16 slice XL). The headed login
// is never driven here — it would open a real browser and hit x.com, which the
-// suite must never do.
+// suite must never do. The browser source reads a FIXTURE Firefox store written
+// under the test corpus, never a real browser profile.
import { test, expect } from "@playwright/test";
-import { resetData } from "./helpers";
+import { readJson, resetData, resolvePath, writeSettings } from "./helpers";
+// By relative path: a runtime import of the package specifier does not resolve
+// under playwright's loader (digest.spec.ts says why).
+import { writeFirefoxCookieStore } from "../../common/social/__fixtures__/firefoxCookieStore";
+
test("settings page renders the X session section", async ({ page }) => {
await resetData("empty");
await page.goto("/settings");
@@ -13,4 +19,60 @@ test("settings page renders the X session section", async ({ page }) => {
// never click it (it would open a real browser window and hit x.com).
await expect(page.getByLabel("connect x account")).toBeVisible();
await expect(page.getByLabel("refresh x cookies")).toBeDisabled();
+ // No cookiesFromBrowser in the fixture settings and no profile: the
+ // read-time default is the profile.
+ await expect(page.getByLabel("x cookie source", { exact: true })).toHaveValue("auto");
+ await expect(page.getByLabel("x cookie source in use")).toHaveText(
+ "In use: Connected profile (automatic)",
+ );
+});
+
+test("the login source select persists, and Check shows a status line", async ({ page }) => {
+ await resetData("empty");
+ const profileDir = resolvePath("test-transcripts/firefox-fixture/xl.default-release");
+ await writeFirefoxCookieStore(profileDir, [
+ {
+ host: ".x.com",
+ name: "auth_token",
+ value: "fixture-not-a-real-token",
+ lastAccessed: Date.UTC(2026, 9, 1, 8, 30, 0) * 1000,
+ },
+ { host: ".example.com", name: "session", value: "not-x" },
+ ]);
+ const spec = `firefox:${profileDir}`;
+ await writeSettings({ cookiesFromBrowser: spec });
+
+ await page.goto("/settings");
+ const section = page.locator("[data-x-session]");
+ const select = page.getByLabel("x cookie source", { exact: true });
+ const inUse = page.getByLabel("x cookie source in use");
+
+ // cookiesFromBrowser set, no profile connected: the browser, by default.
+ await expect(select).toHaveValue("auto");
+ await expect(inUse).toHaveText(`In use: Browser login (${spec}) (automatic)`);
+
+ await page.getByLabel("check x login").click();
+ await expect(page.getByLabel("x login status")).toContainText(
+ "An X login is visible in firefox, outside its containers (where gallery-dl reads); " +
+ "its auth_token was last used 2026-10-01 08:30:00 UTC.",
+ );
+
+ // Choose the profile: stored, shown, and still chosen after a reload.
+ await select.selectOption("profile");
+ await expect(section.getByRole("status")).toHaveText("Login source saved.");
+ await expect(inUse).toHaveText("In use: Connected profile");
+ expect((await readJson<{ social?: unknown }>("test-settings.json")).social).toEqual({
+ x: { cookieSource: "profile" },
+ });
+
+ await page.reload();
+ await expect(select).toHaveValue("profile");
+ await expect(inUse).toHaveText("In use: Connected profile");
+ await page.getByLabel("check x login").click();
+ await expect(page.getByLabel("x login status")).toContainText("No profile is connected.");
+
+ // Back to automatic: nothing stored, the default decides again.
+ await select.selectOption("auto");
+ await expect(inUse).toHaveText(`In use: Browser login (${spec}) (automatic)`);
+ expect((await readJson<{ social?: unknown }>("test-settings.json")).social).toEqual({ x: {} });
});
diff --git a/plans/FACTS.md b/plans/FACTS.md
@@ -8193,3 +8193,97 @@ phase deletes from the destination.
- **Not covered:** a corpus-wide
writer mid-channel when a move starts; a resumed or reconciled move-out is charged the whole tree
against the destination's free space, not the remainder (move-back charges the remainder).
+
+## Connecting an X account (verified 2026-10-01, branch `r16/x-login`)
+
+- **Why Google refused the Connect window: the automation signals.** Measured on this box under
+ Xvfb (headed, no window on the operator's display; the slice's `xl-exp/`), Playwright 1.59.1
+ driving system Chromium 153 and the bundled build:
+ - Playwright's defaults: `navigator.webdriver === true` and the bar "Chrome is being controlled
+ by automated test software" (that is `--enable-automation`).
+ - `ignoreDefaultArgs: ["--enable-automation"]` alone: the bar goes, **`navigator.webdriver` stays
+ true** — Playwright's `--remote-debugging-pipe` sets it. `--disable-blink-features=
+ AutomationControlled` is what turns it off (false, headed and headless).
+ - A system Chrome then draws "You are using an unsupported command-line flag" for
+ `--disable-blink-features=AutomationControlled`, and for Playwright's default `--no-sandbox`.
+ `chromiumSandbox: true` removes the second (the sandbox starts on this box for both builds).
+ `--test-type` removes the first: it is Chromium's internal test-harness switch, which makes the
+ browser skip its startup bars (`AddInfoBarsIfNecessary` returns before the bad-flags prompt), and
+ it sets no automation signal (measured in review: `navigator.webdriver`, `window.chrome`, the
+ user agent and the plugin count are the same with and without it; `runtime_features.cc` turns
+ `AutomationControlled` on only for `--enable-automation`, `--headless` and the debugging pipe or
+ port). ChromeDriver passes `--test-type=webdriver`; the bare switch is used here. It is NOT the
+ documented route: that is the `CommandLineFlagSecurityWarningsEnabled` policy, machine-wide,
+ root to set, and it would silence the operator's everyday browser too. The bundled Chrome for
+ Testing draws neither bar (it honours Playwright's `--disable-infobars`).
+ - Not hidden by any of this: the debugging pipe itself, and the headless user agent
+ (`HeadlessChrome/…`). Google's sign-in may still refuse; X's password login is the reliable path.
+- **The launch options are one pure builder**, `buildXBrowserLaunchOptions`
+ (`common/social/xBrowser.ts`): `ignoreDefaultArgs: ["--enable-automation"]` ONLY, `args`
+ `--disable-blink-features=AutomationControlled` and `--test-type`, the sandbox on for the headed
+ window (it retries without when the host cannot start one). **Never `ignoreDefaultArgs: true`:**
+ it also drops `--password-store=basic`, and a system Chromium would then encrypt the profile's
+ cookies with the desktop keyring's key, which the bundled headless build that refreshes the jar
+ cannot read.
+- **Which browser opens.** `ARCHILYZER_X_BROWSER` (a path, or a name on PATH; one that is not an
+ executable refuses the connect), else the first of `chromium`, `google-chrome`,
+ `google-chrome-stable`, `chrome` on PATH, else Playwright's bundled Chromium. Connect logs which
+ (`[x-session] Opening Chromium 153.0.8010.47 Arch Linux at /usr/bin/chromium (found on PATH)…`).
+- **The bundled build is chromium-1217, Chrome for Testing 147.0.7727.15** (and its headless shell,
+ the same revision): `playwright-core@1.59.1`'s `browsers.json`. `~/.cache/ms-playwright` also holds
+ chromium-1234 (151) and two older revisions; this Playwright does not use them.
+- **The profile/executable pairing.** A connect writes `archilyzer-browser.json` into the profile
+ dir (`{ executablePath | null, version, recordedAt }`). The headless refresh and the Playwright
+ fallback fetcher open the profile with the bundled build and fall back to the recorded executable
+ when the bundled launch throws. **The refresh never replaces a jar holding an `auth_token` with one
+ that has none:** a bundled read that comes back without the login is read again with the recorded
+ executable, and when no read finds it the jar is kept and the refresh refuses, saying so — a
+ profile written by a newer system browser could open in the bundled build without an error and
+ without its cookies. Measured: a profile written by system Chromium 153 (a cookie added, headed and
+ headless) opens in the bundled headless shell 147 with the cookie's value readable; `Last Version`
+ stays `153.0.8010.47`; 153 re-opens it after. So the fallback is a guard, not the common path.
+- **The X login source, `social.x.cookieSource`** (`common/social/xCookieSource.ts`, pure).
+ `"browser"` | `"profile"`; absent = **resolved at read time, never stored**: `"browser"` when
+ `cookiesFromBrowser` is set and no profile is connected (connected = the broker's exported jar
+ carries an `auth_token`, `readXSessionStatus().looksAuthenticated`), else `"profile"`. A stored
+ choice always wins. The fetch resolves it per channel against the channel's own
+ `cookiesFromBrowser` over the global one (`fetchPosts` → `resolveXCookieSourceFor`).
+ **The source, not `cookieMode`, governs the X fetchers**: the browser source passes the spec
+ whatever the mode; the profile source keeps the old order (the jar, else the `"always"`-mode
+ spec, else a guest run).
+- **gallery-dl 1.32.9** takes `--cookies-from-browser BROWSER[/DOMAIN][+KEYRING][:PROFILE][::CONTAINER]`
+ (yt-dlp's syntax plus `/DOMAIN`) and reads every browser it supports, Chromium's encrypted store
+ included, on each run. The spec is passed verbatim (`galleryDlCookieChoice`).
+- **Reading Firefox's store ourselves** (`common/social/xBrowserLogin.ts`): for the Playwright
+ fallback and the Settings "Check". The store is found much as gallery-dl finds it — a profile path
+ or name from the spec, else the most recently modified `cookies.sqlite` under
+ `~/.config/mozilla/firefox`, `~/.mozilla/firefox`, Snap, Flatpak, macOS (two levels deep). Not
+ exactly: gallery-dl also honours `$XDG_CONFIG_HOME` and a second Flatpak root
+ (`~/.var/app/org.mozilla.firefox/config/mozilla/firefox`), and for a profile NAME takes the first
+ root holding `<name>/cookies.sqlite` where this takes the newest.
+ `cookies.sqlite` **and its `-wal`** are copied back to back to a private `mkdtemp` dir, and the db
+ alone copied again from there; both are read and removed; only X's rows are selected; the profile
+ is never opened in place. **Two views:** with the WAL (what Firefox holds now — the Playwright
+ fallback uses it) and the main file alone (what gallery-dl sees: it opens `cookies.sqlite`
+ `mode=ro&immutable=1`, which ignores the WAL, and its fallback copies the db file only). A fresh
+ login sits in the WAL until Firefox checkpoints (closing Firefox does it); the Check says so when
+ the `auth_token` is in the WAL only.
+ **Containers are read as gallery-dl reads them** (`cookies.py` `_firefox_cookies_database`): no
+ `::CONTAINER`, or `::none`, = only cookies outside every container (gallery-dl's default; yt-dlp
+ reads them all); `::all` = no filter; `::NAME` = the `userContextId` whose `name` is NAME, else whose
+ `l10nId` is `user-context-NAME` (or ends `-NAME`), else whose `l10nID` is `userContextNAME.label` —
+ case-sensitive. The Check says when an x.com login sits only in a container the spec does not read.
+ **A login counts on x.com only** (`.x.com`, `x.com`, subdomains): gallery-dl's twitter extractor
+ looks its `auth_token` up on `.x.com`; a twitter.com `auth_token` is reported as the old domain's
+ and not counted (`ct0` is not needed — gallery-dl makes one).
+ `expiry` is seconds (read as ms when too large for seconds); `lastAccessed` is PRTime (µs). The
+ Chromium family is NOT read here (its values are encrypted with a keyring key): the readers say so
+ and gallery-dl reads it.
+- **`node:sqlite`** (Node 22.23 here; unflagged since 22.13, an ExperimentalWarning once per process)
+ is loaded at run time through an assembled specifier with `webpackIgnore`
+ (`common/social/nodeSqlite.ts`), like `playwrightRuntime.ts`: the editor's Turbopack build has
+ nothing to resolve, and the repo's `@types/node` (20) has no types for it, so they are structural.
+- **The e2e never reads a real browser.** `x-session.spec.ts` writes a Firefox store under
+ `test-transcripts/` (`common/social/__fixtures__/firefoxCookieStore.ts`, imported by relative path)
+ and points `cookiesFromBrowser` at it as `firefox:<abs path>` — a profile PATH in the spec, so no
+ home directory is searched.
diff --git a/plans/release-16.md b/plans/release-16.md
@@ -25,7 +25,7 @@ slice's prompt carries its ruling, and this record carries what was built. Rules
| DX | `r16/research-setup` | The research-only setup (source → `pnpm install` → `claude mcp add archilyzer` → `/ask`) in one place, the homepage's AI and MCP doc; the sites' and the hub's Use-with-AI page removed and its links pointed at the doc; `README.md` §1/§4 and `mcp/README.md` their own copies (as amended) | `homepage/content/docs/ai-and-mcp.md`, `export/app/use-with-ai/` (removed), the Use with AI links (`export/app/components/{Header,MobileMenu,Footer}.tsx`, `export/app/(workspace)/ask/page.tsx`), `common/lib/{project,corpus}.ts` + `common/bin/compose-site.ts` (what named the page), `mcp/README.md`, `README.md` §1/§4 (wording only), `homepage/e2e/docs.spec.ts`, `export/e2e{,-hub}/use-with-ai-link.spec.ts` and the specs that visited the page |
| FK | `r16/forms-keep-input` | Every editor form keeps what was typed when its action fails: actions return the submitted values with the error, the shared field helpers seed from them | new `editor/app/lib/formState.ts` + test; `editor/app/components/forms/Field.tsx` and the local `Field`s in `SiteForm.tsx`, `ChannelForm.tsx`; every action that returns `{ok:false,error}`/`{error}` (sites, settings, operations/settingsActions, scheduler, storage, homepageActions, cutReleaseAction, channels, videoActions); the 15 forms the ruling lists; e2e `forms-keep-input.spec.ts` (new) + the existing `sites-crud`, `settings`, `channels` specs; records: `plans/FACTS.md`. As shipped, also `editor/app/components/forms/Controlled.tsx` (new) and one tag swap each in `DurationField`, `SocialLinksField`, `SiteMembershipsSection`, `WorkersField` ("Slice FK, as shipped") |
| RM | `r16/move-holds-writers` | A media move holds the channel's writers and mirrors its copy, so a transcription or download during the move cannot fail it | `common/controller/relocateChannelMedia.ts` (+ the saved-video mover if it shares the code) + tests; the lane runners' per-channel skip (`common/controller/autoRunner.ts`, the backfill/digest/normalize/clip-fetch entry points, `common/lib/channelMediaHold.ts`); `common/jobs/jobKinds.ts` (`needsMedia`); the channel page's Storage panel and the rack's reason text; `editor/e2e/relocate*.spec.ts`; records: `plans/FACTS.md`. As shipped, also `common/controller/{channelWriters,relocateDir}.ts` (+ tests), `common/jobs/streamCommand.ts`, `common/views/channelRow.ts`, `editor/app/channels/lib/{mediaBusy,relocationJob}.ts`, `editor/app/storage/lib/{storeBusy,savedVideosJob}.ts`, `editor/app/api/test/stuck-job/route.ts`; the relocate spec is `editor/e2e/channel-storage.spec.ts` ("Slice RM, as shipped") |
-| XL | `r16/x-login` | Connecting an X account works: the Connect window is the operator's own browser without automation signals, and the fetchers can use the operator's browser login directly (`cookiesFromBrowser`) with no window at all | `common/social/{xSessionBroker,xGalleryDlFetcher,fetchers,playwrightRuntime}.ts` + tests; `common/lib/settingsSchema.ts` (one key) + `SETTINGS.md`; `editor/app/settings/{xSessionActions.ts,components/XSessionSection.tsx}`; `editor/e2e/settings*.spec.ts`; records: `plans/FACTS.md`, `ENVIRONMENT.md` if an env var is added |
+| XL | `r16/x-login` | Connecting an X account works: the Connect window is the operator's own browser without automation signals, and the fetchers can use the operator's browser login directly (`cookiesFromBrowser`) with no window at all | `common/social/{xSessionBroker,xGalleryDlFetcher,fetchers,playwrightRuntime}.ts` + tests; `common/lib/settingsSchema.ts` (one key) + `SETTINGS.md`; `editor/app/settings/{xSessionActions.ts,components/XSessionSection.tsx}`; `editor/e2e/settings*.spec.ts`; records: `plans/FACTS.md`, `ENVIRONMENT.md` if an env var is added. As shipped, also `common/social/{xBrowser,xCookieSource,xBrowserLogin,nodeSqlite}.ts` and `__fixtures__/firefoxCookieStore.ts` (new), `xPlaywrightFetcher.ts`, `common/controller/fetchPosts.ts` (the source per channel), `common/lib/{settingsDocs,envVars}.ts`, `editor/app/settings/page.tsx`, and `editor/e2e/x-session.spec.ts` — the spec that covers the section ("Slice XL, as shipped") |
## Slice CK — the ruling (2026-09-30)
@@ -994,6 +994,260 @@ ask) and found it closed by ordering; that item is struck above.
| 5 | `3cc7a5da` | `channel-storage` (13, the stopping case included), `storage-locations`, `channels-storage-columns` | **24 passed**, 1 failed, 5.4 min — `storage-locations` "a volume that came up somewhere else is re-pointed": the first step's `clickUntil` on **Refresh** never saw the volume's identity written to settings in 30 s (the page showed the probe's identity; the action's write did not land). Nothing in that step is the slice's |
| 6 | `3cc7a5da` | `storage-locations` × 2 | **18 passed**, 0 failed, 2.0 min |
+### Slice XL, as shipped — connecting an X account works (2026-10-01)
+
+Branch `r16/x-login` off `main` `a2229d68`, worktree `~/Projects/r12-source-mirror` (editor 5101,
+test 5111, export 5110), one Opus implementer. Scratch files `xl-*` in the job's `tmp`; the
+measurements behind the launch options are `xl-exp/` (two probe scripts, their screenshots, and a
+smoke of the slice's own modules). The ruling is above ("Slice XL — the ruling").
+
+**Measured first** — Playwright 1.59.1 driving system Chromium 153 and the bundled build, headed
+under Xvfb (never on the operator's display) and headless, on scratch profiles; no page ever
+loaded x.com or Google:
+
+| Launch | `navigator.webdriver` | The window's bar |
+|---|---|---|
+| Playwright's defaults, system Chromium | true | "Chrome is being controlled by automated test software" |
+| `ignoreDefaultArgs: ["--enable-automation"]`, sandbox on | **true** — the debugging pipe sets it | none |
+| + `--disable-blink-features=AutomationControlled`, Playwright's `--no-sandbox` | false | "You are using an unsupported command-line flag: --no-sandbox" |
+| the same, sandbox on | false | "… unsupported command-line flag: --disable-blink-features=AutomationControlled" |
+| + `--test-type` | false | none |
+| the bundled Chrome for Testing, the same options, sandbox on or off | false | none |
+
+So the blink flag is what turns `navigator.webdriver` off, and `--test-type` plus the sandbox is
+what keeps a system Chrome's window free of a warning bar. **The bundled build is chromium-1217,
+Chrome for Testing 147** (`playwright-core@1.59.1`'s `browsers.json`), not the cache's
+chromium-1234 (151), which this Playwright does not use. **A profile written by system Chromium 153
+opens in the bundled headless shell 147 with its cookies readable** (a cookie added in 153, read back
+by value in 147; `Last Version` stays 153; 153 re-opens it after), so the refresh keeps the bundled
+build and the recorded executable is a fallback, not the common path.
+
+**What it does.**
+- **The Connect window is the operator's own browser without the automation signals**
+ (`common/social/xBrowser.ts`, used by `connectXAccount`): `ARCHILYZER_X_BROWSER` (a path or a name
+ on PATH; one that is not an executable refuses the connect — it never opens another browser),
+ else the first of `chromium`, `google-chrome`, `google-chrome-stable`, `chrome` on PATH, else the
+ bundled Chromium. One pure options builder, `buildXBrowserLaunchOptions`:
+ `ignoreDefaultArgs: ["--enable-automation"]` only (never `true`: that would drop
+ `--password-store=basic` and a system Chromium would encrypt the profile's cookies with the desktop
+ keyring's key, unreadable by the bundled refresh), `--disable-blink-features=AutomationControlled`,
+ `--test-type`, the sandbox on for the headed window (retried without it, and logged, when the host
+ cannot start one). Connect logs the browser (`[x-session] Opening Chromium 153.0.8010.47 Arch
+ Linux at /usr/bin/chromium (found on PATH). …`) and writes it into the profile
+ (`archilyzer-browser.json`); its success note names it.
+- **The profile opens headless the same way for the refresh and the Playwright fallback fetcher**
+ (`launchXProfile`, and the refresh's own pass): the bundled build with the same signals dropped,
+ the recorded executable when the bundled launch throws. **The refresh never replaces a logged-in jar
+ with one without a login** (the review's L4): a bundled read with no `auth_token` while the jar holds
+ one is read again with the recorded executable, and when no read finds the login the jar is kept
+ and the refresh refuses, saying so ("…the logged-in cookie jar exported <when> was kept, not
+ replaced…").
+- **The login source, `social.x.cookieSource`** (`common/social/xCookieSource.ts`, pure; the one new
+ settings key, `SETTINGS.md` and `settings.json.example` regenerated): `"browser"` | `"profile"`,
+ absent by default and **resolved at read time** — `"browser"` when `cookiesFromBrowser` is set and
+ no profile is connected (no exported jar carrying an `auth_token`), else `"profile"`. `fetchPosts`
+ resolves it per channel (the channel's own `cookiesFromBrowser` over the global) and hands the X
+ fetchers `cookieSource` and `browserCookies` (`PostFetchInput`, `common/social/fetchers.ts`).
+ **The source, not `cookieMode`, governs X**: the browser spec is passed whatever the mode.
+ - **gallery-dl** (`galleryDlCookieChoice`, pure): browser → `--cookies-from-browser <spec>` verbatim
+ (gallery-dl's `BROWSER[/DOMAIN][+KEYRING][:PROFILE][::CONTAINER]`), never the jar, fresh on every
+ run; browser with no spec → a guest run that says why; profile → the jar, else the
+ `"always"`-mode spec, else a guest (the order before the choice existed). The job log names the
+ source each run.
+ - **The Playwright fallback**: browser → a fresh headless context carrying the cookies
+ `xCookiesFromBrowser` reads now; a browser it cannot read refuses by name instead of running
+ logged out. Profile → `launchXProfile`, as before.
+ - **Nitter** uses no X login and is unchanged.
+- **`xCookiesFromBrowser(spec)` and `readXLoginStatus(paths, settings)`**
+ (`common/social/xBrowserLogin.ts`). The read takes the resolved SPEC, not the settings, so a
+ channel's own `cookiesFromBrowser` holds. **Firefox only, read-only**: the store is found much as
+ gallery-dl finds it (a profile path or name, else the most recently modified `cookies.sqlite` under
+ the Firefox roots), `cookies.sqlite` and its `-wal` are copied into a private `mkdtemp` dir, X's rows
+ alone are selected, the copies are removed. **It reads what gallery-dl reads** (the review's M1, L3):
+ the container as gallery-dl reads it — no `::CONTAINER` (or `::none`) is only the cookies outside
+ every container, `::all` is every container, `::NAME` one container matched as gallery-dl matches —
+ and a login counts only as an `auth_token` on x.com. Two views of one copy: with the WAL (what
+ Firefox holds now; the Playwright fallback uses it) and the main file alone (what gallery-dl's
+ immutable open sees). **Chromium's encrypted store is not read here** — gallery-dl (and yt-dlp) read
+ it; for such a spec the Check and the fallback say so. The status: the source in use and whether it
+ was chosen, whether an `auth_token` for x.com is visible (`null` when the source cannot be read), and
+ when it was last seen — the browser's own `lastAccessed` for that cookie, or the jar's export time
+ for the profile; and, for the browser, three plain lines when they apply: the login is in Firefox's
+ write-ahead log only ("Seen in Firefox's write-ahead log only; gallery-dl will see it after Firefox
+ checkpoints (closing Firefox does it)."), an old-domain cookie (a twitter.com `auth_token`) is
+ present and not used, and an x.com login sits only in a container the spec does not read.
+ `node:sqlite` is loaded at run time (`common/social/nodeSqlite.ts`).
+- **The Settings section** (`XSessionSection.tsx`, `xSessionActions.ts`, `settings/page.tsx`): the
+ intro names the two sources; a **Login source** select (Automatic — now: …, Browser login,
+ Connected profile) saved on change through `saveSettings` (Automatic stores `social.x` empty); the
+ source in use; **Check**, whose result is one line; the Connect text says the window is the
+ operator's own browser, that Google's sign-in may still refuse an embedded browser, and that X's
+ password login is the reliable path. Every session action also returns the source in use
+ (connecting or forgetting a profile can move the default).
+- `ARCHILYZER_X_BROWSER` is declared in `common/lib/envVars.ts` (runtime) and `ENVIRONMENT.md`
+ regenerated.
+
+**New accessible names (contracts from here on):** `x cookie source` (the select — `getByLabel`
+needs `{ exact: true }`, the next one contains it), `x cookie source in use`, `check x login`,
+`x login status`. Unchanged: `x session state`, `connect x account`, `refresh x cookies`,
+`clear x session`, the `data-x-session` hook and the `role="status"` / `role="alert"` lines.
+
+**Commits**
+
+| Commit | What |
+|---|---|
+| `cfb6dd10` | `common:` the Connect browser and its options, the login source, the browser-login reader and status, the fetchers, the `social` settings block; SETTINGS.md, settings.json.example, ENVIRONMENT.md |
+| `a266b18a` | `editor:` the X account session's source select, Check and Connect text; the actions; `x-session.spec.ts` |
+| `f295c263` | `plans:` this section; FACTS; the editor changelog |
+| `457f5961` | `plans:` the e2e run |
+| `b3ee652b` | `common:` the review's fixes — M1 (gallery-dl's containers), L1 (the WAL-only line), L3 (x.com-only logins), L4 (the refresh keeps a logged-in jar; `xSessionBroker.test.ts`), I1 (the `--test-type` wording) |
+| `9275ec44` | `plans:` the review, its record and FACTS lines, the gates after it |
+| `86b48d6b` | merge of `main` `ae23a695` (slice RM) |
+| this commit | `plans:` the re-review, the merge, the gates after it |
+
+#### Gates (logs `$T/xl-*.log`)
+
+- **tsc** (all workspaces) clean, 126 s, at `a266b18a`'s tree — after deleting the worktree's stale
+ `export/.next/types` (an old build's, still naming the removed `/use-with-ai` page).
+- **common:** 2,439/2,439, 113 s — the new `xBrowser.test.ts` (9), `xCookieSource.test.ts` (7),
+ `xBrowserLogin.test.ts` (11), `xGalleryDlFetcher.test.ts` (7) and one more `settingsSchema` case;
+ `settingsDocs`, `envVars` (both generated files) and `architecture` green.
+ `archilyzer docs files --check` clean. **Editor unit:** 109/109 (unchanged).
+- **Build:** the capped editor build with the corpus linked (the worktree's own `transcripts/` set
+ aside, `ln -sT` the primary's, `systemd-run --scope -p MemoryMax=5G`, `pnpm --filter editor exec
+ next build`, the link removed and the directory put back): exit 0, 73 s, 1.64 GB peak RSS.
+- **Smoke of the slice's modules** (`xl-exp/smoke.mts`, under Xvfb, a scratch corpus, no
+ navigation): `findXBrowser` picked `/usr/bin/chromium`; the Connect options gave
+ `navigator.webdriver === false`; the record was written; `launchXProfile` opened the profile with
+ the bundled headless build and read the cookie back by value, `navigator.webdriver === false`.
+- **Numbers tool:** none.
+
+ | Run | At | Specs | Result |
+ |---|---|---|---|
+ | 1 | `a266b18a` | `x-session`, `settings`, `cookies-mode`, `forms-keep-input`, `social-channel`, `social-fetcher-picker` | **46 passed**, 0 failed, 3.7 min (after 25 min in the queue behind RM's full suite) |
+
+**Not tested, and cannot be here:** a real Google or X login in the Connect window, and gallery-dl
+or the fallback against x.com — the suite never loads X, and no test reads a real browser profile.
+The operator verifies by hand after the restart (below).
+
+#### Found and left
+
+- **Google may still refuse.** The debugging pipe Playwright drives the window through is still
+ there, and a sign-in page can look for more than `navigator.webdriver`. The section says so and
+ points at X's password login.
+- **The headless paths still say `HeadlessChrome`** in their user agent (the refresh and the
+ Playwright fallback); only `navigator.webdriver` is off there.
+- **Chromium-family specs are read by gallery-dl only.** With `cookiesFromBrowser` naming chrome or
+ chromium, the browser source works for gallery-dl; the Check reports "cannot be read here" (`null`,
+ not "no login") and the Playwright fallback refuses. Reading Chromium's store would mean the
+ keyring decryption gallery-dl and yt-dlp already carry.
+- **The default reads "connected" as "an exported jar with an auth_token".** A profile that is
+ logged in but has never exported (Connect closed before the jar was written) counts as not
+ connected, so the default stays on the browser until a refresh or a connect writes the jar.
+- **The ExperimentalWarning for `node:sqlite`** prints once per editor process, the first time the
+ Check or the fallback reads a browser store.
+- **Not run against the operator's own Firefox.** No run read a real browser profile, so whether
+ `firefox` resolves to the profile the operator logs in to X with (the most recently used one) is
+ the first thing the Check shows after the restart.
+- **Store discovery is close to gallery-dl's, not identical** (review L2). gallery-dl also honours
+ `$XDG_CONFIG_HOME` and a second Flatpak root (`~/.var/app/org.mozilla.firefox/config/mozilla/firefox`),
+ and for a profile NAME it takes the first root holding `<name>/cookies.sqlite` where this reader
+ takes the newest across roots. On a stock Linux Firefox (one root in use) both pick the same store.
+- **From the restart on, every X fetch runs as the operator's own X account** (review I2): the live
+ `settings.json` has `cookiesFromBrowser: "firefox"`, no `social` key and no exported jar, so the
+ default is the browser and every X fetch, scheduled syncs included, runs logged in as the everyday
+ account with no click. As ruled; the operator's steps below say it.
+- **gallery-dl is handed the whole spec** (review I3): `--cookies-from-browser firefox`, with no
+ `/DOMAIN`, loads every cookie outside a container from the operator's Firefox into gallery-dl's
+ process (it sends only X's to X). Giving gallery-dl's argv alone `firefox/.x.com…` would narrow it;
+ not done.
+- **A checkpoint between the two copies** (review I4): the db is copied before its WAL, so a
+ checkpoint and WAL reset in that instant can pair them wrongly — an "unreadable" result, reported as
+ such, or a stale view. Rare; no retry was added.
+- **The whole `cookies.sqlite` sits in a private temp dir during a read** (review I5): every site's
+ rows, mode 0700 under `os.tmpdir()`, removed in a `finally`; only a hard kill leaves it behind.
+
+#### Decisions the operator could overturn
+
+| What I did | The alternative |
+|---|---|
+| `--test-type` in the launch args and the sandbox on for the headed window, so a system Chrome draws no warning bar. `--test-type` is Chromium's internal test-harness switch that skips the startup bars, not the documented route (the machine-wide `CommandLineFlagSecurityWarningsEnabled` policy); it sets no automation signal (measured in review). ChromeDriver passes `--test-type=webdriver`; the bare switch is used | The ruling's two flags alone: the window shows "unsupported command-line flag" for the blink flag (or for `--no-sandbox`) |
+| The refresh and the fallback drop the same signals headless (`launchXProfile`) | Leave them on Playwright's defaults; the ruling named only the Connect window |
+| `ARCHILYZER_X_BROWSER` is a runtime variable read in `xBrowser.ts` | A `paths` entry in `getPaths()` beside `GALLERY_DL_BIN` (it would add a field every `Paths` literal must carry) |
+| "Automatic" is a third option in the select, storing nothing | Two options only: the read-time default would apply until the first choice and never again |
+| The browser source passes the spec whatever `cookieMode` says | Gate it on `cookieMode` like yt-dlp's cookies (`defer` would then run X as a guest) |
+| The browser source's Playwright fallback refuses a browser it cannot read | Fall back to the profile, or run logged out |
+| `xCookiesFromBrowser(spec)` takes the resolved spec | The prompt's `(paths, settings)`: a channel's own `cookiesFromBrowser` would be ignored |
+
+#### After the restart, by hand (the operator)
+
+0. **Every X fetch, scheduled syncs included, runs as your own account from now on** — your
+ everyday Firefox's X login, with no click (the default with `cookiesFromBrowser: "firefox"` and no
+ connected profile). If that is not wanted, choose **Connected profile** on `/settings` straight
+ after the restart, before the next scheduled X sync.
+1. `/settings` → X account session: **In use** should read `Browser login (firefox) (automatic)`
+ (the live `settings.json` has `cookiesFromBrowser: "firefox"` and no exported jar). **Check**: an
+ X login visible in firefox, outside its containers, with when its `auth_token` was last used. If it
+ says none, log in to x.com in that Firefox profile (not in a container — or set
+ `cookiesFromBrowser` to `firefox::<container>`) and Check again. If it says "Seen in Firefox's
+ write-ahead log only", gallery-dl will not see the login until Firefox checkpoints: close Firefox
+ once (or wait), then Check again before step 2.
+2. One X channel's **Fetch posts**: the job log says `X login: the browser (firefox), read by
+ gallery-dl.` before gallery-dl runs.
+3. Optionally **Connect X account**: the window should be Chromium 153, with no "controlled by
+ automated test software" bar; log in with X's username and password (not Google), close the
+ window. The source stays as chosen; on Automatic it moves to the connected profile.
+
+
+#### Review
+
+**Verdict: SHIP AFTER FIXES** (`xl-review.md` in the job's scratch): one Medium, four Lows, five
+infos. The coordinator ruled M1, L1, L3 and L4 in, the I1 wording, and L2 and I2–I5 into "Found and
+left" (I2 also into the operator's steps).
+
+| Finding | Where |
+|---|---|
+| M1: with no `::CONTAINER` the reader took cookies from every container (yt-dlp's default); gallery-dl, the fetcher this login feeds, takes only cookies outside every container and accepts `::all` — so the Check could say "visible" for a login gallery-dl never reads | `b3ee652b`: `firefoxContainerScope` / `inContainerScope` mirror gallery-dl 1.32.9's `_firefox_cookies_database` and its SQL (none = no container, `::all` = no filter, `::NAME` by `name`, `l10nId`, `l10nID`, case-sensitive); the container test expects `["default"]` with no container and has `::all`, `l10nId` and a 2-vs-12 case; the Check names a login that sits only in another container |
+| L1: the Check read the WAL; gallery-dl opens the db `immutable=1`, which ignores it, so just after a login the Check could say "visible" while gallery-dl runs as a guest | `b3ee652b`: one copy, two views (with the WAL; the main file alone); a login in the WAL only adds "Seen in Firefox's write-ahead log only; gallery-dl will see it after Firefox checkpoints (closing Firefox does it)."; the operator's step 1 says what to do |
+| L2: store discovery not exactly gallery-dl's | "Found and left"; FACTS and the module comment no longer say "as gallery-dl and yt-dlp find it" |
+| L3: "visible" also counted an `auth_token` on twitter.com; gallery-dl looks on `.x.com` | `b3ee652b`: `isXComAuth` counts x.com only (the Check and the fallback's warning); a twitter.com `auth_token` is reported as "An old-domain cookie is present too … gallery-dl does not use it." |
+| L4: the refresh fell back to the recorded browser only on a throw; a bundled read that opens a newer profile without its cookies would replace a logged-in jar with an empty one | `b3ee652b`: a bundled read with no `auth_token` while the jar holds one is read again with the recorded executable; when no read finds it the jar is kept and the refresh refuses, saying so. `refreshXCookies` takes an injectable `chromium`; `xSessionBroker.test.ts` (6 cases: login found, empty bundled read → recorded, nothing found → jar kept, no record → jar kept, a jar without a login replaced, bundled throws → recorded) |
+| I1: `--test-type` verified to set no automation signal; the wording "what ChromeDriver passes" was loose, and it is not the documented route | `b3ee652b` (`xBrowser.ts`'s comment), FACTS, the decisions table: Chromium's internal test-harness switch that skips the startup bars; the documented route is the machine-wide `CommandLineFlagSecurityWarningsEnabled` policy; ChromeDriver passes `--test-type=webdriver`, the bare switch is used |
+| I2: from the restart every X fetch runs as the operator's account | "Found and left"; the operator's step 0 |
+| I3: gallery-dl is handed the whole spec (every non-container cookie loads into its process) | "Found and left" |
+| I4: a checkpoint between the db and WAL copies can pair them wrongly | "Found and left" |
+| I5: the whole store sits in a private temp dir during a read | "Found and left" |
+
+**Gates after the review** (at `b3ee652b`; logs `$T/xl-tsc3.log`, `xl-common2.log`, `xl-build2.log`,
+`xl-e2e2.log`, `xl-e2e3.log`). `git merge main` was a no-op: `main` was still `a2229d68` (RM had not
+landed).
+- tsc (all workspaces) clean, 59 s. common **2,450/2,450**, 82 s (+11: the container case rewritten,
+ the two-views and x.com-domain cases, three status cases, `xSessionBroker.test.ts`'s 6). Editor unit
+ **109/109**. `archilyzer docs files --check`, `settings example --check`, `docs env --check`: exit 0.
+- The capped editor build with the corpus linked: exit 0, 69 s, 1.65 GB peak RSS, no warnings; the link
+ removed and the worktree's own `transcripts/` put back.
+
+ | Run | At | Specs | Result |
+ |---|---|---|---|
+ | 2 | `b3ee652b` | `x-session`, `settings` (also matches `operation-settings`), `social-channel` | 18 passed, **1 failed**, 2.5 min — `social-channel` "fetch-posts writes month-sharded JSONL…" hit the 30 s test timeout. The dev server was cold (started straight after the build), and the case is slow on this host anyway: 23.8 s in run 1, its offline neighbours about 12 s each. The gallery-dl path gained only a `stat` of the session dir |
+ | 3 | `b3ee652b` | `social-channel`, `x-session` | **7 passed**, 0 failed, 1.8 min (the fetch-posts case 25.7 s) |
+
+ Not measured: the same case on `main`'s code, to say how much of its time is older than this slice.
+
+#### Re-review and brought to main (2026-10-01)
+
+**Re-review: SHIP.** `86b48d6b` merges `main` `ae23a695` (slice RM's merge) into `9275ec44`. Three
+conflicts, all records, none in code: `plans/FACTS.md` keeps RM's appended section, then XL's;
+this file keeps both slices-table rows and RM's record, with this section after it; `editor/CHANGELOG.md`
+keeps one `## [Unreleased]`: RM's bullet, then XL's. Nothing RM changed is in a file XL touched.
+
+**Gates after the merge** (at `86b48d6b`; logs `$T/xl-tsc4.log`, `xl-common3.log`, `xl-build3.log`,
+`xl-e2e4.log`): tsc (all workspaces) clean, 155 s; common **2,484/2,484**, 108 s (RM's 2,438 plus
+XL's 46); editor unit **109/109**; `docs files --check`, `settings example --check`, `docs env --check`
+exit 0; the capped editor build with the corpus linked exit 0, 49 s, 1.65 GB peak, the link removed and
+the worktree's own `transcripts/` put back; e2e `x-session` and `channel-storage` (the two slices'
+nearest specs): **15 passed**, 0 failed, 1.9 min.
+
## Rollout
Both slices are export- and homepage-side; the editor and umtool are not rebuilt for this release.
diff --git a/settings.json.example b/settings.json.example
@@ -5,6 +5,9 @@
"transcriptionApps": {},
"cookiesFromBrowser": "",
"cookieMode": "when-required",
+ "social": {
+ "x": {}
+ },
"sleepBetweenDownloadsSeconds": 10,
"downloadFormat": "auto",
"minFreeDiskGB": 5,
