// The X Article half of a post capture, against fakes only: a page that
// answers the capture's evaluate calls from recorded snapshots and the written
// HTML fixture, and a request context that serves images from a table. No
// browser, no network.
//
// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test social/xArticleCapture.test.ts

import { test } from "node:test";
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { mkdtemp, readdir, readFile, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import type { PageLike } from "./playwrightRuntime";
import {
  ARTICLE_SHOT_MAX_HEIGHT,
  captureXArticle,
  classifyXArticleSnapshot,
  fullSizeImageUrl,
  imageExtension,
  type XArticleSnapshot,
} from "./xArticleCapture";
import { captureXPosts, type XCaptureDeps, type XPostSnapshot } from "./xPostCapture";
import { readPostCapture, writePostCapture } from "./postCapture";
import type { PostCaptureInput } from "./fetchers";

const HERE = path.dirname(fileURLToPath(import.meta.url));
const FIXTURE = await readFile(path.join(HERE, "__fixtures__", "x-article.html"), "utf8");
const sha = (b: string | Uint8Array) => createHash("sha256").update(b).digest("hex");
const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 7, 7]);
const NOW = () => new Date("2026-02-03T04:05:06.000Z");
const LINK = { articleId: "999", url: "https://x.com/i/article/999" };

const COVER = "https://pbs.twimg.com/media/CoverAbc?format=jpg&name=small";
const BODY_IMG = "https://pbs.twimg.com/media/BodyImg1?format=png&name=small";
const orig = (u: string) => fullSizeImageUrl(u);

const articleSnap = (height = 2400): XArticleSnapshot => ({
  path: "/i/article/999",
  text: "A Worked Example",
  root: { rect: { x: 0, y: 53.5, width: 600.2, height }, marker: '[data-testid="twitterArticleReadView"]' },
});
const noArticle = (text: string, p = "/i/article/999"): XArticleSnapshot => ({ path: p, text, root: null });

type Served = Record<string, { status: number; body?: string; type?: string }>;

type FakeArticlePage = PageLike & { calls: string[]; shots: unknown[]; fetched: string[] };

// A page for articles: the snapshot script answers `snaps` in turn (the last
// repeats), the HTML script the fixture, the lazy-load walk nothing; images
// come from `served` (every full-size picture by default).
function articlePage(
  snaps: XArticleSnapshot[],
  opts: { html?: string; served?: Served; gotoFails?: boolean; noRequest?: boolean } = {},
): FakeArticlePage {
  let i = 0;
  const calls: string[] = [];
  const shots: unknown[] = [];
  const fetched: string[] = [];
  const served: Served = opts.served ?? {
    [orig(COVER)]: { status: 200, body: "cover bytes", type: "image/jpeg" },
    [orig(BODY_IMG)]: { status: 200, body: "chart bytes", type: "image/png" },
  };
  const p: FakeArticlePage = {
    calls,
    shots,
    fetched,
    goto: async (url: string) => {
      calls.push(`goto ${url}`);
      if (opts.gotoFails) throw new Error("net::ERR_TIMED_OUT\nat goto");
    },
    waitForSelector: async () => undefined,
    waitForTimeout: async () => undefined,
    on: () => {},
    evaluate: async (script: string) => {
      if (script.includes("outerHTML")) return opts.html ?? FIXTURE;
      if (script.includes("scrollTo")) {
        calls.push("load-lazy");
        return 3;
      }
      calls.push("snapshot");
      return snaps[Math.min(i++, snaps.length - 1)];
    },
    screenshot: async (o?: unknown) => {
      shots.push(o);
      return PNG;
    },
  };
  if (!opts.noRequest) {
    p.request = {
      get: async (url: string) => {
        fetched.push(url);
        const s = served[url] ?? { status: 404 };
        return {
          ok: () => s.status >= 200 && s.status < 300,
          status: () => s.status,
          headers: (): Record<string, string> => (s.type ? { "content-type": s.type } : {}),
          body: async () => new TextEncoder().encode(s.body ?? ""),
        };
      },
    };
  }
  return p;
}

const tmpDir = async () => path.join(await mkdtemp(path.join(os.tmpdir(), "xarticle-")), "1");

// --- classification and helpers -------------------------------------------------

test("classify: an article page with a root is captured; walls, deletions and refusals as a post's", () => {
  assert.deepEqual(classifyXArticleSnapshot(articleSnap()), { state: "captured" });
  const login = classifyXArticleSnapshot(noArticle("", "/i/flow/login"));
  assert.equal(login.state, "login-wall");
  assert.ok(login.stop);
  assert.equal(classifyXArticleSnapshot(noArticle("Log in Sign up")).state, "login-wall");
  assert.deepEqual(classifyXArticleSnapshot(noArticle("Hmm...this page doesn’t exist.")), { state: "deleted" });
  assert.deepEqual(classifyXArticleSnapshot(noArticle("This account is suspended")), { state: "unavailable" });
  const refused = classifyXArticleSnapshot(noArticle("Something went wrong. Try reloading."));
  assert.equal(refused.state, "error");
  assert.ok(refused.stop);
  assert.match(refused.error ?? "", /instead of the article/);
  assert.deepEqual(classifyXArticleSnapshot(noArticle("")), { state: "error", error: "The article did not render." });
});

test("images: the full-size picture of an X media URL, and an extension from the format, path or type", () => {
  assert.equal(fullSizeImageUrl(COVER), "https://pbs.twimg.com/media/CoverAbc?format=jpg&name=orig");
  assert.equal(fullSizeImageUrl("https://example.com/a.png?name=small"), "https://example.com/a.png?name=small");
  assert.equal(fullSizeImageUrl("not a url"), "not a url");
  assert.equal(imageExtension(COVER), "jpg");
  assert.equal(imageExtension("https://example.com/p/pic.JPEG"), "jpg");
  assert.equal(imageExtension("https://example.com/p/pic", "image/webp; q=1"), "webp");
  assert.equal(imageExtension("https://example.com/p/pic"), "img");
});

// --- one article ------------------------------------------------------------------

test("article: read, shot whole, images fetched full-size through the page; every file recorded", async () => {
  const dir = await tmpDir();
  const p = articlePage([articleSnap(), articleSnap(2500.4)]);
  const gaps: number[] = [];
  const res = await captureXArticle(p, LINK, dir, { now: NOW, imageGap: async () => void gaps.push(1) });
  assert.equal(res.stop, undefined);
  assert.equal(p.calls[0], "goto https://x.com/i/article/999");
  assert.ok(p.calls.includes("load-lazy"));
  // The box re-read after the lazy images loaded.
  assert.deepEqual(p.shots, [
    { type: "png", fullPage: true, clip: { x: 0, y: 53, width: 601, height: 2501 } },
  ]);
  assert.deepEqual(p.fetched, [orig(COVER), orig(BODY_IMG)]);
  assert.deepEqual(gaps, [1], "a gap between images, none before the first");

  const r = res.record;
  assert.equal(r.state, "captured");
  assert.equal(r.title, "A Worked Example & Its Notes");
  assert.equal(r.blocks, 12);
  assert.equal(r.extraction, "structured");
  assert.equal(r.trimmed, undefined);
  assert.equal(r.error, undefined);
  assert.deepEqual(
    r.files.map((f) => [f.name, f.url]),
    [
      ["article.json", LINK.url],
      ["article.md", LINK.url],
      ["article.html", LINK.url],
      ["article.png", LINK.url],
      ["article-img-1.jpg", orig(COVER)],
      ["article-img-2.png", orig(BODY_IMG)],
    ],
  );
  const img = r.files.find((f) => f.name === "article-img-2.png")!;
  assert.deepEqual([img.bytes, img.sha256], ["chart bytes".length, sha("chart bytes")]);
  assert.deepEqual(new Uint8Array(await readFile(path.join(dir, "article.png"))), PNG);
  assert.equal(await readFile(path.join(dir, "article.html"), "utf8"), FIXTURE);

  const json = JSON.parse(await readFile(path.join(dir, "article.json"), "utf8"));
  assert.equal(json.articleId, "999");
  assert.equal(json.url, LINK.url);
  assert.equal(json.author, "Demo Author");
  assert.equal(json.handle, "@demo_author");
  assert.equal(json.publishedAt, "2026-01-02T03:04:05.000Z");
  assert.equal(json.capturedAt, "2026-02-03T04:05:06.000Z");
  assert.equal(json.blocks.length, 12);
  assert.deepEqual(json.blocks[8], {
    type: "image",
    src: BODY_IMG,
    text: "A chart of the numbers",
    file: "article-img-2.png",
  });
  assert.deepEqual(json.blocks[9], {
    type: "embedded-post",
    text: "An embedded post's own words.",
    href: "https://x.com/other_user/status/4444444444",
  });
  const md = await readFile(path.join(dir, "article.md"), "utf8");
  assert.match(md, /^# A Worked Example & Its Notes\n/);
  assert.match(md, /!\[A chart of the numbers\]\(<article-img-2\.png>\)/);
  assert.match(md, /> Embedded post: <https:\/\/x\.com\/other_user\/status\/4444444444>/);
});

test("article: a tall one is shot to the cap and recorded trimmed", async () => {
  const dir = await tmpDir();
  const p = articlePage([articleSnap(50_000)]);
  const res = await captureXArticle(p, LINK, dir, { now: NOW });
  assert.equal(res.record.trimmed, true);
  assert.equal((p.shots[0] as { clip: { height: number } }).clip.height, ARTICLE_SHOT_MAX_HEIGHT);
  assert.equal(JSON.parse(await readFile(path.join(dir, "article.json"), "utf8")).trimmed, true);
});

test("article: a full-size picture refused falls back to the src as rendered; one not served at all is noted", async () => {
  const dir = await tmpDir();
  const p = articlePage([articleSnap()], {
    served: { [COVER]: { status: 200, body: "small cover", type: "image/jpeg" } },
  });
  const res = await captureXArticle(p, LINK, dir, { now: NOW });
  assert.deepEqual(p.fetched, [orig(COVER), COVER, orig(BODY_IMG), BODY_IMG]);
  assert.equal(res.record.state, "captured");
  assert.deepEqual(
    res.record.files.filter((f) => f.name.startsWith("article-img-")).map((f) => [f.name, f.url]),
    [["article-img-1.jpg", COVER]],
  );
  assert.match(res.record.error ?? "", /1 image\(s\) could not be fetched: HTTP 404/);
  const json = JSON.parse(await readFile(path.join(dir, "article.json"), "utf8"));
  assert.equal(json.blocks[8].file, undefined, "an image not saved links to its src");
});

test("article: a page with no request context still saves the text, and says the images are missing", async () => {
  const res = await captureXArticle(articlePage([articleSnap()], { noRequest: true }), LINK, await tmpDir(), {
    now: NOW,
  });
  assert.equal(res.record.state, "captured");
  assert.match(res.record.error ?? "", /no request context/);
  assert.equal(res.record.files.length, 4);
});

test("article: a re-capture with fewer images leaves no older numbered file behind", async () => {
  const dir = await tmpDir();
  await captureXArticle(articlePage([articleSnap()]), LINK, dir, { now: NOW });
  await writeFile(path.join(dir, "article-img-3.jpg"), "stale");
  await captureXArticle(articlePage([articleSnap()], { html: "<div data-testid=\"twitterArticleRichTextView\"><p>Just text</p></div>" }), LINK, dir, { now: NOW });
  const names = (await readdir(dir)).sort();
  assert.deepEqual(names, ["article.html", "article.json", "article.md", "article.png"]);
});

test("article: a login wall stops the run; deleted is recorded; a failed load is an error; no file is written", async () => {
  for (const [p, state, stops] of [
    [articlePage([noArticle("", "/i/flow/login")]), "login-wall", true],
    [articlePage([noArticle("Something went wrong. Try reloading.")]), "error", true],
    [articlePage([noArticle("Hmm...this page doesn’t exist.")]), "deleted", false],
    [articlePage([articleSnap()], { gotoFails: true }), "error", false],
  ] as const) {
    const dir = await tmpDir();
    const res = await captureXArticle(p, LINK, dir, { now: NOW });
    assert.equal(res.record.state, state);
    assert.equal(Boolean(res.stop), stops, state);
    assert.deepEqual(res.record.files, []);
    assert.equal(res.record.blocks, 0);
    assert.equal(p.shots.length, 0);
    assert.deepEqual(await readdir(dir).catch(() => []), []);
  }
});

// --- in the run ----------------------------------------------------------------

const post = (articleHrefs?: string[]): XPostSnapshot => ({
  path: "/someone/status/1",
  text: "the post",
  article: { rect: { x: 10, y: 120, width: 598, height: 300 }, sensitive: false, ...(articleHrefs ? { articleHrefs } : {}) },
});

// Post pages by id, article pages by article id, one routed page over both.
function runHarness(
  postsById: Record<string, XPostSnapshot>,
  articlesById: Record<string, XArticleSnapshot[]>,
): { deps: XCaptureDeps; pauses: number[]; visits: string[]; downloads: string[] } {
  const visits: string[] = [];
  const pauses: number[] = [];
  const downloads: string[] = [];
  let current: PageLike = articlePage([]);
  const articlePages = Object.fromEntries(
    Object.entries(articlesById).map(([id, snaps]) => [id, articlePage(snaps)]),
  );
  const routed: PageLike = {
    goto: async (url: string, o?: unknown) => {
      visits.push(url);
      const id = url.split("/").pop()!;
      if (url.includes("/article/")) {
        current = articlePages[id];
      } else {
        const snap = postsById[id];
        current = {
          ...articlePage([]),
          evaluate: async (s: string) => (s.includes("imgs.length") ? 0 : snap),
          screenshot: async () => PNG,
        };
      }
      return current.goto(url, o);
    },
    waitForSelector: async () => undefined,
    waitForTimeout: async () => undefined,
    on: () => {},
    evaluate: (s: string) => current.evaluate(s),
    screenshot: (o) => current.screenshot(o),
    request: { get: (url, o) => current.request!.get(url, o) },
  };
  return {
    visits,
    pauses,
    downloads,
    deps: {
      openPage: async () => ({ page: routed, close: async () => {} }),
      downloadMedia: async ({ id }) => {
        downloads.push(id);
        return { ok: true, files: [] };
      },
      pauseMs: () => 5_000,
      pause: async (ms) => void pauses.push(ms),
      articleImagePauseMs: () => 1_000,
      now: NOW,
    },
  };
}

async function input(ids: string[], over: Partial<PostCaptureInput> = {}): Promise<PostCaptureInput> {
  return {
    ids,
    handle: "example_user",
    outDir: await mkdtemp(path.join(os.tmpdir(), "posts-media-")),
    signal: new AbortController().signal,
    ...over,
  };
}

const ARCHIVED = new Map([["11", { text: "https://x.com/i/article/999" }]]);

test("run: a post whose archived text is an article link — shot, media, then the article, each paced", async () => {
  const h = runHarness({ "11": post(), "22": post() }, { "999": [articleSnap()] });
  const inp = await input(["11", "22"], { archived: ARCHIVED });
  const res = await captureXPosts(inp, h.deps);
  assert.deepEqual(h.visits, [
    "https://x.com/i/status/11",
    "https://x.com/i/article/999",
    "https://x.com/i/status/22",
  ]);
  // Five contacts (two shots, two downloads, one article): four post gaps,
  // and one short gap between the article's two images.
  assert.deepEqual(h.pauses, [5_000, 5_000, 1_000, 5_000, 5_000]);
  assert.deepEqual(
    res.outcomes.map((o) => [o.id, o.state, o.files]),
    [
      ["11", "captured", 7],
      ["22", "captured", 1],
    ],
  );
  const rec = (await readPostCapture(path.join(inp.outDir, "11")))!;
  assert.equal(rec.article?.state, "captured");
  assert.equal(rec.article?.articleId, "999");
  assert.equal(rec.article?.title, "A Worked Example & Its Notes");
  assert.equal(rec.article?.blocks, 12);
  assert.equal(rec.article?.files.length, 6);
  assert.equal((await readPostCapture(path.join(inp.outDir, "22")))!.article, undefined);

  // Captured is settled: a second run opens nothing.
  const again = runHarness({ "11": post(), "22": post() }, { "999": [articleSnap()] });
  assert.deepEqual((await captureXPosts(inp, again.deps)).outcomes, []);
  assert.deepEqual(again.visits, []);

  // force reads it again.
  const forced = runHarness({ "11": post(), "22": post() }, { "999": [articleSnap()] });
  await captureXPosts({ ...inp, ids: ["11"], force: true }, forced.deps);
  assert.deepEqual(forced.visits, ["https://x.com/i/status/11", "https://x.com/i/article/999"]);
});

test("run: an article found only on the post's card is opened too; articles: false opens none", async () => {
  const h = runHarness({ "11": post(["/demo_author/article/999"]) }, { "999": [articleSnap()] });
  const inp = await input(["11"]);
  await captureXPosts(inp, h.deps);
  assert.deepEqual(h.visits, ["https://x.com/i/status/11", "https://x.com/demo_author/article/999"]);
  assert.equal((await readPostCapture(path.join(inp.outDir, "11")))!.article?.state, "captured");

  const off = runHarness({ "11": post(["/demo_author/article/999"]) }, { "999": [articleSnap()] });
  const inp2 = await input(["11"], { archived: ARCHIVED, articles: false });
  await captureXPosts(inp2, off.deps);
  assert.deepEqual(off.visits, ["https://x.com/i/status/11"]);
  assert.equal((await readPostCapture(path.join(inp2.outDir, "11")))!.article, undefined);
});

test("run: a post captured before articles owes only its article — no second shot or download", async () => {
  const first = runHarness({ "11": post() }, { "999": [articleSnap()] });
  const inp = await input(["11"], { articles: false });
  await captureXPosts(inp, first.deps);

  const h = runHarness({ "11": post() }, { "999": [articleSnap()] });
  const res = await captureXPosts({ ...inp, articles: true, archived: ARCHIVED }, h.deps);
  assert.deepEqual(h.visits, ["https://x.com/i/article/999"]);
  assert.deepEqual(h.downloads, []);
  assert.equal(res.outcomes[0].availability, undefined, "an article-only pass says nothing about the post's liveness");
  const rec = (await readPostCapture(path.join(inp.outDir, "11")))!;
  assert.equal(rec.state, "captured");
  assert.ok(rec.shot, "the shot is kept");
  assert.equal(rec.article?.state, "captured");
});

test("run: an article behind a login wall stops the run with needsCookies; the rest untouched", async () => {
  const h = runHarness({ "11": post(), "22": post() }, { "999": [noArticle("", "/i/flow/login")] });
  const inp = await input(["11", "22"], { archived: ARCHIVED, media: false });
  const res = await captureXPosts(inp, h.deps);
  assert.equal(res.needsCookies, true);
  assert.match(res.stoppedEarly ?? "", /Stopped at 11/);
  assert.deepEqual(h.visits, ["https://x.com/i/status/11", "https://x.com/i/article/999"]);
  const rec = (await readPostCapture(path.join(inp.outDir, "11")))!;
  assert.equal(rec.state, "captured", "the post itself was shot");
  assert.equal(rec.article?.state, "login-wall");
  assert.equal(await readPostCapture(path.join(inp.outDir, "22")), null);

  // A login wall is owed again on the next run.
  const next = runHarness({ "11": post(), "22": post() }, { "999": [articleSnap()] });
  await captureXPosts({ ...inp, ids: ["11"] }, next.deps);
  assert.deepEqual(next.visits, ["https://x.com/i/article/999"]);
});

test("run: a deleted article is recorded and settled; the run goes on", async () => {
  const h = runHarness({ "11": post(), "22": post() }, { "999": [noArticle("Hmm...this page doesn’t exist.")] });
  const inp = await input(["11", "22"], { archived: ARCHIVED, media: false });
  const res = await captureXPosts(inp, h.deps);
  assert.equal(res.stoppedEarly, undefined);
  assert.equal(res.outcomes.length, 2);
  const rec = (await readPostCapture(path.join(inp.outDir, "11")))!;
  assert.equal(rec.article?.state, "deleted");
  const again = runHarness({ "11": post(), "22": post() }, { "999": [articleSnap()] });
  assert.deepEqual((await captureXPosts(inp, again.deps)).outcomes, []);
  assert.deepEqual(again.visits, []);
});

test("run: a deleted post opens no article", async () => {
  const h = runHarness(
    { "11": { path: "/i/status/11", text: "This post was deleted by the post author.", article: null } },
    { "999": [articleSnap()] },
  );
  const inp = await input(["11"], { archived: ARCHIVED });
  await captureXPosts(inp, h.deps);
  assert.deepEqual(h.visits, ["https://x.com/i/status/11"]);
});

test("run: an older record's article is not lost by a run that does not read it", async () => {
  const inp = await input(["11"], { archived: ARCHIVED });
  const dir = path.join(inp.outDir, "11");
  const article = {
    articleId: "999",
    url: LINK.url,
    capturedAt: "2026-01-01T00:00:00.000Z",
    state: "captured" as const,
    blocks: 3,
    files: [],
  };
  await writePostCapture(dir, {
    version: 1,
    id: "11",
    url: "https://x.com/i/status/11",
    capturedAt: "2026-01-01T00:00:00.000Z",
    state: "captured",
    shot: { name: "shot.png", bytes: 1, sha256: "x" },
    mediaState: "error",
    media: [],
    article,
  });
  const h = runHarness({ "11": post() }, {});
  await captureXPosts(inp, h.deps);
  assert.deepEqual(h.downloads, ["11"], "only the owed media");
  assert.deepEqual(h.visits, []);
  assert.deepEqual((await readPostCapture(dir))!.article, article);
});