// The forum-thread walk, capture and page loader over fakes: no browser, no
// network. SYNTHETIC pages (__fixtures__/xenforoPages.ts).
//
// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test social/xenforoFetcher.test.ts

import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtemp, readFile, readdir } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import type { Post } from "../lib/posts";
import type { PostFetchInput } from "./fetchers";
import { loadForumPage, presentableUserAgent, type ForumPageLoader } from "./forumSession";
import type { PageLike } from "./playwrightRuntime";
import {
  captureForumPosts,
  DEFAULT_PAGE_PAUSE_MS,
  downloadableMedia,
  forumBlockFailure,
  jitteredPause,
  LAST_PAGE_PROBE,
  MIN_PAGE_PAUSE_MS,
  walkXenforoThread,
  xenforoFetcher,
} from "./xenforoFetcher";
import {
  CHALLENGE_PAGE,
  CAPTCHA_PAGE,
  NOT_FOUND_PAGE,
  ORIGIN,
  THREAD_URL,
  threadPage,
  type FakePost,
} from "./__fixtures__/xenforoPages";

const T0 = 1_760_000_000;
const LAST = 5;
const PER_PAGE = 3;

// Post k (1-based position) of the thread, on page ceil(k / 3).
function fakePost(position: number): FakePost {
  return {
    id: 9000 + position,
    author: `Member${position % 4}`,
    userId: 50 + (position % 4),
    ts: T0 + position * 600,
    position,
    body: `Words of post ${position}.`,
  };
}

function pageHtml(page: number): string {
  const posts: FakePost[] = [];
  for (let k = (page - 1) * PER_PAGE + 1; k <= page * PER_PAGE; k++) posts.push(fakePost(k));
  return threadPage({ page, last: LAST, posts });
}

const pageOf = (url: string): number => {
  const m = /page-(\d+)$/.exec(url);
  if (!m) return 1;
  return Math.min(Number(m[1]), LAST); // past the end → the last page
};

type FakeLoader = ForumPageLoader & { urls: string[]; closed: number };

function fakeLoader(
  override: (url: string, page: number) => { html: string; status?: number } | undefined = () => undefined,
): FakeLoader {
  const loader: FakeLoader = {
    urls: [],
    closed: 0,
    async load(url) {
      loader.urls.push(url);
      const page = pageOf(url);
      const o = override(url, page);
      return {
        html: o?.html ?? pageHtml(page),
        ...(o?.status ? { status: o.status } : { status: 200 }),
        url: `${THREAD_URL}${page > 1 ? `page-${page}` : ""}`,
      };
    },
    async close() {
      loader.closed++;
    },
  };
  return loader;
}

function input(over: Partial<PostFetchInput> = {}): PostFetchInput {
  return {
    accountUrl: THREAD_URL,
    handle: "the-teapot-collectors-thread.4242",
    channelSlug: "teapots",
    seenIds: new Set(),
    signal: new AbortController().signal,
    ...over,
  };
}

function deps(loader: ForumPageLoader) {
  const sleeps: number[] = [];
  return {
    sleeps,
    deps: {
      loader,
      pauseMs: () => 11_000,
      sleep: async (ms: number) => {
        sleeps.push(ms);
      },
    },
  };
}

const ids = (posts: Post[]) => posts.map((p) => Number(p.id) - 9000);

test("first run: starts at the last page, walks newest first to page 1, paced between loads", async () => {
  const loader = fakeLoader();
  const { deps: d, sleeps } = deps(loader);
  const log: string[] = [];
  const res = await walkXenforoThread(input({ onLog: (l) => log.push(l) }), d);
  assert.equal(res.complete, true);
  assert.equal(res.cursor, undefined);
  assert.equal(loader.urls[0], `${THREAD_URL}page-${LAST_PAGE_PROBE}`);
  assert.deepEqual(loader.urls.slice(1), [4, 3, 2].map((n) => `${THREAD_URL}page-${n}`).concat([THREAD_URL]));
  // Every page's posts, each page in thread order, newest page first.
  assert.deepEqual(ids(res.posts), [13, 14, 15, 10, 11, 12, 7, 8, 9, 4, 5, 6, 1, 2, 3]);
  // One pause before every load but the first, none in parallel.
  assert.deepEqual(sleeps, [11_000, 11_000, 11_000, 11_000]);
  assert.equal(loader.closed, 1);
  assert.match(log[0], /^Page 5\/5: 3 post\(s\) parsed, 3 new\.$/);
});

test("a pages cap reads the latest N pages and leaves the cursor at the next one", async () => {
  const loader = fakeLoader();
  const res = await walkXenforoThread(input({ pages: 2 }), deps(loader).deps);
  assert.equal(res.complete, false);
  assert.equal(res.cursor, "3");
  assert.deepEqual(ids(res.posts), [13, 14, 15, 10, 11, 12]);
  assert.equal(loader.urls.length, 2);
});

test("resume: a run with a cursor catches up from the last page, then continues the older pages there", async () => {
  const loader = fakeLoader();
  // An earlier run read pages 5 and 4 and stopped (cursor 3). Post 9 (page 3)
  // shifted in from a page already read: archived, and it does not end the
  // backfill.
  const seen = new Set(["9009", "9010", "9011", "9012", "9013"]);
  const res = await walkXenforoThread(input({ cursor: "3", seenIds: seen }), deps(loader).deps);
  assert.equal(res.complete, true);
  assert.deepEqual(loader.urls, [
    `${THREAD_URL}page-${LAST_PAGE_PROBE}`,
    `${THREAD_URL}page-3`,
    `${THREAD_URL}page-2`,
    THREAD_URL,
  ]);
  // The newest first (posts 14, 15 arrived since), then the older pages.
  assert.deepEqual(ids(res.posts), [14, 15, 7, 8, 4, 5, 6, 1, 2, 3]);
});

test("resume with a pages cap: the latest page is always read first, the cap spans both", async () => {
  const seen = new Set(["9007", "9008", "9009", "9010", "9011", "9012", "9013", "9014", "9015"]);
  const one = fakeLoader();
  const r1 = await walkXenforoThread(input({ cursor: "2", seenIds: seen, pages: 1 }), deps(one).deps);
  assert.deepEqual(one.urls, [`${THREAD_URL}page-${LAST_PAGE_PROBE}`]);
  assert.equal(r1.cursor, "2", "nothing new, and the older pages still owed");
  const two = fakeLoader();
  const r2 = await walkXenforoThread(input({ cursor: "2", seenIds: seen, pages: 2 }), deps(two).deps);
  assert.deepEqual(two.urls, [`${THREAD_URL}page-${LAST_PAGE_PROBE}`, `${THREAD_URL}page-2`]);
  assert.equal(r2.cursor, "1");
  assert.deepEqual(ids(r2.posts), [4, 5, 6]);
});

test("incremental: stops on the page holding an already-archived post, keeping its new ones", async () => {
  const loader = fakeLoader();
  const seen = new Set(["9001", "9002", "9003", "9004", "9005", "9006", "9007", "9008", "9009", "9010"]);
  const res = await walkXenforoThread(input({ seenIds: seen }), deps(loader).deps);
  assert.equal(res.complete, true);
  assert.deepEqual(ids(res.posts), [13, 14, 15, 11, 12]);
  assert.equal(loader.urls.length, 2);
});

test("a full re-walk (stopAtKnown false) does not stop at archived posts", async () => {
  const loader = fakeLoader();
  const res = await walkXenforoThread(
    input({ seenIds: new Set(["9013", "9010"]), stopAtKnown: false }),
    deps(loader).deps,
  );
  assert.equal(res.complete, true);
  assert.equal(loader.urls.length, 5);
  assert.equal(res.posts.length, 13);
});

test("the watermark stops the walk", async () => {
  const since = new Date((T0 + 10 * 600) * 1000).toISOString(); // post 10's time
  const res = await walkXenforoThread(input({ since }), deps(fakeLoader()).deps);
  assert.equal(res.complete, true);
  assert.deepEqual(ids(res.posts), [13, 14, 15, 11, 12]);
});

test("a post limit is checked after a whole page", async () => {
  const res = await walkXenforoThread(input({ limit: 4 }), deps(fakeLoader()).deps);
  assert.equal(res.complete, false);
  assert.equal(res.cursor, "3");
  assert.equal(res.posts.length, 6);
});

test("checkpoints: each page's posts are saved with the next page as the resume point", async () => {
  const saved: { posts: number[]; cursor: string }[] = [];
  const res = await walkXenforoThread(
    input({
      pages: 3,
      onCheckpoint: async (cp) => {
        saved.push({ posts: ids(cp.posts), cursor: cp.cursor });
      },
    }),
    deps(fakeLoader()).deps,
  );
  assert.deepEqual(saved, [
    { posts: [13, 14, 15], cursor: "4" },
    { posts: [10, 11, 12], cursor: "3" },
    { posts: [7, 8, 9], cursor: "2" },
  ]);
  assert.equal(res.posts.length, 0, "checkpointed posts are not handed back again");
  assert.equal(res.cursor, "2");
});

test("drain stops between pages, keeping the next page; cancel stops too", async () => {
  const drain = new AbortController();
  const loader = fakeLoader((_url, page) => {
    if (page === 4) drain.abort();
    return undefined;
  });
  const res = await walkXenforoThread(input({ drain: drain.signal }), deps(loader).deps);
  assert.equal(res.drained, true);
  assert.equal(res.complete, false);
  assert.equal(res.cursor, "3");
  assert.deepEqual(ids(res.posts), [13, 14, 15, 10, 11, 12]);

  const cancel = new AbortController();
  const loader2 = fakeLoader((_url, page) => {
    if (page === 5) cancel.abort();
    return undefined;
  });
  const res2 = await walkXenforoThread(input({ signal: cancel.signal }), deps(loader2).deps);
  assert.equal(res2.complete, false);
  assert.equal(loader2.urls.length, 1);
});

test("a browser check that will not clear stops the run, typed, with the cursor kept and Connect named", async () => {
  const loader = fakeLoader((_url, page) => (page === 4 ? { html: CHALLENGE_PAGE } : undefined));
  const res = await walkXenforoThread(input(), deps(loader).deps);
  assert.equal(res.complete, false);
  assert.equal(res.needsCookies, true);
  assert.equal(res.cursor, "4");
  assert.match(res.error ?? "", /browser check/);
  assert.match(res.error ?? "", /Connect forum session/);
  assert.deepEqual(ids(res.posts), [13, 14, 15], "what was read before the check is kept");
  assert.equal(loader.urls.length, 2, "never retried");
});

test("a captcha on the very first load keeps no cursor; a 429 is not a session problem", async () => {
  const captcha = await walkXenforoThread(
    input(),
    deps(fakeLoader(() => ({ html: CAPTCHA_PAGE }))).deps,
  );
  assert.equal(captcha.needsCookies, true);
  assert.equal(captcha.cursor, undefined);
  assert.match(captcha.error ?? "", /captcha/);

  const limited = await walkXenforoThread(
    input({ cursor: "2" }),
    deps(fakeLoader(() => ({ html: "<html><body>slow down</body></html>", status: 429 }))).deps,
  );
  // Refused on the catch-up load: the stored cursor is kept, not lost.
  assert.equal(limited.needsCookies, undefined);
  assert.equal(limited.cursor, "2");
  assert.match(limited.error ?? "", /HTTP 429/);
});

test("an article thread: its first post atop every page never stops the walk, and is taken once", async () => {
  const article = fakePost(1);
  const withArticle = (page: number) => {
    const posts: FakePost[] = [];
    for (let k = (page - 1) * PER_PAGE + 1; k <= page * PER_PAGE; k++) if (k > 1) posts.push(fakePost(k));
    return { html: threadPage({ page, last: LAST, posts, article }) };
  };
  // Incremental: the starter is archived, and old — neither ends the walk on
  // the last page; the archived post 10 on page 4 does.
  const seen = new Set(["9001", "9010"]);
  const since = new Date((T0 + 10 * 600) * 1000).toISOString();
  const res = await walkXenforoThread(
    input({ seenIds: seen, since }),
    deps(fakeLoader((_url, page) => withArticle(page))).deps,
  );
  assert.equal(res.complete, true);
  assert.deepEqual(ids(res.posts), [13, 14, 15, 11, 12]);
  // A first run takes the starter once, not once per page.
  const all = await walkXenforoThread(input(), deps(fakeLoader((_url, page) => withArticle(page))).deps);
  assert.equal(all.posts.filter((p) => p.id === "9001").length, 1);
  assert.equal(all.posts.length, 15);
});

test("a page that parses to no posts stops rather than stepping past it", async () => {
  const empty = threadPage({ page: 4, last: LAST, posts: [] });
  const res = await walkXenforoThread(
    input(),
    deps(fakeLoader((_url, page) => (page === 4 ? { html: empty } : undefined))).deps,
  );
  assert.equal(res.complete, false);
  assert.equal(res.cursor, "4");
  assert.match(res.error ?? "", /no posts/);
});

test("a bad cursor and a URL that is not a thread are refused before any load", async () => {
  const loader = fakeLoader();
  assert.match((await walkXenforoThread(input({ cursor: "x" }), deps(loader).deps)).error ?? "", /not a page number/);
  assert.match(
    (await walkXenforoThread(input({ accountUrl: "https://forum.example/members/a.1/" }), deps(loader).deps)).error ?? "",
    /not a XenForo thread URL/,
  );
  assert.equal(loader.urls.length, 0);
});

test("pacing: jittered around the base, never below the floor", () => {
  assert.equal(jitteredPause(DEFAULT_PAGE_PAUSE_MS, () => 0), Math.round(DEFAULT_PAGE_PAUSE_MS * 0.85));
  assert.ok(jitteredPause(DEFAULT_PAGE_PAUSE_MS, () => 0.999) <= 20_000);
  assert.ok(jitteredPause(DEFAULT_PAGE_PAUSE_MS, () => 0) >= 10_000);
  assert.equal(jitteredPause(100, () => 0), Math.round(MIN_PAGE_PAUSE_MS * 0.85));
});

test("block failures: which ones ask for Connect", () => {
  assert.equal(forumBlockFailure({ kind: "challenge", detail: "x" }, "forum.example").needsCookies, true);
  assert.equal(forumBlockFailure({ kind: "login", detail: "x" }, "forum.example").needsCookies, true);
  assert.equal(forumBlockFailure({ kind: "blocked", detail: "HTTP 403" }, "forum.example").needsCookies, false);
  assert.equal(forumBlockFailure({ kind: "not-found", detail: "x" }, "forum.example").needsCookies, false);
});

test("the fetcher claims thread URLs only, and probes offline", async () => {
  assert.equal(xenforoFetcher.detect(THREAD_URL), true);
  assert.equal(xenforoFetcher.detect("https://x.com/someone"), false);
  assert.equal(xenforoFetcher.detect("https://forum.example/members/a.1/"), false);
  const probe = await xenforoFetcher.probe(`${THREAD_URL}page-3`);
  assert.deepEqual(probe, {
    ok: true,
    name: "The Teapot Collectors Thread",
    handle: "the-teapot-collectors-thread.4242",
    url: THREAD_URL,
  });
});

// --- the page loader ---------------------------------------------------------------

function fakePage(script: { status?: number; htmls: string[]; href?: string }) {
  let clock = 0;
  let reads = 0;
  const page = {
    waits: 0,
    async goto() {
      return { status: () => script.status ?? 200 };
    },
    async waitForSelector() {},
    async waitForTimeout(ms: number) {
      page.waits++;
      clock += ms;
    },
    async evaluate(fn: string) {
      if (fn === "location.href") return script.href ?? THREAD_URL;
      const html = script.htmls[Math.min(reads, script.htmls.length - 1)];
      reads++;
      return html;
    },
    on() {},
    async screenshot() {
      return new Uint8Array();
    },
  };
  return { page: page as unknown as PageLike & { waits: number }, now: () => clock };
}

test("loader: a browser check is waited out and the thread that follows is returned", async () => {
  const thread = pageHtml(2);
  const { page, now } = fakePage({ status: 403, htmls: [CHALLENGE_PAGE, CHALLENGE_PAGE, thread] });
  const log: string[] = [];
  const res = await loadForumPage(page, `${THREAD_URL}page-2`, {
    signal: new AbortController().signal,
    onLog: (l) => log.push(l),
    now,
  });
  assert.equal(res.html, thread);
  assert.equal(res.status, undefined, "the check's 403 does not describe the page that followed");
  assert.equal(res.challengeMs, 4_000);
  assert.match(log.join("\n"), /cleared in 4 s/);
});

test("loader: a check still up at the timeout is handed back for the walk to judge", async () => {
  const { page, now } = fakePage({ status: 403, htmls: [CHALLENGE_PAGE] });
  const res = await loadForumPage(page, THREAD_URL, {
    signal: new AbortController().signal,
    timeoutMs: 10_000,
    now,
  });
  assert.equal(res.html, CHALLENGE_PAGE);
  assert.ok((res.challengeMs ?? 0) >= 10_000);
});

test("loader: a plain page comes straight back with its status", async () => {
  const { page, now } = fakePage({ status: 200, htmls: [pageHtml(1)] });
  const res = await loadForumPage(page, THREAD_URL, { signal: new AbortController().signal, now });
  assert.equal(res.status, 200);
  assert.equal(res.challengeMs, undefined);
  assert.equal(page.waits, 0);
});

test("the headless agent presents as the browser it is", () => {
  assert.equal(
    presentableUserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/147.0.0.0 Safari/537.36"),
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36",
  );
});

// --- capture -------------------------------------------------------------------------

function captureLoader(pages: Record<string, string>, rect: { x: number; y: number; width: number; height: number } | null) {
  const urls: string[] = [];
  const got: string[] = [];
  const page = {
    async goto() {
      return null;
    },
    async waitForSelector() {},
    async waitForTimeout() {},
    async evaluate(fn: string) {
      if (fn.includes("getBoundingClientRect")) return { rect };
      return 0;
    },
    on() {},
    async screenshot() {
      return new Uint8Array([137, 80, 78, 71]);
    },
    request: {
      async get(url: string) {
        got.push(url);
        return {
          ok: () => !url.includes("missing"),
          status: () => (url.includes("missing") ? 404 : 200),
          headers: () => ({ "content-type": "image/jpeg" }),
          body: async () => new Uint8Array([1, 2, 3]),
        };
      },
    },
  } as unknown as PageLike;
  return {
    urls,
    got,
    loader: {
      async page() {
        return page;
      },
      async load(url: string) {
        urls.push(url);
        return { html: pages[url] ?? pageHtml(1), status: 200, url };
      },
      async close() {},
    },
  };
}

test("capture: shoots the post's article and downloads its file media through the profile", async () => {
  const outDir = await mkdtemp(path.join(os.tmpdir(), "forum-capture-"));
  const { loader, urls, got } = captureLoader({}, { x: 10, y: 400, width: 700, height: 300 });
  const sleeps: number[] = [];
  const res = await captureForumPosts(
    {
      ids: ["9001", "9002"],
      handle: "t",
      accountUrl: THREAD_URL,
      outDir,
      archived: new Map([
        [
          "9001",
          {
            text: "x",
            url: `${ORIGIN}/posts/9001/`,
            media: [
              { kind: "image", url: "https://images.example/a.jpg" },
              { kind: "embed", url: "https://www.youtube.com/watch?v=AbCdEfGhIjK" },
              { kind: "attachment", url: `${ORIGIN}/attachments/scan-jpg.900/` },
            ],
          },
        ],
      ]),
      signal: new AbortController().signal,
    },
    { loader, pauseMs: () => 7_000, sleep: async (ms) => void sleeps.push(ms), now: () => new Date(0) },
  );
  assert.deepEqual(res.outcomes.map((o) => [o.id, o.state, o.files]), [
    ["9001", "captured", 3],
    ["9002", "captured", 1],
  ]);
  assert.deepEqual(urls, [`${ORIGIN}/posts/9001/`, `${ORIGIN}/posts/9002/`]);
  assert.deepEqual(got, ["https://images.example/a.jpg", `${ORIGIN}/attachments/scan-jpg.900/`], "embeds are pages, not files");
  // Every contact after the first is paced: 2 downloads + the second shot.
  assert.equal(sleeps.length, 3);
  assert.deepEqual((await readdir(path.join(outDir, "9001"))).sort(), ["capture.json", "media-1.jpg", "media-2.jpg", "shot.png"]);
  const rec = JSON.parse(await readFile(path.join(outDir, "9001", "capture.json"), "utf8"));
  assert.equal(rec.state, "captured");
  assert.equal(rec.mediaState, "ok");
  assert.equal(rec.url, `${ORIGIN}/posts/9001/`);
  const rec2 = JSON.parse(await readFile(path.join(outDir, "9002", "capture.json"), "utf8"));
  assert.equal(rec2.mediaState, "none");
});

test("capture: a missing post is deleted; a check that will not clear stops the run for Connect", async () => {
  const outDir = await mkdtemp(path.join(os.tmpdir(), "forum-capture-"));
  const gone = captureLoader({ [`${ORIGIN}/posts/1/`]: NOT_FOUND_PAGE }, null);
  const res = await captureForumPosts(
    { ids: ["1"], handle: "t", accountUrl: THREAD_URL, outDir, signal: new AbortController().signal, media: false },
    { loader: gone.loader, pauseMs: () => 0, sleep: async () => {} },
  );
  assert.equal(res.outcomes[0].state, "deleted");
  assert.equal(res.outcomes[0].availability, "deleted");

  const walled = captureLoader({ [`${ORIGIN}/posts/2/`]: CHALLENGE_PAGE }, null);
  const res2 = await captureForumPosts(
    { ids: ["2", "3"], handle: "t", accountUrl: THREAD_URL, outDir, signal: new AbortController().signal },
    { loader: walled.loader, pauseMs: () => 0, sleep: async () => {} },
  );
  assert.equal(res2.needsCookies, true);
  assert.match(res2.stoppedEarly ?? "", /Connect forum session/);
  assert.equal(res2.outcomes.length, 1, "the rest are left for a later run");
});

test("downloadable media: files only", () => {
  assert.deepEqual(
    downloadableMedia([
      { kind: "image", url: "https://a.example/1.png" },
      { kind: "link-card", url: "https://a.example/page" },
      { kind: "video", url: "./Saved_files/v.mp4" },
      { kind: "video", url: "https://a.example/v.mp4" },
    ]),
    [
      { kind: "image", url: "https://a.example/1.png" },
      { kind: "video", url: "https://a.example/v.mp4" },
    ],
  );
});

test("capture: downloads what the post's page holds NOW, beside what the archive kept", async () => {
  const outDir = await mkdtemp(path.join(os.tmpdir(), "forum-capture-live-"));
  const video = "https://uploads.kiwifarms.st/data/video/9665/9665533-abc.mp4?hash=Zz";
  const player =
    `<div class="ephyra-player ephyra-player--video" data-media-type="video" data-duration-label="1:11"` +
    ` data-source-fallback="${video.replace("https:", "")}" data-filename="clip.mp4"></div>`;
  const postUrl = `${ORIGIN}/posts/9100/`;
  const html = threadPage({
    page: 1,
    last: 1,
    posts: [{ id: 9100, author: "Member1", userId: 51, ts: T0, position: 1, body: `for those that want proof:${player}` }],
  });
  const { loader, got } = captureLoader({ [postUrl]: html }, { x: 0, y: 0, width: 600, height: 200 });
  const res = await captureForumPosts(
    {
      ids: ["9100"],
      handle: "t",
      accountUrl: THREAD_URL,
      outDir,
      // Archived by the parser of its day: the video was a bare "1:11", no media.
      archived: new Map([["9100", { text: "for those that want proof:\n1:11", url: postUrl, media: [{ kind: "image", url: "https://images.example/b.jpg" }] }]]),
      signal: new AbortController().signal,
    },
    { loader, pauseMs: () => 7_000, sleep: async () => {}, now: () => new Date(0) },
  );
  assert.deepEqual(res.outcomes.map((o) => [o.id, o.state, o.files]), [["9100", "captured", 3]]);
  assert.deepEqual(got, [video, "https://images.example/b.jpg"], "the page's video first, then the archive's image");
  const rec = JSON.parse(await readFile(path.join(outDir, "9100", "capture.json"), "utf8"));
  assert.equal(rec.mediaState, "ok");
  assert.deepEqual(rec.media.map((m: { url: string }) => m.url), [video, "https://images.example/b.jpg"]);
});