// capturePosts over a registered fake fetcher: what is refused before any // capture, and how the pages' verdicts land in the availability sidecar. // // Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test controller/capturePosts.test.ts import { test } from "node:test"; import assert from "node:assert/strict"; import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; const ROOT = await mkdtemp(path.join(os.tmpdir(), "captureposts-")); process.env.TRANSCRIPTS_DIR = path.join(ROOT, "transcripts"); const { getPaths } = await import("../lib/paths"); const { registerSocialFetcher } = await import("../social/fetchers"); const { readPostAvailability, writePostAvailability, writePosts } = await import("../lib/posts-server"); const { capturePosts, NOTHING_TO_CAPTURE } = await import("./capturePosts"); type Input = import("../social/fetchers").PostCaptureInput; type Result = import("../social/fetchers").PostCaptureResult; // A capturing fetcher whose answer each test sets. let calls: Input[] = []; let answer: Result = { outcomes: [] }; registerSocialFetcher({ id: "test-capture", label: "Test capture", platform: "bluesky", fields: {}, detect: () => false, probe: async () => ({ ok: true }), fetch: async () => ({ posts: [], complete: true }), captureByIds: async (input) => { calls.push(input); return answer; }, }); registerSocialFetcher({ id: "test-capture-x", label: "Test capture (X)", platform: "twitter", fields: {}, detect: () => false, probe: async () => ({ ok: true }), fetch: async () => ({ posts: [], complete: true }), captureByIds: async (input) => { calls.push(input); return answer; }, }); registerSocialFetcher({ id: "test-no-capture", label: "No-capture fetcher", platform: "bluesky", fields: {}, detect: () => false, probe: async () => ({ ok: true }), fetch: async () => ({ posts: [], complete: true }), }); const channelsDir = path.join(ROOT, "transcripts", "channels"); async function makeChannel(slug: string, over: Record = {}): Promise { const root = path.join(channelsDir, slug); await mkdir(root, { recursive: true }); await writeFile( path.join(root, "config.json"), JSON.stringify({ handling: "transcribe", sourceKind: "social", platform: "bluesky", postFetcher: "test-capture", socialHandle: "example.bsky.social", name: "Example", url: "https://bsky.app/profile/example.bsky.social", ...over, }), ); await writeFile(path.join(root, "posts-archive"), "bluesky aaa\nbluesky bbb\nbluesky ccc\n"); return root; } const run = (slug: string, ids: string[], over: Record = {}) => capturePosts({ paths: getPaths(), slug, settings: {}, ids, ...over }); test("refused before any capture: nothing asked for, no ids, not social, no capture, ids not archived", async () => { await makeChannel("demo-social"); await makeChannel("demo-nocapture", { postFetcher: "test-no-capture" }); await mkdir(path.join(channelsDir, "demo-video"), { recursive: true }); await writeFile( path.join(channelsDir, "demo-video", "config.json"), JSON.stringify({ handling: "transcribe", name: "Video", url: "https://example.com/c" }), ); calls = []; assert.equal((await run("demo-social", ["aaa"], { shots: false, media: false })).error, NOTHING_TO_CAPTURE); assert.match((await run("demo-social", [])).error ?? "", /No post ids/); assert.equal((await run("demo-video", ["aaa"])).error, "demo-video is not a social channel."); assert.equal((await run("demo-nocapture", ["aaa"])).error, "No-capture fetcher cannot capture posts."); assert.equal( (await run("demo-social", ["aaa", "zzz", "yyy"])).error, "2 id(s) not in demo-social's posts archive: zzz, yyy", ); assert.equal(calls.length, 0); }); test("the fetcher gets the ids (deduped), the channel's posts-media dir and the halves asked for", async () => { const root = await makeChannel("demo-args"); calls = []; answer = { outcomes: [] }; const res = await run("demo-args", ["aaa", "bbb", "aaa"], { media: false, force: true }); assert.equal(res.ok, true); assert.equal(calls.length, 1); assert.deepEqual(calls[0].ids, ["aaa", "bbb"]); assert.equal(calls[0].outDir, path.join(root, "posts-media")); assert.equal(calls[0].handle, "example.bsky.social"); assert.equal(calls[0].media, false); assert.equal(calls[0].force, true); }); test("deleted and walled posts are recorded in posts-availability.json; a login wall is not", async () => { const root = await makeChannel("demo-avail"); await writePostAvailability(root, { aaa: { availability: "available", checkedAt: "2026-01-01T00:00:00.000Z" }, }); answer = { outcomes: [ { id: "aaa", state: "deleted", availability: "deleted", files: 0 }, { id: "bbb", state: "unavailable", availability: "account_unavailable", files: 0 }, { id: "ccc", state: "login-wall", files: 0 }, ], needsCookies: true, stoppedEarly: "X asked to log in. Stopped at ccc.", }; const res = await run("demo-avail", ["aaa", "bbb", "ccc"]); assert.equal(res.ok, false); assert.equal(res.needsCookies, true); assert.match(res.error ?? "", /Stopped at ccc/); const map = await readPostAvailability(root); assert.equal(map.aaa.availability, "deleted"); // The change keeps the moment it was last seen up. assert.deepEqual(map.aaa.history, [{ availability: "available", at: "2026-01-01T00:00:00.000Z" }]); assert.equal(map.bbb.availability, "account_unavailable"); assert.equal(map.ccc, undefined); }); test("a run stopped by the source fails the job; one cancelled by the operator does not", async () => { await makeChannel("demo-stop"); answer = { outcomes: [], stoppedEarly: "X is refusing pages right now." }; const stopped = await run("demo-stop", ["aaa"]); assert.equal(stopped.ok, false); assert.match(stopped.error ?? "", /refusing/); const ac = new AbortController(); ac.abort(); answer = { outcomes: [], stoppedEarly: "Cancelled; the rest are left for a later run." }; const cancelled = await run("demo-stop", ["aaa"], { signal: ac.signal }); assert.equal(cancelled.ok, true); }); test("the articles: on by default, refused alone only when not asked for by name, archived text passed (X)", async () => { const root = path.join(channelsDir, "demo-x"); await mkdir(root, { recursive: true }); await writeFile( path.join(root, "config.json"), JSON.stringify({ handling: "transcribe", sourceKind: "social", platform: "twitter", postFetcher: "test-capture-x", socialHandle: "example_user", name: "Example (X)", url: "https://x.com/example_user", }), ); const post = (id: string, text: string, links: string[] = []) => ({ id, slug: `demo-x/${id}`, channelSlug: "demo-x", author: "example_user", createdAt: "2026-01-02T03:04:05.000Z", uploadDate: "20260102", text, url: `https://x.com/example_user/status/${id}`, platform: "twitter" as const, isReply: false, isRepost: false, links, }); await writePosts(root, [ post("111", "https://x.com/i/article/999"), post("222", "a plain post", ["https://example.com/page"]), post("333", "not asked for"), ]); answer = { outcomes: [] }; calls = []; assert.equal((await run("demo-x", ["111", "222"])).ok, true); assert.equal(calls[0].articles, undefined); const archived = calls[0].archived ?? new Map(); assert.deepEqual([...archived.keys()].sort(), ["111", "222"]); assert.deepEqual(archived.get("111"), { text: "https://x.com/i/article/999", links: [] }); assert.deepEqual(archived.get("222"), { text: "a plain post", links: ["https://example.com/page"] }); calls = []; assert.equal((await run("demo-x", ["111"], { articles: false })).ok, true); assert.equal(calls[0].articles, false); assert.equal(calls[0].archived, undefined); // Both halves off: refused, unless the articles are asked for by name. calls = []; assert.equal((await run("demo-x", ["111"], { shots: false, media: false })).error, NOTHING_TO_CAPTURE); assert.equal( (await run("demo-x", ["111"], { shots: false, media: false, articles: false })).error, NOTHING_TO_CAPTURE, ); assert.equal(calls.length, 0); assert.equal((await run("demo-x", ["111"], { shots: false, media: false, articles: true })).ok, true); assert.equal(calls[0].articles, true); });