// The post capture's on-disk record (capture.json), what a capture still owes // a post, and the guard that keeps `posts-media/` out of every export. // // Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test social/postCapture.test.ts import { test } from "node:test"; import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import { mkdir, mkdtemp, readdir, readFile, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { articleImageFilename, CAPTURE_FILENAME, captureAvailability, captureWork, describeCapturedFile, isArticleFile, listCapturedMediaFiles, POSTS_MEDIA_DIRNAME, postCaptureDir, postsMediaDir, readPostCapture, SHOT_FILENAME, writePostCapture, type ArticleCaptureRecord, type PostCaptureRecord, } from "./postCapture"; const sha = (b: string | Buffer) => createHash("sha256").update(b).digest("hex"); function record(over: Partial = {}): PostCaptureRecord { return { version: 1, id: "123", url: "https://x.com/i/status/123", capturedAt: "2026-01-02T03:04:05.000Z", state: "captured", mediaState: "ok", media: [], ...over, }; } test("layout: posts-media// beside the channel's data, and an id is never a path", () => { assert.equal(postsMediaDir("/c/channels/demo-channel"), "/c/channels/demo-channel/posts-media"); assert.equal(POSTS_MEDIA_DIRNAME, "posts-media"); assert.equal(postCaptureDir("/o", "1234"), "/o/1234"); assert.equal(postCaptureDir("/o", "3kabc_d-e"), "/o/3kabc_d-e"); for (const id of ["../x", "a/b", "", ".", "..", "a b"]) { assert.throws(() => postCaptureDir("/o", id), /is not a post id/, JSON.stringify(id)); } }); test("capture.json: each file's sha256 and byte size, the URLs, round-tripped", async () => { const dir = path.join(await mkdtemp(path.join(os.tmpdir(), "capture-")), "123"); await mkdir(dir, { recursive: true }); const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 1, 2, 3]); await writeFile(path.join(dir, SHOT_FILENAME), png); await writeFile(path.join(dir, "123_1.jpg"), "jpeg bytes"); const shot = await describeCapturedFile(dir, SHOT_FILENAME, "https://x.com/i/status/123"); const media = await describeCapturedFile(dir, "123_1.jpg", "https://pbs.example/m.jpg"); assert.deepEqual(shot, { name: SHOT_FILENAME, bytes: png.length, sha256: sha(png), url: "https://x.com/i/status/123", }); assert.equal(media.sha256, sha("jpeg bytes")); assert.equal(media.bytes, "jpeg bytes".length); // No URL known: the key is absent, never "undefined". assert.ok(!("url" in (await describeCapturedFile(dir, "123_1.jpg")))); const rec = record({ shot, media: [media] }); await writePostCapture(dir, rec); assert.deepEqual(JSON.parse(await readFile(path.join(dir, CAPTURE_FILENAME), "utf8")), rec); assert.deepEqual(await readPostCapture(dir), rec); // The record and the shot are not media. assert.deepEqual(await listCapturedMediaFiles(dir), ["123_1.jpg"]); // Nor is the article half, whatever a later gallery-dl run finds beside it. for (const name of ["article.json", "article.md", "article.png", "article.html", "article-img-1.jpg"]) { await writeFile(path.join(dir, name), "x"); } assert.deepEqual(await listCapturedMediaFiles(dir), ["123_1.jpg"]); }); test("capture.json: absent, unparseable or of another shape reads as no capture", async () => { const dir = await mkdtemp(path.join(os.tmpdir(), "capture-")); assert.equal(await readPostCapture(dir), null); await writeFile(path.join(dir, CAPTURE_FILENAME), "{ truncated"); assert.equal(await readPostCapture(dir), null); await writeFile(path.join(dir, CAPTURE_FILENAME), JSON.stringify({ version: 2, id: "1" })); assert.equal(await readPostCapture(dir), null); assert.deepEqual(await listCapturedMediaFiles(path.join(dir, "missing")), []); }); test("availability: only what the page said about the post — never a verdict from a login wall or an error", () => { assert.equal(captureAvailability("captured"), "available"); assert.equal(captureAvailability("deleted"), "deleted"); assert.equal(captureAvailability("unavailable"), "account_unavailable"); assert.equal(captureAvailability("login-wall"), undefined); assert.equal(captureAvailability("error"), undefined); }); test("what a capture owes: nothing for a settled post, only the missing half otherwise, everything when forced", () => { const all = { shots: true, media: true, force: false }; assert.deepEqual(captureWork(null, all), { shot: true, media: true, article: false }); assert.deepEqual(captureWork(null, { ...all, media: false }), { shot: true, media: false, article: false }); const done = record({ shot: { name: SHOT_FILENAME, bytes: 1, sha256: "x" }, mediaState: "ok" }); assert.deepEqual(captureWork(done, all), { shot: false, media: false, article: false }); assert.deepEqual(captureWork(record({ ...done, mediaState: "none" }), all), { shot: false, media: false, article: false }); // The media failed (or was never asked for): only the media is owed. assert.deepEqual(captureWork(record({ ...done, mediaState: "error" }), all), { shot: false, media: true, article: false }); assert.deepEqual(captureWork(record({ ...done, mediaState: "skipped" }), all), { shot: false, media: true, article: false }); // A shot that failed is owed again. assert.deepEqual(captureWork(record({ state: "error", mediaState: "ok" }), all), { shot: true, media: false, article: false }); // Deleted is settled: asking again is a request for the same answer. assert.deepEqual(captureWork(record({ state: "deleted", mediaState: "skipped" }), all), { shot: false, media: false, article: false, }); // force re-takes whatever was asked for. assert.deepEqual(captureWork(done, { ...all, force: true }), { shot: true, media: true, article: false }); assert.deepEqual(captureWork(record({ state: "deleted" }), { shots: false, media: true, force: true }), { shot: false, media: true, article: false, }); }); test("what a capture owes the article half: only when asked, settled once captured, deleted or walled", () => { const all = { shots: true, media: true, force: false, articles: true }; const done = record({ shot: { name: SHOT_FILENAME, bytes: 1, sha256: "x" }, mediaState: "ok" }); const article = (state: ArticleCaptureRecord["state"]): ArticleCaptureRecord => ({ articleId: "9", url: "https://x.com/i/article/9", capturedAt: "2026-01-02T03:04:05.000Z", state, blocks: 0, files: [], }); assert.equal(captureWork(null, all).article, true); assert.equal(captureWork(null, { ...all, articles: false }).article, false); // Not asked for by name: the older callers' shape owes no article. assert.equal(captureWork(null, { shots: true, media: true, force: false }).article, false); // A post shot before articles were captured owes only its article. assert.deepEqual(captureWork(done, all), { shot: false, media: false, article: true }); for (const state of ["captured", "deleted", "unavailable"] as const) { assert.equal(captureWork(record({ ...done, article: article(state) }), all).article, false, state); } for (const state of ["error", "login-wall"] as const) { assert.equal(captureWork(record({ ...done, article: article(state) }), all).article, true, state); } // force re-reads a captured article; a deleted post owes nothing. assert.equal(captureWork(record({ ...done, article: article("captured") }), { ...all, force: true }).article, true); assert.equal(captureWork(record({ state: "deleted" }), all).article, false); }); test("article files are named by the layout, images numbered from 1", () => { assert.equal(articleImageFilename(1, "jpg"), "article-img-1.jpg"); assert.ok(isArticleFile("article.md")); assert.ok(isArticleFile("article-img-12.png")); assert.ok(!isArticleFile("123_1.jpg")); assert.ok(!isArticleFile("articles.txt")); }); // THE EXPORT NEVER PUBLISHES A CAPTURE — but the one a report cites. The // export serves the index's JSON pages and copies trees out of export/public — // it never reads a channel directory — so nothing it builds can carry // posts-media/. Held here the cheap way: no module of the export build, the // publish layer or the export app names the directory, except EXACTLY the one // that copies a report site's CITED captures (publish/citedPostCaptures.ts, // whose own test proves it copies nothing else). const CITED_CAPTURE_COPIER = path.join("common", "publish", "citedPostCaptures.ts"); test("no export or publish source names posts-media/ but the cited-capture copier", async () => { const HERE = path.dirname(fileURLToPath(import.meta.url)); const repo = path.resolve(HERE, "..", ".."); const roots = [ path.join(repo, "common", "publish"), path.join(repo, "common", "bin"), path.join(repo, "export", "app"), path.join(repo, "export", "lib"), ]; const offenders: string[] = []; const allowed: string[] = []; const walk = async (dir: string): Promise => { let entries; try { entries = await readdir(dir, { withFileTypes: true }); } catch { return; } for (const e of entries) { if (e.name === "node_modules" || e.name.startsWith(".")) continue; const p = path.join(dir, e.name); if (e.isDirectory()) await walk(p); else if (/\.(ts|tsx|mjs|js)$/.test(e.name)) { const text = await readFile(p, "utf8"); if (text.includes(POSTS_MEDIA_DIRNAME) || text.includes("POSTS_MEDIA_DIRNAME") || text.includes("social/postCapture")) { const rel = path.relative(repo, p); if (rel === CITED_CAPTURE_COPIER) allowed.push(rel); else offenders.push(rel); } } } }; for (const r of roots) await walk(r); assert.deepEqual(offenders, []); // The exception exists and is the only one: if it moves, this moves with it. assert.deepEqual(allowed, [CITED_CAPTURE_COPIER]); });