// Where a post capture lands on disk, and the record that says what it holds. // // Layout, per channel: // channels//posts-media//shot.png — the post, as rendered // channels//posts-media// — its attached media // channels//posts-media//capture.json — this module's record // channels//posts-media//article.* — the X Article the post // links to, when it is one: article.json (its blocks), article.md, // article.png, article.html (the root as X served it) and // article-img-. (its inline images) — xArticleCapture.ts // // A directory per post, unlike the posts themselves (month-sharded JSONL): a // capture is a handful of files, and only for the posts someone asked for. It // sits BESIDE `data/` and `media/`, never in them — no video reader walks it, // and the media tier does not move it. // // EDITOR-ONLY. The export build serves the index's JSON pages and never reads a // channel directory, so nothing here is published (postCapture.test.ts holds // the export's sources to that). // // SERVER-ONLY (node:fs). import { createHash } from "node:crypto"; import { createReadStream } from "node:fs"; import { readdir, stat } from "node:fs/promises"; import path from "node:path"; import { readJsonFile, writeJsonAtomic } from "../lib/jsonFile-server"; import type { PostAvailability } from "../lib/posts"; export const POSTS_MEDIA_DIRNAME = "posts-media"; export const CAPTURE_FILENAME = "capture.json"; export const SHOT_FILENAME = "shot.png"; export const ARTICLE_JSON_FILENAME = "article.json"; export const ARTICLE_MD_FILENAME = "article.md"; export const ARTICLE_SHOT_FILENAME = "article.png"; export const ARTICLE_HTML_FILENAME = "article.html"; // The n-th (1-based) inline image of an article. export function articleImageFilename(n: number, ext: string): string { return `article-img-${n}.${ext}`; } // A file of the article half, never a medium of the post. export function isArticleFile(name: string): boolean { return /^article(\.|-img-)/.test(name); } export function postsMediaDir(channelRoot: string): string { return path.join(channelRoot, POSTS_MEDIA_DIRNAME); } // One post's capture directory. The id is checked here, at the one place a // post id becomes a path: a platform-native id is digits (X) or a short // alphanumeric key (Bluesky), never a separator. export function postCaptureDir(outDir: string, id: string): string { if (!/^[A-Za-z0-9_-]{1,64}$/.test(id)) { throw new Error(`"${id}" is not a post id`); } return path.join(outDir, id); } // What a capture found the post to be. // captured — the post rendered, and what was asked for is on disk // deleted — the platform says the post is gone // unavailable — the post is behind its account's wall (protected, // suspended, withheld): the post may exist, it cannot be shown // login-wall — the platform asked to log in: the session's state, not the // post's, so the run stops // error — the page or the download failed; a later run tries again export type PostCaptureState = | "captured" | "deleted" | "unavailable" | "login-wall" | "error"; // The liveness a capture state says about the post, for the availability // sidecar. A login wall and an error say nothing about the post itself — the // sidecar never records a verdict from ignorance. export function captureAvailability( state: PostCaptureState, ): PostAvailability | undefined { switch (state) { case "captured": return "available"; case "deleted": return "deleted"; case "unavailable": return "account_unavailable"; default: return undefined; } } export type CapturedFile = { // The file's name inside the post's directory. name: string; bytes: number; sha256: string; // Where it came from: the media URL for an attachment, the post URL for the // screenshot. url?: string; }; // How the media half went: downloaded ("ok"), the post has none ("none"), not // asked for or not attempted ("skipped"), or failed ("error"). export type CaptureMediaState = "ok" | "none" | "skipped" | "error"; // The article half: an X Article the post links to, opened and read // (xArticleCapture.ts). `state` is the article page's, in the post's terms: a // deleted or walled article is settled, an error is owed again, a login wall // stopped the run. export type ArticleCaptureRecord = { articleId: string; url: string; capturedAt: string; state: PostCaptureState; title?: string; // The blocks read, for a captured article (article.json holds them). blocks: number; // How the body was read: by X's markers, or the root's text split at block // elements because the markers were missing. extraction?: "structured" | "fallback"; // article.json, article.md, article.png, article.html, each image. files: CapturedFile[]; // article.png stops at a height cap; the article ran longer. trimmed?: boolean; error?: string; }; export type PostCaptureRecord = { version: 1; id: string; // The post's URL the capture read. url: string; capturedAt: string; state: PostCaptureState; // The post sat behind a sensitive-media interstitial, which the capture // opened before shooting. sensitive?: boolean; // The screenshot, when one was taken. shot?: CapturedFile; mediaState: CaptureMediaState; media: CapturedFile[]; // The X Article the post links to, when one was captured or tried. article?: ArticleCaptureRecord; error?: string; }; export async function fileDigest( file: string, ): Promise<{ bytes: number; sha256: string }> { const hash = createHash("sha256"); await new Promise((resolve, reject) => { const s = createReadStream(file); s.on("data", (chunk) => hash.update(chunk)); s.on("error", reject); s.on("end", () => resolve()); }); const { size } = await stat(file); return { bytes: size, sha256: hash.digest("hex") }; } export async function describeCapturedFile( dir: string, name: string, url?: string, ): Promise { const digest = await fileDigest(path.join(dir, name)); return { name, ...digest, ...(url ? { url } : {}) }; } // The media files in a post's directory: everything but the shot, the record, // the article half and a download's leftovers. export async function listCapturedMediaFiles(dir: string): Promise { let names: string[]; try { names = await readdir(dir); } catch { return []; } return names .filter( (n) => n !== SHOT_FILENAME && n !== CAPTURE_FILENAME && !isArticleFile(n) && !n.endsWith(".part") && !n.startsWith("."), ) .sort(); } export async function readPostCapture( dir: string, ): Promise { const read = await readJsonFile(path.join(dir, CAPTURE_FILENAME)); if (!read.ok) return null; const v = read.value as Partial | null; if (!v || typeof v !== "object" || v.version !== 1 || typeof v.id !== "string") { return null; } return v as PostCaptureRecord; } export async function writePostCapture( dir: string, record: PostCaptureRecord, ): Promise { await writeJsonAtomic(path.join(dir, CAPTURE_FILENAME), record, { mkdir: true }); } // Which halves of a capture this run still owes a post, given what is on disk. // A deleted post is settled: the platform said so, and asking again costs a // request for the same answer (`force` asks anyway). A shot on disk is kept; a // media download that did not finish ("error", or never attempted) is owed. // The article half is owed only if the post links to one (the caller knows); // one captured, deleted or walled is settled, one that failed is owed. export function captureWork( existing: PostCaptureRecord | null, wanted: { shots: boolean; media: boolean; force: boolean; articles?: boolean }, ): { shot: boolean; media: boolean; article: boolean } { const articles = wanted.articles ?? false; if (wanted.force || !existing) { return { shot: wanted.shots, media: wanted.media, article: articles }; } if (existing.state === "deleted") return { shot: false, media: false, article: false }; return { shot: wanted.shots && !existing.shot, media: wanted.media && existing.mediaState !== "ok" && existing.mediaState !== "none", article: articles && !articleSettled(existing.article), }; } export function articleSettled(article: ArticleCaptureRecord | undefined): boolean { return ( article?.state === "captured" || article?.state === "deleted" || article?.state === "unavailable" ); }