commit 25dc0dc29913e5a0b0a1221797bf8c6f5f805b98
parent a0e73ebe37e6f0f743a387712455d066e758163f
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Wed, 7 Oct 2026 13:17:54 -0400
capture: a forum post's media is what its page holds now, beside the archive's
captureForumPosts downloaded only the media in the ARCHIVED record, which is
whatever the parser of its day saw. Posts archived before the Kiwi Farms
player fix carry their forum-hosted videos as a bare duration and no media,
so a re-capture saved only the screenshot. The shot step already loads the
post's page; it now parses it too, and the capture downloads the page's media
first, then anything only the archive kept (deduped by URL). A video gets a
10-minute download timeout instead of an image's minute.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
2 files changed, 65 insertions(+), 7 deletions(-)
diff --git a/common/social/xenforoFetcher.test.ts b/common/social/xenforoFetcher.test.ts
@@ -544,3 +544,35 @@ test("downloadable media: files only", () => {
],
);
});
+
+test("capture: downloads what the post's page holds NOW, beside what the archive kept", async () => {
+ const outDir = await mkdtemp(path.join(os.tmpdir(), "forum-capture-live-"));
+ const video = "https://uploads.kiwifarms.st/data/video/9665/9665533-abc.mp4?hash=Zz";
+ const player =
+ `<div class="ephyra-player ephyra-player--video" data-media-type="video" data-duration-label="1:11"` +
+ ` data-source-fallback="${video.replace("https:", "")}" data-filename="clip.mp4"></div>`;
+ const postUrl = `${ORIGIN}/posts/9100/`;
+ const html = threadPage({
+ page: 1,
+ last: 1,
+ posts: [{ id: 9100, author: "Member1", userId: 51, ts: T0, position: 1, body: `for those that want proof:${player}` }],
+ });
+ const { loader, got } = captureLoader({ [postUrl]: html }, { x: 0, y: 0, width: 600, height: 200 });
+ const res = await captureForumPosts(
+ {
+ ids: ["9100"],
+ handle: "t",
+ accountUrl: THREAD_URL,
+ outDir,
+ // Archived by the parser of its day: the video was a bare "1:11", no media.
+ archived: new Map([["9100", { text: "for those that want proof:\n1:11", url: postUrl, media: [{ kind: "image", url: "https://images.example/b.jpg" }] }]]),
+ signal: new AbortController().signal,
+ },
+ { loader, pauseMs: () => 7_000, sleep: async () => {}, now: () => new Date(0) },
+ );
+ assert.deepEqual(res.outcomes.map((o) => [o.id, o.state, o.files]), [["9100", "captured", 3]]);
+ assert.deepEqual(got, [video, "https://images.example/b.jpg"], "the page's video first, then the archive's image");
+ const rec = JSON.parse(await readFile(path.join(outDir, "9100", "capture.json"), "utf8"));
+ assert.equal(rec.mediaState, "ok");
+ assert.deepEqual(rec.media.map((m: { url: string }) => m.url), [video, "https://images.example/b.jpg"]);
+});
diff --git a/common/social/xenforoFetcher.ts b/common/social/xenforoFetcher.ts
@@ -32,7 +32,7 @@
import { mkdir, writeFile } from "node:fs/promises";
import path from "node:path";
import { getPaths } from "../lib/paths";
-import type { Post } from "../lib/posts";
+import type { Post, PostMedia } from "../lib/posts";
import {
registerSocialFetcher,
type PostCaptureInput,
@@ -365,6 +365,10 @@ export type ForumShotResult = {
shot?: CapturedFile;
error?: string;
stop?: string;
+ // The post's media as its page reads NOW, parsed from the page just loaded.
+ // An archived record keeps what the parser of its day saw: before the
+ // player fix, a forum-hosted video was a bare duration and no media.
+ liveMedia?: PostMedia[];
};
export async function shootForumPost(
@@ -391,6 +395,14 @@ export async function shootForumPost(
stop: why.error,
};
}
+ let liveMedia: PostMedia[] | undefined;
+ try {
+ liveMedia = parseXenforoThreadPage(got.html, { channelSlug: "capture", pageUrl: postUrl }).posts.find(
+ (p) => p.id === id,
+ )?.media;
+ } catch {
+ liveMedia = undefined;
+ }
await page.waitForTimeout(1_000);
await page.evaluate(PREPARE_SCRIPT(id)).catch(() => {});
const snap = (await page.evaluate(SNAPSHOT_SCRIPT(id))) as ForumPostSnapshot;
@@ -413,7 +425,7 @@ export async function shootForumPost(
}
await mkdir(dir, { recursive: true });
await writeFile(path.join(dir, SHOT_FILENAME), png);
- return { state: "captured", shot: await describeCapturedFile(dir, SHOT_FILENAME, postUrl) };
+ return { state: "captured", shot: await describeCapturedFile(dir, SHOT_FILENAME, postUrl), ...(liveMedia ? { liveMedia } : {}) };
}
const EXT_BY_TYPE: Record<string, string> = {
@@ -438,12 +450,22 @@ function extFor(url: string, contentType: string | undefined): string {
// carries as files. Embeds and link cards are pages, not files.
export function downloadableMedia(
media: ReadonlyArray<{ kind: string; url: string }> | undefined,
+ ...more: ReadonlyArray<ReadonlyArray<{ kind: string; url: string }> | undefined>
): { kind: string; url: string }[] {
- return (media ?? []).filter(
- (m) => (m.kind === "image" || m.kind === "attachment" || m.kind === "video") && /^https?:\/\//.test(m.url),
- );
+ const out: { kind: string; url: string }[] = [];
+ for (const list of [media, ...more]) {
+ for (const m of list ?? []) {
+ if (!(m.kind === "image" || m.kind === "attachment" || m.kind === "video")) continue;
+ if (!/^https?:\/\//.test(m.url) || out.some((x) => x.url === m.url)) continue;
+ out.push(m);
+ }
+ }
+ return out;
}
+// A video can be hundreds of megabytes; an image is not.
+const MEDIA_TIMEOUT_MS = { video: 600_000, other: 60_000 };
+
export type ForumCaptureDeps = {
loader: ForumPageLoader & { page(): Promise<PageLike> };
pauseMs: () => number;
@@ -496,6 +518,7 @@ export async function captureForumPosts(
archived?.url && /^https?:\/\//.test(archived.url) ? archived.url : `${thread.origin}/posts/${id}/`;
let state: PostCaptureState | undefined = work.shot ? undefined : existing?.state;
+ let liveMedia: PostMedia[] | undefined;
let shot = work.shot ? undefined : existing?.shot;
let error: string | undefined;
let stop: string | undefined;
@@ -516,13 +539,15 @@ export async function captureForumPosts(
shot = res.shot;
error = res.error;
stop = res.stop;
+ liveMedia = res.liveMedia;
}
let mediaState: CaptureMediaState = work.media ? "skipped" : (existing?.mediaState ?? "skipped");
let media: CapturedFile[] = work.media ? [] : (existing?.media ?? []);
const postIsThere = state === undefined || state === "captured";
if (work.media && postIsThere && !stop) {
- const files = downloadableMedia(archived?.media);
+ // What the page holds now first, then anything only the archive kept.
+ const files = downloadableMedia(liveMedia, archived?.media);
if (files.length === 0) {
mediaState = "none";
state ??= "captured";
@@ -533,7 +558,8 @@ export async function captureForumPosts(
await contact();
if (signal.aborted) return stopped("Cancelled; the rest are left for a later run.");
try {
- const res = await page.request!.get(m.url, { timeout: 60_000, failOnStatusCode: false });
+ const timeout = m.kind === "video" ? MEDIA_TIMEOUT_MS.video : MEDIA_TIMEOUT_MS.other;
+ const res = await page.request!.get(m.url, { timeout, failOnStatusCode: false });
if (!res.ok()) {
failed++;
onLog?.(`${id}: media ${n + 1} answered HTTP ${res.status()}.`);
