commit 6c7ddce2d596256b52732849f32d009e22422e2f
parent a0e73ebe37e6f0f743a387712455d066e758163f
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Wed, 7 Oct 2026 13:17:54 -0400
Merge kf-capture-live-media (forum capture downloads the media a post's page holds now)
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
2 files changed, 65 insertions(+), 7 deletions(-)
diff --git a/common/social/xenforoFetcher.test.ts b/common/social/xenforoFetcher.test.ts
@@ -544,3 +544,35 @@ test("downloadable media: files only", () => {
],
);
});
+
+test("capture: downloads what the post's page holds NOW, beside what the archive kept", async () => {
+ const outDir = await mkdtemp(path.join(os.tmpdir(), "forum-capture-live-"));
+ const video = "https://uploads.kiwifarms.st/data/video/9665/9665533-abc.mp4?hash=Zz";
+ const player =
+ `<div class="ephyra-player ephyra-player--video" data-media-type="video" data-duration-label="1:11"` +
+ ` data-source-fallback="${video.replace("https:", "")}" data-filename="clip.mp4"></div>`;
+ const postUrl = `${ORIGIN}/posts/9100/`;
+ const html = threadPage({
+ page: 1,
+ last: 1,
+ posts: [{ id: 9100, author: "Member1", userId: 51, ts: T0, position: 1, body: `for those that want proof:${player}` }],
+ });
+ const { loader, got } = captureLoader({ [postUrl]: html }, { x: 0, y: 0, width: 600, height: 200 });
+ const res = await captureForumPosts(
+ {
+ ids: ["9100"],
+ handle: "t",
+ accountUrl: THREAD_URL,
+ outDir,
+ // Archived by the parser of its day: the video was a bare "1:11", no media.
+ archived: new Map([["9100", { text: "for those that want proof:\n1:11", url: postUrl, media: [{ kind: "image", url: "https://images.example/b.jpg" }] }]]),
+ signal: new AbortController().signal,
+ },
+ { loader, pauseMs: () => 7_000, sleep: async () => {}, now: () => new Date(0) },
+ );
+ assert.deepEqual(res.outcomes.map((o) => [o.id, o.state, o.files]), [["9100", "captured", 3]]);
+ assert.deepEqual(got, [video, "https://images.example/b.jpg"], "the page's video first, then the archive's image");
+ const rec = JSON.parse(await readFile(path.join(outDir, "9100", "capture.json"), "utf8"));
+ assert.equal(rec.mediaState, "ok");
+ assert.deepEqual(rec.media.map((m: { url: string }) => m.url), [video, "https://images.example/b.jpg"]);
+});
diff --git a/common/social/xenforoFetcher.ts b/common/social/xenforoFetcher.ts
@@ -32,7 +32,7 @@
import { mkdir, writeFile } from "node:fs/promises";
import path from "node:path";
import { getPaths } from "../lib/paths";
-import type { Post } from "../lib/posts";
+import type { Post, PostMedia } from "../lib/posts";
import {
registerSocialFetcher,
type PostCaptureInput,
@@ -365,6 +365,10 @@ export type ForumShotResult = {
shot?: CapturedFile;
error?: string;
stop?: string;
+ // The post's media as its page reads NOW, parsed from the page just loaded.
+ // An archived record keeps what the parser of its day saw: before the
+ // player fix, a forum-hosted video was a bare duration and no media.
+ liveMedia?: PostMedia[];
};
export async function shootForumPost(
@@ -391,6 +395,14 @@ export async function shootForumPost(
stop: why.error,
};
}
+ let liveMedia: PostMedia[] | undefined;
+ try {
+ liveMedia = parseXenforoThreadPage(got.html, { channelSlug: "capture", pageUrl: postUrl }).posts.find(
+ (p) => p.id === id,
+ )?.media;
+ } catch {
+ liveMedia = undefined;
+ }
await page.waitForTimeout(1_000);
await page.evaluate(PREPARE_SCRIPT(id)).catch(() => {});
const snap = (await page.evaluate(SNAPSHOT_SCRIPT(id))) as ForumPostSnapshot;
@@ -413,7 +425,7 @@ export async function shootForumPost(
}
await mkdir(dir, { recursive: true });
await writeFile(path.join(dir, SHOT_FILENAME), png);
- return { state: "captured", shot: await describeCapturedFile(dir, SHOT_FILENAME, postUrl) };
+ return { state: "captured", shot: await describeCapturedFile(dir, SHOT_FILENAME, postUrl), ...(liveMedia ? { liveMedia } : {}) };
}
const EXT_BY_TYPE: Record<string, string> = {
@@ -438,12 +450,22 @@ function extFor(url: string, contentType: string | undefined): string {
// carries as files. Embeds and link cards are pages, not files.
export function downloadableMedia(
media: ReadonlyArray<{ kind: string; url: string }> | undefined,
+ ...more: ReadonlyArray<ReadonlyArray<{ kind: string; url: string }> | undefined>
): { kind: string; url: string }[] {
- return (media ?? []).filter(
- (m) => (m.kind === "image" || m.kind === "attachment" || m.kind === "video") && /^https?:\/\//.test(m.url),
- );
+ const out: { kind: string; url: string }[] = [];
+ for (const list of [media, ...more]) {
+ for (const m of list ?? []) {
+ if (!(m.kind === "image" || m.kind === "attachment" || m.kind === "video")) continue;
+ if (!/^https?:\/\//.test(m.url) || out.some((x) => x.url === m.url)) continue;
+ out.push(m);
+ }
+ }
+ return out;
}
+// A video can be hundreds of megabytes; an image is not.
+const MEDIA_TIMEOUT_MS = { video: 600_000, other: 60_000 };
+
export type ForumCaptureDeps = {
loader: ForumPageLoader & { page(): Promise<PageLike> };
pauseMs: () => number;
@@ -496,6 +518,7 @@ export async function captureForumPosts(
archived?.url && /^https?:\/\//.test(archived.url) ? archived.url : `${thread.origin}/posts/${id}/`;
let state: PostCaptureState | undefined = work.shot ? undefined : existing?.state;
+ let liveMedia: PostMedia[] | undefined;
let shot = work.shot ? undefined : existing?.shot;
let error: string | undefined;
let stop: string | undefined;
@@ -516,13 +539,15 @@ export async function captureForumPosts(
shot = res.shot;
error = res.error;
stop = res.stop;
+ liveMedia = res.liveMedia;
}
let mediaState: CaptureMediaState = work.media ? "skipped" : (existing?.mediaState ?? "skipped");
let media: CapturedFile[] = work.media ? [] : (existing?.media ?? []);
const postIsThere = state === undefined || state === "captured";
if (work.media && postIsThere && !stop) {
- const files = downloadableMedia(archived?.media);
+ // What the page holds now first, then anything only the archive kept.
+ const files = downloadableMedia(liveMedia, archived?.media);
if (files.length === 0) {
mediaState = "none";
state ??= "captured";
@@ -533,7 +558,8 @@ export async function captureForumPosts(
await contact();
if (signal.aborted) return stopped("Cancelled; the rest are left for a later run.");
try {
- const res = await page.request!.get(m.url, { timeout: 60_000, failOnStatusCode: false });
+ const timeout = m.kind === "video" ? MEDIA_TIMEOUT_MS.video : MEDIA_TIMEOUT_MS.other;
+ const res = await page.request!.get(m.url, { timeout, failOnStatusCode: false });
if (!res.ok()) {
failed++;
onLog?.(`${id}: media ${n + 1} answered HTTP ${res.status()}.`);
