commit 74dff241593b9d4303d848e4b50841d3ca55d3c8
parent 21982be9852a50527e9ac65d3a9f4ef0ef40aa2e
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 6 Aug 2026 19:15:42 -0400
Track deleted posts, the way videos track availability
A deleted post is the artefact a commentary archive exists to preserve, and
the only way to know one was deleted is to have looked. This gives posts the
same deleted-detection videos already have.
Mirrors controller/checkAvailability.ts throughout:
- verdicts live in a sidecar (posts-availability.json) rather than rewriting
the append-only JSONL, and buildIndex merges them into the served pages —
the same shape as per-video availability.json merged into a summary
- history is appended only when a verdict CHANGES, preserving the moment a
post was first seen deleted
- a failed check records `error`, never `deleted`: we never report a deletion
from ignorance. Likewise a suspended/protected account records
`account_unavailable`, since that can reverse and is not the post's doing
- `stale` mode checks least-recently-checked first, so repeated runs sweep
the whole archive instead of re-checking the same head
The sidecar is per channel, not per post: posts live in month-sharded JSONL
with no per-post directory, so the per-video file layout does not transfer.
checkAvailability is an OPTIONAL capability on SocialFetcher. Bluesky answers
25 posts per unauthenticated getPosts call — absence from the response IS the
deletion signal — so a whole channel is a handful of requests. The Nitter path
checks one status page per post, which is far heavier, hence the per-run cap.
gallery-dl has no cheap probe and says so rather than guessing; the UI disables
the sweep and explains that switching scraping method enables it.
The posts availability sidecar is folded into the channel's index signature,
so a deletion sweep rebuilds the served pages even though no shard changed.
Deleted posts surface as the same rose "Deleted" badge videos use, in result
cards and in the modal (whose source link becomes "Original (deleted)"), and
they participate in the existing VideoState availability filter rather than
getting a parallel control.
Verified live against the real 829-post Bluesky archive: two sweeps of 120
found 45 genuinely deleted posts, the second sweep covered a different 120
(589 left unchecked), and the rebuilt pages carried isDeleted through.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Diffstat:
19 files changed, 804 insertions(+), 9 deletions(-)
diff --git a/common/bin/check-post-availability.ts b/common/bin/check-post-availability.ts
@@ -0,0 +1,54 @@
+#!/usr/bin/env tsx
+// Check which of a social channel's archived posts have been deleted at the
+// source.
+//
+// pnpm --filter yt-dlp-transcript-common exec tsx \
+// bin/check-post-availability.ts --slug <channel> [--mode stale|unchecked|all] [--limit N]
+
+import { getPaths } from "../lib/paths";
+import {
+ CHECK_POST_AVAILABILITY_MODES,
+ checkPostAvailability,
+ type CheckPostAvailabilityMode,
+} from "../controller/checkPostAvailability";
+import { parseFlags } from "./_parseFlags";
+
+const flags = parseFlags(process.argv.slice(2));
+const slug = flags.slug;
+if (!slug) {
+ console.error(
+ "Usage: check-post-availability.ts --slug <channel> [--mode stale|unchecked|all] [--limit N]",
+ );
+ process.exit(2);
+}
+const mode = (flags.mode ?? "stale") as CheckPostAvailabilityMode;
+if (!CHECK_POST_AVAILABILITY_MODES.includes(mode)) {
+ console.error(`--mode must be one of: ${CHECK_POST_AVAILABILITY_MODES.join(", ")}`);
+ process.exit(2);
+}
+const limitRaw = flags.limit ? Number(flags.limit) : undefined;
+
+checkPostAvailability({
+ paths: getPaths(),
+ slug,
+ mode,
+ limit:
+ typeof limitRaw === "number" && Number.isFinite(limitRaw) && limitRaw > 0
+ ? Math.floor(limitRaw)
+ : undefined,
+ onLog: (line) => console.log(line),
+})
+ .then((r) => {
+ if (!r.ok) {
+ console.error(`Failed: ${r.error}`);
+ process.exit(1);
+ }
+ console.log(
+ `Checked ${r.checked}; ${r.newlyDeleted.length} newly deleted; ` +
+ `${r.deleted} deleted overall; ${r.remaining} never checked.`,
+ );
+ })
+ .catch((err) => {
+ console.error(err);
+ process.exit(1);
+ });
diff --git a/common/components/PostModal.tsx b/common/components/PostModal.tsx
@@ -154,6 +154,19 @@ function PostBody({
<time dateTime={post.createdAt}>{formatWhen(post.createdAt)}</time>
{post.isRepost && <Badge>repost</Badge>}
{post.isReply && <Badge>reply</Badge>}
+ {/* The whole point of archiving: this post no longer exists upstream. */}
+ {post.isDeleted && (
+ <span
+ title={
+ post.availabilityCheckedAt
+ ? `Confirmed removed from the source (checked ${new Date(post.availabilityCheckedAt).toLocaleString()})`
+ : "Removed from the source platform"
+ }
+ className="rounded bg-rose-100 px-1.5 py-0.5 text-[10px] uppercase tracking-wide text-rose-800 dark:bg-rose-900/40 dark:text-rose-200"
+ >
+ deleted
+ </span>
+ )}
</header>
<p className={"whitespace-pre-wrap " + (compact ? "text-sm" : "text-base mt-2")}>
@@ -184,7 +197,7 @@ function PostBody({
rel="noopener noreferrer"
className="underline"
>
- Open original
+ {post.isDeleted ? "Original (deleted)" : "Open original"}
</a>
{post.engagement?.likes != null && (
<span>{post.engagement.likes} likes</span>
diff --git a/common/components/SearchResults.tsx b/common/components/SearchResults.tsx
@@ -494,6 +494,9 @@ const ResultCard = memo(function ResultCard({
{group.post ? (
<>
<span className="truncate flex-1 min-w-0">{group.post.text}</span>
+ {/* A deleted post is the archive's most valuable artefact — flag
+ it with the same badge videos use. */}
+ {group.post.isDeleted && <VideoStateBadge state="deleted" />}
<PostBadge platform={group.post.platform} />
</>
) : (
diff --git a/common/components/SearchSessionContext.tsx b/common/components/SearchSessionContext.tsx
@@ -858,6 +858,11 @@ function useSearchSessionState() {
if (!post) continue;
if (committedDateFrom && post.uploadDate < committedDateFrom) continue;
if (committedDateTo && post.uploadDate > committedDateTo) continue;
+ // A post is either still up or deleted at the source — the two states of
+ // the six that can apply to it — so it participates in the same
+ // availability filter videos use.
+ const postState: VideoState = post.isDeleted ? "deleted" : "available";
+ if (!committedStates.has(postState)) continue;
out.push({
slug,
// A post has no title; the card renders its body, and the author
@@ -867,8 +872,7 @@ function useSearchSessionState() {
date: post.createdAt,
isLivestream: false,
ageRestricted: false,
- // A post has no channel listing to fall out of.
- state: "available",
+ state: postState,
platform: post.platform,
uploadDate: post.uploadDate,
hits: hitsBySlug.get(slug) ?? [],
@@ -888,6 +892,7 @@ function useSearchSessionState() {
accentOf,
committedDateFrom,
committedDateTo,
+ committedStates,
]);
const leafStates = treeProgress?.leafStates ?? new Map<string, LeafState>();
diff --git a/common/controller/buildIndex.ts b/common/controller/buildIndex.ts
@@ -99,6 +99,7 @@ import {
import { loadMaybeMissing } from "./quickAvailabilityCheck";
import {
POSTS_MANIFEST_VERSION,
+ isPostGone,
SITE_POSTS_MANIFEST_VERSION,
comparePostsNewestFirst,
postsPageFileName,
@@ -112,6 +113,8 @@ import {
channelPostsDir,
listPostShards,
readPostShard,
+ readPostAvailability,
+ postsAvailabilityPath,
} from "../lib/posts-server";
import { isSocialChannel } from "../lib/channelConfig";
import {
@@ -1319,6 +1322,14 @@ export async function buildIndex({
/* shard vanished mid-scan — treat as absent */
}
}
+ // Include the availability sidecar: a deletion sweep changes what the
+ // served pages must say, even though no shard changed.
+ try {
+ const st = await stat(postsAvailabilityPath(channelRoot));
+ sigParts.push(`avail:${st.size}:${st.mtimeMs}`);
+ } catch {
+ /* never checked */
+ }
const signature = sigParts.join("|");
const prevStat = channelPostsStatsDb.get(channelSlug);
const manifestPresent = await readFile(
@@ -1345,6 +1356,10 @@ export async function buildIndex({
const pk = key as PostIndexKey;
if (pk[1] === channelSlug) posts.remove(pk);
}
+ // Merge the deleted-post sidecar in, exactly as video availability is
+ // merged into a summary: the sidecar is the source of truth, the flag on
+ // the served record is derived. The JSONL is never rewritten.
+ const postAvailability = await readPostAvailability(channelRoot);
const channelPosts: Post[] = [];
for (const shard of shards) {
channelPosts.push(...(await readPostShard(channelRoot, shard)));
@@ -1355,6 +1370,12 @@ export async function buildIndex({
for (const post of channelPosts) byId.set(post.id, post);
const ordered = [...byId.values()].sort(comparePostsNewestFirst);
for (const post of ordered) {
+ const rec = postAvailability[post.id];
+ if (rec) {
+ post.availability = rec.availability;
+ post.availabilityCheckedAt = rec.checkedAt;
+ if (isPostGone(rec.availability)) post.isDeleted = true;
+ }
posts.put([post.createdAt, channelSlug, post.id], post);
}
diff --git a/common/controller/checkPostAvailability.ts b/common/controller/checkPostAvailability.ts
@@ -0,0 +1,208 @@
+// "Are these archived posts still up?" — the posts analogue of
+// controller/checkAvailability.ts.
+//
+// For a commentary archive this is arguably the most valuable sweep there is:
+// a deleted post is exactly the artefact the archive exists to preserve, and
+// the only way to know one was deleted is to have looked.
+//
+// Same discipline as the video checker:
+// - the append-only JSONL is never rewritten; verdicts live in a sidecar
+// that the index merges into the served pages
+// - history is appended only when a verdict CHANGES, preserving the moment a
+// post was first seen deleted
+// - a failed check records `error`, never `deleted` — we never report a
+// deletion from ignorance
+//
+// Modes mirror the video checker's: `stale` re-checks the least recently
+// checked first (the default, cheap and incremental), `unchecked` only covers
+// posts never looked at, `all` re-checks everything.
+
+import path from "node:path";
+import { readChannelConfig } from "./channels";
+import type { Paths } from "../lib/paths";
+import { isSocialChannel } from "../lib/channelConfig";
+import {
+ mergePostAvailability,
+ readAllPosts,
+ readPostAvailability,
+ writePostAvailability,
+} from "../lib/posts-server";
+import type { PostAvailability, PostAvailabilityMap } from "../lib/posts";
+import {
+ handleFromAccountUrl,
+ resolveSocialFetcher,
+} from "../social/fetchers";
+import "../social/blueskyFetcher";
+import "../social/xGalleryDlFetcher";
+import "../social/xPlaywrightFetcher";
+import "../social/xNitterFetcher";
+
+export type CheckPostAvailabilityMode = "stale" | "unchecked" | "all";
+
+export const CHECK_POST_AVAILABILITY_MODES: CheckPostAvailabilityMode[] = [
+ "stale",
+ "unchecked",
+ "all",
+];
+
+// Default ceiling per run. Bluesky answers 25 posts per request so it could
+// take far more, but a per-post browser check (X) cannot — one bound keeps the
+// job predictable on both.
+export const DEFAULT_POST_CHECK_LIMIT = 200;
+
+export type CheckPostAvailabilityOptions = {
+ paths: Paths;
+ slug: string;
+ mode?: CheckPostAvailabilityMode;
+ limit?: number;
+ onLog?: (line: string) => void;
+ signal?: AbortSignal;
+};
+
+export type CheckPostAvailabilityResult = {
+ ok: boolean;
+ checked: number;
+ deleted: number;
+ newlyDeleted: string[];
+ changed: number;
+ remaining: number;
+ error?: string;
+};
+
+// Which posts to look at this run, oldest-checked first so repeated runs sweep
+// the whole archive rather than re-checking the same head every time.
+export function selectPostsToCheck(
+ ids: ReadonlyArray<string>,
+ availability: PostAvailabilityMap,
+ mode: CheckPostAvailabilityMode,
+ limit: number,
+): string[] {
+ const candidates = ids.filter((id) => {
+ const rec = availability[id];
+ if (mode === "all") return true;
+ if (mode === "unchecked") return !rec;
+ // "stale": everything is a candidate, ordered by how long ago we looked.
+ return true;
+ });
+ if (mode === "stale") {
+ candidates.sort((a, b) => {
+ // Never-checked first, then least-recently-checked.
+ const ra = availability[a]?.checkedAt ?? "";
+ const rb = availability[b]?.checkedAt ?? "";
+ return ra.localeCompare(rb);
+ });
+ }
+ return candidates.slice(0, Math.max(0, limit));
+}
+
+export async function checkPostAvailability(
+ opts: CheckPostAvailabilityOptions,
+): Promise<CheckPostAvailabilityResult> {
+ const { paths, slug, onLog, signal } = opts;
+ const mode = opts.mode ?? "stale";
+ const limit = opts.limit ?? DEFAULT_POST_CHECK_LIMIT;
+ const log = (line: string) => onLog?.(line);
+ const channelRoot = path.join(paths.channelsDir, slug);
+ const empty = {
+ checked: 0,
+ deleted: 0,
+ newlyDeleted: [] as string[],
+ changed: 0,
+ remaining: 0,
+ };
+
+ const config = await readChannelConfig(paths, slug);
+ if (!config) {
+ return { ok: false, ...empty, error: `No such channel: ${slug}` };
+ }
+ if (!isSocialChannel(config)) {
+ return { ok: false, ...empty, error: `${slug} is not a social channel.` };
+ }
+
+ const fetcher = resolveSocialFetcher(config.postFetcher, config.url);
+ if (!fetcher) {
+ return { ok: false, ...empty, error: `No post fetcher configured for ${slug}.` };
+ }
+ if (!fetcher.checkAvailability) {
+ // Better to say so than to guess. gallery-dl, for instance, has no cheap
+ // per-post liveness probe, so its channels should switch fetcher to sweep.
+ return {
+ ok: false,
+ ...empty,
+ error:
+ `${fetcher.label} cannot check post availability. ` +
+ `Switch this channel's scraping method to one that can.`,
+ };
+ }
+
+ const handle = config.socialHandle ?? handleFromAccountUrl(config.url ?? "") ?? "";
+ if (!handle) {
+ return { ok: false, ...empty, error: `No account handle for ${slug}.` };
+ }
+
+ const posts = await readAllPosts(channelRoot);
+ if (posts.length === 0) {
+ return { ok: true, ...empty };
+ }
+ const availability = await readPostAvailability(channelRoot);
+ const ids = posts.map((p) => p.id);
+ const batch = selectPostsToCheck(ids, availability, mode, limit);
+ if (batch.length === 0) {
+ log(`Nothing to check for ${slug} (mode: ${mode}).`);
+ return { ok: true, ...empty };
+ }
+
+ log(
+ `Checking ${batch.length} of ${ids.length} archived post(s) for ${slug} ` +
+ `via ${fetcher.label} (mode: ${mode}).`,
+ );
+
+ const controller = new AbortController();
+ let observations: Map<string, PostAvailability>;
+ try {
+ observations = await fetcher.checkAvailability({
+ ids: batch,
+ handle,
+ signal: signal ?? controller.signal,
+ onLog: log,
+ });
+ } catch (err) {
+ const message = (err as Error).message;
+ log(`[error] ${message}`);
+ return { ok: false, ...empty, error: message };
+ }
+
+ const checkedAt = new Date().toISOString();
+ const { map, newlyDeleted, changed } = mergePostAvailability(
+ availability,
+ observations,
+ checkedAt,
+ );
+ await writePostAvailability(channelRoot, map);
+
+ const deleted = Object.values(map).filter(
+ (r) => r.availability === "deleted",
+ ).length;
+ const checkedIds = new Set(Object.keys(map));
+ const remaining = ids.filter((id) => !checkedIds.has(id)).length;
+
+ if (newlyDeleted.length > 0) {
+ log(
+ `${newlyDeleted.length} post(s) newly detected as DELETED: ${newlyDeleted.slice(0, 10).join(", ")}` +
+ (newlyDeleted.length > 10 ? ` (+${newlyDeleted.length - 10} more)` : ""),
+ );
+ }
+ log(
+ `Done: ${observations.size} checked, ${changed} changed, ${deleted} deleted overall, ` +
+ `${remaining} never checked.`,
+ );
+
+ return {
+ ok: true,
+ checked: observations.size,
+ deleted,
+ newlyDeleted,
+ changed,
+ remaining,
+ };
+}
diff --git a/common/jobs/jobKinds.ts b/common/jobs/jobKinds.ts
@@ -226,6 +226,15 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
bookmarkable: true,
queueKeyStrategy: "platform",
},
+ // The posts analogue of the video availability check: which archived posts
+ // have since been deleted at the source.
+ "check-post-availability": {
+ kind: "check-post-availability",
+ label: "Check deleted posts",
+ drainable: true,
+ bookmarkable: true,
+ queueKeyStrategy: "platform",
+ },
sync: {
kind: "sync",
label: "Sync",
diff --git a/common/lib/posts-server.test.ts b/common/lib/posts-server.test.ts
@@ -146,3 +146,79 @@ test("a truncated JSONL line does not poison the rest of its shard", async () =>
assert.deepEqual(posts.map((p) => p.id).sort(), ["good1", "good2"]);
});
});
+
+// ─── deleted-post tracking ───
+
+test("availability merges, and history records only real changes", async () => {
+ const { mergePostAvailability } = await import("./posts-server");
+ const t1 = "2026-01-01T00:00:00.000Z";
+ const t2 = "2026-01-02T00:00:00.000Z";
+ const t3 = "2026-01-03T00:00:00.000Z";
+
+ // First observation: no history yet.
+ const a = mergePostAvailability({}, new Map([["p1", "available"]]), t1);
+ assert.equal(a.map.p1.availability, "available");
+ assert.equal(a.map.p1.history?.length ?? 0, 0);
+ assert.deepEqual(a.newlyDeleted, []);
+ assert.equal(a.changed, 1);
+
+ // Same verdict again: only the checkedAt moves, no history growth.
+ const b = mergePostAvailability(a.map, new Map([["p1", "available"]]), t2);
+ assert.equal(b.map.p1.checkedAt, t2);
+ assert.equal(b.map.p1.history?.length ?? 0, 0);
+ assert.equal(b.changed, 0);
+
+ // A real change appends the PREVIOUS verdict, preserving when it was true.
+ const c = mergePostAvailability(b.map, new Map([["p1", "deleted"]]), t3);
+ assert.equal(c.map.p1.availability, "deleted");
+ assert.deepEqual(c.newlyDeleted, ["p1"]);
+ assert.equal(c.map.p1.history?.at(-1)?.availability, "available");
+ assert.equal(c.map.p1.history?.at(-1)?.at, t2);
+
+ // Already-deleted is not "newly" deleted on a later sweep.
+ const d = mergePostAvailability(c.map, new Map([["p1", "deleted"]]), t3);
+ assert.deepEqual(d.newlyDeleted, []);
+});
+
+test("a failed check records error, never a deletion", async () => {
+ const { mergePostAvailability } = await import("./posts-server");
+ const r = mergePostAvailability(
+ {},
+ new Map([["p1", "error"], ["p2", "account_unavailable"]]),
+ "2026-01-01T00:00:00.000Z",
+ );
+ // Neither is evidence the post itself was removed.
+ assert.deepEqual(r.newlyDeleted, []);
+ assert.equal(r.map.p1.availability, "error");
+ assert.equal(r.map.p2.availability, "account_unavailable");
+});
+
+test("availability round-trips through the sidecar file", async () => {
+ const { readPostAvailability, writePostAvailability } = await import("./posts-server");
+ await withTempChannel(async (root) => {
+ assert.deepEqual(await readPostAvailability(root), {});
+ await writePostAvailability(root, {
+ p1: { availability: "deleted", checkedAt: "2026-01-01T00:00:00.000Z" },
+ bad: { availability: "nonsense", checkedAt: "x" } as never,
+ });
+ const back = await readPostAvailability(root);
+ assert.equal(back.p1.availability, "deleted");
+ // A malformed entry is dropped rather than crashing the read.
+ assert.equal(back.bad, undefined);
+ });
+});
+
+test("selectPostsToCheck sweeps least-recently-checked first", async () => {
+ const { selectPostsToCheck } = await import("../controller/checkPostAvailability");
+ const avail = {
+ a: { availability: "available" as const, checkedAt: "2026-03-01T00:00:00.000Z" },
+ b: { availability: "available" as const, checkedAt: "2026-01-01T00:00:00.000Z" },
+ };
+ const ids = ["a", "b", "c"];
+ // "c" was never checked, so it sorts first; then the oldest check.
+ assert.deepEqual(selectPostsToCheck(ids, avail, "stale", 3), ["c", "b", "a"]);
+ // Repeated runs therefore cover the whole archive rather than the same head.
+ assert.deepEqual(selectPostsToCheck(ids, avail, "stale", 1), ["c"]);
+ assert.deepEqual(selectPostsToCheck(ids, avail, "unchecked", 10), ["c"]);
+ assert.equal(selectPostsToCheck(ids, avail, "all", 10).length, 3);
+});
diff --git a/common/lib/posts-server.ts b/common/lib/posts-server.ts
@@ -17,9 +17,13 @@ import { appendFile } from "node:fs/promises";
import { readArchive } from "./archive";
import {
comparePostsNewestFirst,
+ isPostAvailability,
monthShardFromCreatedAt,
parsePost,
type Post,
+ type PostAvailability,
+ type PostAvailabilityMap,
+ type PostAvailabilityRecord,
type PostPlatform,
} from "./posts";
@@ -207,6 +211,93 @@ export async function latestPostCreatedAt(
}
// ---------------------------------------------------------------------------
+// Availability sidecar ("is this post still up?")
+// ---------------------------------------------------------------------------
+//
+// One file per channel rather than per post: posts live in month-sharded JSONL
+// with no per-post directory, so the per-video availability.json layout does
+// not transfer. The JSONL itself stays append-only and is never rewritten —
+// the sidecar is the source of truth and is merged into the served page tree
+// at index time, exactly as video availability is merged into a summary.
+
+export const POSTS_AVAILABILITY_FILENAME = "posts-availability.json";
+
+export function postsAvailabilityPath(channelRoot: string): string {
+ return path.join(channelRoot, POSTS_AVAILABILITY_FILENAME);
+}
+
+export async function readPostAvailability(
+ channelRoot: string,
+): Promise<PostAvailabilityMap> {
+ try {
+ const raw = await readFile(postsAvailabilityPath(channelRoot), "utf8");
+ const parsed = JSON.parse(raw) as unknown;
+ if (!parsed || typeof parsed !== "object") return {};
+ const out: PostAvailabilityMap = {};
+ for (const [id, rec] of Object.entries(parsed as Record<string, unknown>)) {
+ const r = rec as Partial<PostAvailabilityRecord>;
+ if (!isPostAvailability(r?.availability)) continue;
+ if (typeof r.checkedAt !== "string") continue;
+ out[id] = {
+ availability: r.availability,
+ checkedAt: r.checkedAt,
+ ...(Array.isArray(r.history) ? { history: r.history } : {}),
+ };
+ }
+ return out;
+ } catch {
+ return {};
+ }
+}
+
+export async function writePostAvailability(
+ channelRoot: string,
+ map: PostAvailabilityMap,
+): Promise<void> {
+ const file = postsAvailabilityPath(channelRoot);
+ await mkdir(path.dirname(file), { recursive: true });
+ const tmp = `${file}.tmp-${process.pid}`;
+ await writeFile(tmp, JSON.stringify(map, null, 2) + "\n");
+ await rename(tmp, file);
+}
+
+// Fold new observations in, appending to history ONLY when the availability
+// actually changes. That keeps the file small and, more importantly, preserves
+// the moment a post was first seen deleted — which is the fact worth keeping.
+export function mergePostAvailability(
+ existing: PostAvailabilityMap,
+ observations: ReadonlyMap<string, PostAvailability>,
+ checkedAt: string,
+): { map: PostAvailabilityMap; newlyDeleted: string[]; changed: number } {
+ const map: PostAvailabilityMap = { ...existing };
+ const newlyDeleted: string[] = [];
+ let changed = 0;
+ for (const [id, availability] of observations) {
+ const prev = map[id];
+ if (prev?.availability === availability) {
+ // Same verdict — just refresh when we last looked.
+ map[id] = { ...prev, checkedAt };
+ continue;
+ }
+ changed++;
+ if (availability === "deleted" && prev?.availability !== "deleted") {
+ newlyDeleted.push(id);
+ }
+ map[id] = {
+ availability,
+ checkedAt,
+ history: [
+ ...(prev?.history ?? []),
+ ...(prev
+ ? [{ availability: prev.availability, at: prev.checkedAt }]
+ : []),
+ ],
+ };
+ }
+ return { map, newlyDeleted, changed };
+}
+
+// ---------------------------------------------------------------------------
// Fetch state sidecar
// ---------------------------------------------------------------------------
diff --git a/common/lib/posts.ts b/common/lib/posts.ts
@@ -67,8 +67,64 @@ export type Post = {
links: string[]; // expanded outbound urls
mediaCount?: number; // counted, not archived (v1)
engagement?: PostEngagement;
+ // Merged in at index time from the channel's availability sidecar (the same
+ // shape videos use: the stored record is the source of truth, the flag on
+ // the served record is derived). Absent = never checked, which is NOT the
+ // same as "still live".
+ isDeleted?: boolean;
+ availability?: PostAvailability;
+ availabilityCheckedAt?: string;
};
+// Whether an archived post is still live at its source. The post analogue of
+// video Availability — and arguably the more valuable half of the archive: a
+// deleted post is exactly the thing a commentary archive exists to preserve.
+//
+// Deliberately narrower than the video enum. A post has no members-only or
+// age-gated equivalent we can distinguish per-item; what we can tell apart is
+// "the post is gone", "the whole account is gone/protected" (which is not the
+// post's fault and may reverse), and "the check itself failed".
+export type PostAvailability =
+ | "available"
+ | "deleted"
+ | "account_unavailable"
+ | "error";
+
+export const POST_AVAILABILITY_VALUES: ReadonlyArray<PostAvailability> = [
+ "available",
+ "deleted",
+ "account_unavailable",
+ "error",
+];
+
+export function isPostAvailability(v: unknown): v is PostAvailability {
+ return (
+ typeof v === "string" &&
+ (POST_AVAILABILITY_VALUES as string[]).includes(v)
+ );
+}
+
+// Only `deleted` is treated as "gone for good". `account_unavailable` can
+// reverse (a protected account reopening, a suspension lifted) and `error` is
+// usually transient — neither is evidence the post itself was removed, so
+// neither is surfaced as a deletion.
+export function isPostGone(a: PostAvailability | null | undefined): boolean {
+ return a === "deleted";
+}
+
+export type PostAvailabilityRecord = {
+ availability: PostAvailability;
+ checkedAt: string;
+ // Appended only when the availability CHANGES, so the file stays small and
+ // the first-seen-deleted moment is preserved. Mirrors the video record.
+ history?: { availability: PostAvailability; at: string }[];
+};
+
+// postId -> record, stored once per channel. Videos keep a file per video dir,
+// but posts have no per-post directory (they live in month-sharded JSONL), so
+// the sidecar is per channel.
+export type PostAvailabilityMap = Record<string, PostAvailabilityRecord>;
+
export type PostPage = Post[];
export type ChannelPostsManifest = {
@@ -213,6 +269,11 @@ export function parsePost(raw: unknown): Post | null {
if (quoted) post.quoted = quoted;
const repostOf = parsePostRef(r.repostOf);
if (repostOf) post.repostOf = repostOf;
+ if (r.isDeleted === true) post.isDeleted = true;
+ if (isPostAvailability(r.availability)) post.availability = r.availability;
+ if (typeof r.availabilityCheckedAt === "string") {
+ post.availabilityCheckedAt = r.availabilityCheckedAt;
+ }
if (typeof r.mediaCount === "number" && Number.isFinite(r.mediaCount)) {
post.mediaCount = Math.max(0, Math.floor(r.mediaCount));
}
diff --git a/common/social/blueskyFetcher.ts b/common/social/blueskyFetcher.ts
@@ -18,6 +18,7 @@ import {
postPermalink,
uploadDateFromCreatedAt,
type Post,
+ type PostAvailability,
type PostRef,
} from "../lib/posts";
import {
@@ -483,4 +484,58 @@ function handleFromUrl(url: string): string | null {
}
}
+// app.bsky.feed.getPosts takes up to 25 AT-URIs and returns ONLY the posts
+// that still exist — so an id absent from the response is a deletion. That
+// makes the whole sweep a handful of unauthenticated batch calls, which is why
+// Bluesky can afford to check an entire channel routinely.
+const GET_POSTS_BATCH = 25;
+
+blueskyFetcher.checkAvailability = async ({ ids, handle, signal, onLog }) => {
+ const out = new Map<string, PostAvailability>();
+ if (ids.length === 0) return out;
+
+ // The AT-URI needs the account's DID, not its handle.
+ let did: string;
+ try {
+ did = await resolveHandle(handle, signal);
+ } catch (err) {
+ // The account itself is unreachable — that is NOT evidence any individual
+ // post was deleted, so report it as such rather than mass-marking.
+ onLog?.(`Could not resolve @${handle}: ${(err as Error).message}`);
+ for (const id of ids) out.set(id, "account_unavailable");
+ return out;
+ }
+
+ for (let i = 0; i < ids.length; i += GET_POSTS_BATCH) {
+ if (signal.aborted) break;
+ const batch = ids.slice(i, i + GET_POSTS_BATCH);
+ const params = new URLSearchParams();
+ for (const id of batch) {
+ params.append("uris", `at://${did}/app.bsky.feed.post/${id}`);
+ }
+ try {
+ const body = await getJson<{ posts?: BskyPostView[] }>(
+ `${APPVIEW}/xrpc/app.bsky.feed.getPosts?${params.toString()}`,
+ signal,
+ );
+ const alive = new Set(
+ (body.posts ?? [])
+ .map((p) => rkeyFromAtUri(p.uri))
+ .filter((r): r is string => !!r),
+ );
+ for (const id of batch) {
+ out.set(id, alive.has(id) ? "available" : "deleted");
+ }
+ } catch (err) {
+ // A failed batch must not read as 25 deletions.
+ onLog?.(`Availability batch failed: ${(err as Error).message}`);
+ for (const id of batch) out.set(id, "error");
+ }
+ }
+ onLog?.(
+ `Checked ${out.size} post(s): ${[...out.values()].filter((v) => v === "deleted").length} deleted.`,
+ );
+ return out;
+};
+
registerSocialFetcher(blueskyFetcher);
diff --git a/common/social/fetchers.ts b/common/social/fetchers.ts
@@ -13,7 +13,7 @@
// settings form consumes listTranscriptionApps() — so this module never lands
// in the client bundle.
-import type { Post, PostPlatform } from "../lib/posts";
+import type { Post, PostAvailability, PostPlatform } from "../lib/posts";
export type PostFetchInput = {
// The account's canonical URL as configured on the channel.
@@ -81,6 +81,16 @@ export type SocialFetcherFields = {
limit?: boolean;
};
+// Input for a deleted-post sweep. Ids are the platform-native post ids, and
+// `author` is the account handle they belong to (some platforms need it to
+// build a lookup key).
+export type PostAvailabilityInput = {
+ ids: ReadonlyArray<string>;
+ handle: string;
+ signal: AbortSignal;
+ onLog?: (line: string) => void;
+};
+
export type SocialFetcher = {
id: string;
label: string;
@@ -91,6 +101,13 @@ export type SocialFetcher = {
// Cheap metadata lookup for the channel form's "Fetch details" button.
probe(url: string, signal?: AbortSignal): Promise<SocialFetcherProbe>;
fetch(input: PostFetchInput): Promise<PostFetchResult>;
+ // Optional: report which of these posts are still live at the source. A
+ // fetcher without it simply cannot answer the question, and the sweep skips
+ // that channel rather than guessing — never reporting "deleted" from
+ // ignorance. Ids absent from the returned map are left unchanged.
+ checkAvailability?(
+ input: PostAvailabilityInput,
+ ): Promise<Map<string, PostAvailability>>;
};
// Populated by registerSocialFetcher() from each fetcher module. Indirection
diff --git a/common/social/xNitterFetcher.ts b/common/social/xNitterFetcher.ts
@@ -24,6 +24,7 @@ import {
postPermalink,
uploadDateFromCreatedAt,
type Post,
+ type PostAvailability,
type PostRef,
} from "../lib/posts";
import { importPlaywright } from "./playwrightRuntime";
@@ -381,4 +382,52 @@ function handleOf(url: string): string | null {
}
+// A deleted tweet's Nitter status page renders an error rather than a tweet,
+// so presence of `.main-tweet .tweet-content` is the liveness signal. One page
+// load per post makes this much heavier than Bluesky's batch lookup, so the
+// controller caps how many it checks per run.
+xNitterFetcher.checkAvailability = async ({ ids, handle, signal, onLog }) => {
+ const out = new Map<string, PostAvailability>();
+ if (ids.length === 0) return out;
+
+ const { chromium } = await importPlaywright();
+ const browser = await chromium.launch({ headless: true });
+ try {
+ const ctx = await browser.newContext({
+ userAgent:
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+ });
+ const page = await ctx.newPage();
+ const [instance] = nitterInstances();
+
+ for (const id of ids) {
+ if (signal.aborted) break;
+ try {
+ await page.goto(`${instance}/${handle}/status/${id}`, {
+ waitUntil: "domcontentloaded",
+ timeout: 30_000,
+ });
+ const verdict = (await page.evaluate(`(() => {
+ const hasTweet = !!document.querySelector('.main-tweet .tweet-content, .conversation .tweet-content');
+ const body = (document.body.textContent || '').toLowerCase();
+ if (hasTweet) return 'available';
+ if (/tweet not found|not found|deleted|no longer exists/.test(body)) return 'deleted';
+ if (/protected|suspended|does not exist|user not found/.test(body)) return 'account_unavailable';
+ return 'error';
+ })()`)) as PostAvailability;
+ out.set(id, verdict);
+ } catch {
+ // A failed load is not evidence of deletion.
+ out.set(id, "error");
+ }
+ }
+ } finally {
+ await browser.close().catch(() => {});
+ }
+ onLog?.(
+ `Checked ${out.size} post(s): ${[...out.values()].filter((v) => v === "deleted").length} deleted.`,
+ );
+ return out;
+};
+
registerSocialFetcher(xNitterFetcher);
diff --git a/editor/app/channels/[slug]/components/SocialChannelPanel.tsx b/editor/app/channels/[slug]/components/SocialChannelPanel.tsx
@@ -7,7 +7,11 @@
// which is exactly what the state sidecar records.
import { useState, useTransition } from "react";
-import { fetchPostsAction, setPostFetcherAction } from "../socialActions";
+import {
+ checkPostAvailabilityAction,
+ fetchPostsAction,
+ setPostFetcherAction,
+} from "../socialActions";
export type SocialChannelState = {
postCount: number;
@@ -22,6 +26,10 @@ export type SocialChannelState = {
// Which fetcher currently drives this channel, and every fetcher that could.
fetcherId: string;
availableFetchers: { id: string; label: string }[];
+ // Deleted-post sweep state (the posts analogue of video availability).
+ deletedCount: number;
+ checkedCount: number;
+ canCheckAvailability: boolean;
};
export function SocialChannelPanel({
@@ -49,6 +57,12 @@ export function SocialChannelPanel({
}
});
+ const checkDeleted = () =>
+ startTransition(async () => {
+ const res = await checkPostAvailabilityAction(slug);
+ if (res.ok) void res.stream.cancel();
+ });
+
const run = (full: boolean) =>
startTransition(async () => {
const res = await fetchPostsAction(slug, undefined, full);
@@ -74,6 +88,14 @@ export function SocialChannelPanel({
<Stat label="Archived posts" value={String(state.postCount)} />
<Stat label="Month shards" value={String(state.shards.length)} />
<Stat
+ label="Deleted"
+ value={
+ state.checkedCount === 0
+ ? "not checked"
+ : `${state.deletedCount} of ${state.checkedCount} checked`
+ }
+ />
+ <Stat
label="Last fetch"
value={
state.lastFetchedAt
@@ -137,6 +159,20 @@ export function SocialChannelPanel({
</button>
<button
type="button"
+ onClick={checkDeleted}
+ disabled={pending || !state.canCheckAvailability}
+ aria-label="check deleted posts"
+ title={
+ state.canCheckAvailability
+ ? "Check which archived posts have since been deleted at the source. Least-recently-checked first, so repeated runs sweep the whole archive."
+ : "This scraping method has no per-post liveness probe — switch method to enable the sweep."
+ }
+ className="rounded-md border border-border px-3 py-1 text-sm hover:bg-muted disabled:opacity-50"
+ >
+ Check deleted posts
+ </button>
+ <button
+ type="button"
onClick={() => run(true)}
disabled={pending}
aria-label="refetch full history"
diff --git a/editor/app/channels/[slug]/page.tsx b/editor/app/channels/[slug]/page.tsx
@@ -9,8 +9,10 @@ import { isSocialChannel } from "yt-dlp-transcript-common/lib/channelConfig";
import {
countPosts,
listPostShards,
+ readPostAvailability,
readPostFetchState,
} from "yt-dlp-transcript-common/lib/posts-server";
+import { isPostGone } from "yt-dlp-transcript-common/lib/posts";
import { resolveSocialFetcher } from "yt-dlp-transcript-common/social/fetchers";
import "yt-dlp-transcript-common/social/blueskyFetcher";
import "yt-dlp-transcript-common/social/xGalleryDlFetcher";
@@ -166,11 +168,13 @@ export default async function ChannelDetailPage({
// rail instead (see SocialChannelPanel).
if (isSocialChannel(config)) {
const channelRoot = path.join(paths.channelsDir, slug);
- const [postCount, shards, fetchState] = await Promise.all([
+ const [postCount, shards, fetchState, postAvailability] = await Promise.all([
countPosts(channelRoot),
listPostShards(channelRoot),
readPostFetchState(channelRoot),
+ readPostAvailability(channelRoot),
]);
+ const availabilityRecords = Object.values(postAvailability);
const fetcher = resolveSocialFetcher(config.postFetcher, config.url);
const socialRunningJobs = getRegistry()
.list()
@@ -205,6 +209,10 @@ export default async function ChannelDetailPage({
fetcherLabel: fetcher?.label ?? "no fetcher configured",
fetcherId: fetcher?.id ?? "",
availableFetchers: await listPostFetchersFor(config.platform ?? ""),
+ deletedCount: availabilityRecords.filter((r) => isPostGone(r.availability))
+ .length,
+ checkedCount: availabilityRecords.length,
+ canCheckAvailability: typeof fetcher?.checkAvailability === "function",
handle: config.socialHandle ?? slug,
accountUrl: config.url,
}}
diff --git a/editor/app/channels/[slug]/socialActions.ts b/editor/app/channels/[slug]/socialActions.ts
@@ -20,6 +20,10 @@ import {
import { isSocialChannel } from "yt-dlp-transcript-common/lib/channelConfig";
import { fetchPosts } from "yt-dlp-transcript-common/controller/fetchPosts";
import {
+ checkPostAvailability,
+ type CheckPostAvailabilityMode,
+} from "yt-dlp-transcript-common/controller/checkPostAvailability";
+import {
getSocialFetcher,
listSocialFetchers,
} from "yt-dlp-transcript-common/social/fetchers";
@@ -77,6 +81,45 @@ async function registerBuiltinSocialFetchers(): Promise<void> {
await import("yt-dlp-transcript-common/social/xNitterFetcher");
}
+// The posts analogue of the video availability check.
+export async function checkPostAvailabilityAction(
+ slug: string,
+ queueKey?: string,
+ mode?: CheckPostAvailabilityMode,
+ limit?: number,
+): Promise<StreamActionResult> {
+ const paths = getPaths();
+ const config = await readChannelConfig(paths, slug);
+ if (!config) return { ok: false, error: `No such channel: ${slug}` };
+ if (!isSocialChannel(config)) {
+ return { ok: false, error: `${slug} is not a social channel.` };
+ }
+ const key = resolveQueueKey(downloadQueueKey(config), queueKey);
+ return runManagedFunction({
+ kind: "check-post-availability",
+ queueKey: key,
+ paths,
+ channelSlug: slug,
+ spec: {
+ kind: "check-post-availability",
+ slug,
+ params: { queueKey, mode, limit },
+ },
+ fn: async (onLog, signal) => {
+ const result = await checkPostAvailability({
+ paths,
+ slug,
+ mode,
+ limit,
+ onLog,
+ signal,
+ });
+ revalidatePath(`/channels/${slug}`);
+ if (!result.ok) throw new Error(result.error ?? "Availability check failed");
+ },
+ });
+}
+
export async function fetchPostsAction(
slug: string,
queueKey?: string,
diff --git a/editor/app/jobs/jobReplayRegistry.ts b/editor/app/jobs/jobReplayRegistry.ts
@@ -40,7 +40,10 @@ import {
type DigestLaneChoice,
} from "../channels/[slug]/digestActions";
import { persistKeptAction } from "../channels/[slug]/persistActions";
-import { fetchPostsAction } from "../channels/[slug]/socialActions";
+import {
+ checkPostAvailabilityAction,
+ fetchPostsAction,
+} from "../channels/[slug]/socialActions";
import {
redownloadIncompleteBucketAction,
redownloadShortAudioBucketAction,
@@ -183,6 +186,15 @@ export const JOB_REPLAY_HANDLERS: Record<string, ReplayHandler> = {
const { p, queueKey } = params(spec);
return syncAction(spec.slug, queueKey, bool(p.fullSweep));
},
+ "check-post-availability": (spec) => {
+ const { p, queueKey } = params(spec);
+ return checkPostAvailabilityAction(
+ spec.slug,
+ queueKey,
+ str(p.mode) as "stale" | "unchecked" | "all" | undefined,
+ num(p.limit),
+ );
+ },
"fetch-posts": (spec) => {
const { p, queueKey } = params(spec);
return fetchPostsAction(spec.slug, queueKey, bool(p.full), num(p.limit));
diff --git a/export/e2e/fixtures/data.ts b/export/e2e/fixtures/data.ts
@@ -10,6 +10,7 @@ export const POST_CHANNEL_SLUG = "test-social";
export const POST_ROOT_ID = "post-root-1";
export const POST_REPLY_ID = "post-reply-1";
export const POST_OTHER_ID = "post-other-1";
+export const POST_DELETED_ID = "post-deleted-1";
export const VIDEO_TRANSCRIPT_ONLY = "vid-transcript-only";
export const VIDEO_CHAT_SMALL = "vid-chat-small";
@@ -454,11 +455,11 @@ export function postsManifest() {
{
name: POST_CHANNEL,
slug: POST_CHANNEL_SLUG,
- postCount: 3,
+ postCount: 4,
platform: "bluesky" as const,
},
],
- totalCount: 3,
+ totalCount: 4,
generatedAt: new Date().toISOString(),
};
}
@@ -474,6 +475,7 @@ export function channelPostsManifest() {
[POST_ROOT_ID]: 0,
[POST_REPLY_ID]: 0,
[POST_OTHER_ID]: 0,
+ [POST_DELETED_ID]: 0,
},
};
}
@@ -490,6 +492,12 @@ export function postsPage() {
replyTo: { platform: "bluesky", id: POST_ROOT_ID },
createdAt: "2026-02-03T11:00:00.000Z",
}),
+ makePost(POST_DELETED_ID, "a deleted gamma post", {
+ isDeleted: true,
+ availability: "deleted",
+ availabilityCheckedAt: "2026-02-04T00:00:00.000Z",
+ createdAt: "2026-02-03T12:00:00.000Z",
+ }),
makePost(POST_OTHER_ID, "an unrelated omega post", {
createdAt: "2026-02-02T09:00:00.000Z",
uploadDate: "20260202",
diff --git a/export/e2e/posts-search.spec.ts b/export/e2e/posts-search.spec.ts
@@ -2,6 +2,7 @@ import { expect, test, type Page } from "@playwright/test";
import {
CHANNEL_SLUG,
POST_CHANNEL_SLUG,
+ POST_DELETED_ID,
POST_OTHER_ID,
POST_REPLY_ID,
POST_ROOT_ID,
@@ -185,3 +186,28 @@ test.describe("social-post corpus — search", () => {
]);
});
});
+
+test("a deleted post is flagged in results and in the modal", async ({ page }) => {
+ // Defined outside the describe above, so it installs its own route mocks.
+ await installRoutes(page);
+ // Preserving posts that no longer exist upstream is the point of the
+ // archive, so the deletion has to be visible rather than silent.
+ const tree: SGroup = {
+ k: "g",
+ o: "AND",
+ // A term unique to the deleted fixture, so the other specs' expected
+ // result sets stay untouched.
+ c: [{ k: "l", q: "gamma", s: "posts" }],
+ };
+ await page.goto(`/?qt=${qt(tree)}`);
+ const deletedSlug = `${POST_CHANNEL_SLUG}/${POST_DELETED_ID}`;
+ await expectResultSlugs(page, [deletedSlug]);
+
+ const card = page.locator(`[data-card-header][data-result-slug="${deletedSlug}"]`);
+ await expect(card.getByText("Deleted", { exact: true })).toBeVisible();
+ await card.locator("[data-card-open]").click();
+ const modal = page.locator("[data-post-modal]");
+ await expect(modal).toBeVisible();
+ await expect(modal.getByText("deleted", { exact: true }).first()).toBeVisible();
+ await expect(modal.getByRole("link", { name: /Original \(deleted\)/ })).toBeVisible();
+});
