archilyzer

commit b33f845548f1e021de48bcb617b9b8c399924034
parent df0ab16ec786e040a1b4eefbad20fa1f3a5fab80
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date:   Sun,  4 Oct 2026 22:04:12 -0400

common: fetchPosts refuses an older walk over an account that shows no posts unless forced; takes the job's drain; the CLI takes --force

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Diffstat:
M
common/bin/archilyzer.ts
 | 
2
+-
M
common/bin/fetch-posts.ts
 | 
9
++++++---
M
common/controller/fetchPosts.ts
 | 
43
+++++++++++++++++++++++++++++++++++++++++++

3 files changed, 50 insertions(+), 4 deletions(-)
diff --git a/common/bin/archilyzer.ts b/common/bin/archilyzer.ts
@@ -252,7 +252,7 @@ export const COMMANDS: Command[] = [
   script(["duplicates"], "duplicate-shorts.ts",
     "[--threshold N] [--all-durations] [--blocking title|duration|both] [--near F] [--tolerance N] …  on-demand duplicate detection (after index + stats)", 8192),
   script(["posts", "fetch"], "fetch-posts.ts",
-    "--slug <channel> [--full | --older [--floor YYYY-MM-DD]] [--limit N]  fetch a social channel's posts into its posts corpus (--older: walk back below the oldest archived post)"),
+    "--slug <channel> [--full | --older [--floor YYYY-MM-DD] [--force]] [--limit N]  fetch a social channel's posts into its posts corpus (--older: walk back below the oldest archived post)"),
   script(["posts", "check"], "check-post-availability.ts",
     "--slug <channel> [--mode stale|unchecked|all] [--limit N]  which archived posts were deleted at the source"),
   script(["diarize", "backfill"], "diarize-backfill.ts",
diff --git a/common/bin/fetch-posts.ts b/common/bin/fetch-posts.ts
@@ -2,7 +2,7 @@
 // Fetch social posts for one channel into its on-disk posts corpus.
 //
 //   pnpm --filter yt-dlp-transcript-common exec tsx bin/fetch-posts.ts \
-//     --slug <channel-slug> [--full | --older [--floor YYYY-MM-DD]] [--limit N]
+//     --slug <channel-slug> [--full | --older [--floor YYYY-MM-DD] [--force]] [--limit N]
 //
 // --full re-walks the account's whole history instead of stopping at the
 // stored watermark. New posts are still deduped against the posts-archive, so
@@ -11,7 +11,9 @@
 // --older walks the account's history backwards from the oldest archived post
 // (X: search windows), below what the timeline reaches; --floor is the date it
 // stops at. Its position is saved apart from the timeline's, so a later run
-// continues it and a normal fetch is unaffected.
+// continues it and a normal fetch is unaffected. An account that shows no
+// posts (nothing archived, and the last timeline fetch read none) is refused
+// a search walk unless --force.
 
 import { getPaths } from "../lib/paths";
 import { getSettings } from "../lib/settings";
@@ -22,7 +24,7 @@ const flags = parseFlags(process.argv.slice(2));
 const slug = flags.slug;
 if (!slug) {
   console.error(
-    "Usage: fetch-posts.ts --slug <channel-slug> [--full | --older [--floor YYYY-MM-DD]] [--limit N]",
+    "Usage: fetch-posts.ts --slug <channel-slug> [--full | --older [--floor YYYY-MM-DD] [--force]] [--limit N]",
   );
   process.exit(2);
 }
@@ -40,6 +42,7 @@ fetchPosts({
   full: flags.full === "true",
   older: flags.older === "true",
   floor: flags.floor,
+  force: flags.force === "true",
   limit,
   onLog: (line) => console.log(line),
 })
diff --git a/common/controller/fetchPosts.ts b/common/controller/fetchPosts.ts
@@ -78,9 +78,15 @@ export type FetchPostsOptions = {
   floor?: string;
   // The older walk's pause between two windows. Default: the fetcher's own.
   olderWindowPauseMs?: number;
+  // Walk older posts even on an account that shows none (see
+  // emptyAccountOlderProblem). Only with `older`.
+  force?: boolean;
   limit?: number;
   onLog?: (line: string) => void;
   signal?: AbortSignal;
+  // The job's Drain: the fetch stops at its next resume point and keeps it,
+  // and the run ends as a success, not a failure.
+  drain?: AbortSignal;
 };
 
 // The refusal for both walks at once, shared with the server action so the
@@ -98,6 +104,25 @@ export function olderPostsProblem(
     : `${fetcher.label} cannot fetch older posts.`;
 }
 
+// An older walk over an account that shows no posts is refused unless forced:
+// nothing is archived, and the last timeline fetch finished cleanly having read
+// nothing. Searching such an account window after window is nothing but empty
+// searches. A timeline run that stopped part-way, failed or needed a login
+// says nothing about the account, so it is not grounds to refuse.
+export const EMPTY_ACCOUNT_OLDER_REFUSAL =
+  "The account shows no posts: nothing is archived and the last timeline fetch read none. " +
+  'A search walk would only repeat empty searches — run it with "force" to walk anyway.';
+
+export function emptyAccountOlderProblem(
+  archived: number,
+  state: Pick<PostFetchState, "lastFetchedCount" | "cursor" | "lastError" | "needsCookies"> | null,
+): string | null {
+  if (archived > 0 || !state) return null;
+  if (state.lastFetchedCount !== 0) return null;
+  if (state.cursor || state.lastError || state.needsCookies) return null;
+  return EMPTY_ACCOUNT_OLDER_REFUSAL;
+}
+
 export type FetchPostsResult = {
   ok: boolean;
   written: number;
@@ -105,6 +130,8 @@ export type FetchPostsResult = {
   complete: boolean;
   error?: string;
   needsCookies?: boolean;
+  // Stopped on the job's Drain, at a resume point the next run continues from.
+  drained?: boolean;
 };
 
 export async function fetchPosts(
@@ -165,6 +192,11 @@ export async function fetchPosts(
     };
   }
 
+  if (opts.drain?.aborted) {
+    log("Drained before the fetch started; nothing was read.");
+    return { ok: true, written: 0, skipped: 0, complete: false, drained: true };
+  }
+
   const seenIds = await readSeenPostIds(channelRoot);
   const priorState = await readPostFetchState(channelRoot);
 
@@ -270,6 +302,7 @@ export async function fetchPosts(
       stopAtKnown: !opts.full,
       onCheckpoint,
       signal: effectiveSignal,
+      drain: opts.drain,
       onLog: log,
     });
   } catch (err) {
@@ -332,6 +365,7 @@ export async function fetchPosts(
     complete: result.complete,
     needsCookies: result.needsCookies,
     error: result.error,
+    ...(result.drained ? { drained: true } : {}),
   };
 }
 
@@ -368,6 +402,13 @@ async function fetchOlderPosts(ctx: {
     return { ok: true, written: 0, skipped: 0, complete: true };
   }
 
+  const empty = emptyAccountOlderProblem(ctx.seenIds.size, priorState);
+  if (empty && !opts.force) {
+    log(empty);
+    return { ok: false, written: 0, skipped: 0, complete: false, error: empty };
+  }
+  if (empty) log("[warn] The account shows no posts; walking anyway, as forced.");
+
   const floor = opts.floor ?? prior?.floor;
   let accountCreatedAt = prior?.accountCreatedAt;
   const startedAt = new Date().toISOString();
@@ -449,6 +490,7 @@ async function fetchOlderPosts(ctx: {
       browserCookies: ctx.browserCookies,
       limit: opts.limit,
       signal: opts.signal ?? controller.signal,
+      drain: opts.drain,
       onLog: log,
       position,
       floor,
@@ -498,5 +540,6 @@ async function fetchOlderPosts(ctx: {
     complete: result.complete,
     needsCookies: result.needsCookies,
     error: result.error,
+    ...(result.drained ? { drained: true } : {}),
   };
 }