archilyzer

commit 8c2c9a212933d794b9cf30dba1155ca4c9734609
parent 4bfe6ef1750b406455f15496bf4583a2dc52e117
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date:   Mon,  5 Oct 2026 15:26:36 -0400

sources: an archive.org import asks nothing while archive.org is held or cooling down; a rate limit backs the platform off

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Diffstat:
M
common/controller/archiveOrgImport.test.ts
 | 
1
+
M
common/controller/archiveOrgImport.ts
 | 
4
++++
M
editor/app/channels/[slug]/pipelineActions.ts
 | 
23
+++++++++++++++++++++++

3 files changed, 28 insertions(+), 0 deletions(-)
diff --git a/common/controller/archiveOrgImport.test.ts b/common/controller/archiveOrgImport.test.ts
@@ -166,6 +166,7 @@ test("bulk: a rate-limited file stops the batch at once", async () => {
     },
   });
   assert.equal(n, 1);
+  assert.equal(result.rateLimited, true);
   assert.match(result.stopped ?? "", /rate-limited/);
 });
 
diff --git a/common/controller/archiveOrgImport.ts b/common/controller/archiveOrgImport.ts
@@ -189,6 +189,9 @@ export type ArchiveOrgImportResult = {
   unknown: string[];
   // Why the batch ended before its last file, when it did.
   stopped?: string;
+  // archive.org refused a download as a rate limit: the caller backs the
+  // platform off (jobs/downloadBackoff.ts).
+  rateLimited?: boolean;
 };
 
 export type ArchiveOrgImportDeps = {
@@ -335,6 +338,7 @@ export async function runArchiveOrgImport(opts: {
     result.failed.push({ file: entry.file, error: why });
     log(`  failed: ${why}\n`);
     if (rec?.failureClass === "rate_limit") {
+      result.rateLimited = true;
       result.stopped = "archive.org rate-limited the download; stopping (re-run later — files on disk are skipped)";
       log(`${result.stopped}.\n`);
       break;
diff --git a/editor/app/channels/[slug]/pipelineActions.ts b/editor/app/channels/[slug]/pipelineActions.ts
@@ -520,6 +520,8 @@ export async function importVideoAction(
   const archiveOrg = detectPlatform(videoUrl) === "archiveorg";
   let downloadConfig = channelConfig;
   if (archiveOrg) {
+    const refused = await archiveOrgRefusal(paths, "The import");
+    if (refused) return { ok: false, info: true, error: refused };
     const resolved = await resolveArchiveOrgImportUrl(videoUrl);
     if (!resolved.ok) return { ok: false, error: resolved.error };
     videoUrl = resolved.url;
@@ -600,6 +602,18 @@ export async function importVideoAction(
   });
 }
 
+// archive.org held, or in a rate-limit cooldown: a sentence, else null. An
+// import asks archive.org nothing while it has asked us to wait.
+async function archiveOrgRefusal(paths: Paths, what: string): Promise<string | null> {
+  const held = await heldPlatformRefusal("archiveorg", what, paths);
+  if (held) return held;
+  const remainingMs = await platformCooldownRemainingMs("archiveorg", paths);
+  if (remainingMs > 0) {
+    return `archive.org is in a rate-limit cooldown (${Math.ceil(remainingMs / 1000)}s remaining). ${what} can run once it lapses.`;
+  }
+  return null;
+}
+
 // IMPORT CHOSEN FILES OF ONE archive.org ITEM (`pnpm ops import-archive-org`):
 // one job on archive.org's own queue that imports the files one at a time,
 // with a jittered pause between them, skipping any already downloaded, and
@@ -633,6 +647,8 @@ export async function importArchiveOrgAction(
   if (!opts.dryRun) {
     const err = await lowDiskError(paths, slug);
     if (err) return err;
+    const refused = await archiveOrgRefusal(paths, "The archive.org import");
+    if (refused) return { ok: false, info: true, error: refused };
   }
   const downloadConfig =
     channelConfig.platform === "archiveorg"
@@ -659,6 +675,13 @@ export async function importArchiveOrgAction(
         },
       });
       safeRevalidate([`/channels/${slug}`, "/channels"]);
+      // The platform's shared pacing state learns what archive.org said, so
+      // the next import (and any other archive.org job) backs off or settles.
+      if (result.rateLimited) {
+        await recordDownloadBackoff("archiveorg", paths, "rate_limit");
+      } else if (result.imported.length > 0 && result.failed.length === 0) {
+        await recordPlatformClean("archiveorg", paths);
+      }
       if (result.failed.length > 0 && result.imported.length === 0 && !opts.dryRun) {
         throw new Error(result.stopped ?? `${result.failed.length} file(s) failed`);
       }