archilyzer

commit 7aff8253f112e67eaa9aed9ef7d72759c46bc5a8
parent 5a14b2addb79a5c6ea1535e1b00cc3deccff6172
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date:   Tue,  6 Oct 2026 09:04:31 -0400

wayback refresh: an auto-queue lane's newest picks of the channel are held too

The lanes are jobs of no channel; what each is on is its newest pick in
.auto-queue/state.json. The newest four recent (6 h) picks of the channel are held
like a record a job names.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Diffstat:
M
common/controller/waybackRefresh.test.ts
 | 
16
++++++++++++++--
M
common/controller/waybackRefresh.ts
 | 
27
++++++++++++++++++++++++---

2 files changed, 38 insertions(+), 5 deletions(-)
diff --git a/common/controller/waybackRefresh.test.ts b/common/controller/waybackRefresh.test.ts
@@ -174,13 +174,25 @@ test("a record a live job names is held; a dead writer's job is not", async () =
       );
     await meta("01AAAAAAAAAAAAAAAAAAAAAAAA", `${JW_A}-12345678.mp4`, process.ppid);
     await meta("01BBBBBBBBBBBBBBBBBBBBBBBB", "watch", 2 ** 22 + 12345);
-    const r = await refreshWaybackRecords({ slug: SLUG, paths, titles });
+    // An auto-queue lane's newest pick is in flight; an old one is not.
+    paths.autoQueueStateFile = path.join(paths.transcriptsDir, ".auto-queue", "state.json");
+    await mkdir(path.dirname(paths.autoQueueStateFile), { recursive: true });
+    const at = Date.parse("2026-02-02T00:00:00.000Z");
+    await writeFile(
+      paths.autoQueueStateFile,
+      JSON.stringify({
+        transcription: { picks: [{ at, leafId: "x", videoId: `${JW_B}-12345678.mp4`, channelSlug: SLUG }] },
+        download: { picks: [{ at: at - 7 * 3600 * 1000, leafId: "x", videoId: "watch", channelSlug: SLUG }] },
+      }),
+    );
+    const r = await refreshWaybackRecords({ slug: SLUG, paths, titles, now: () => new Date(at + 60_000) });
+    assert.equal(r.records.find((x) => x.from === `${JW_B}-12345678.mp4`)!.rename, "held");
     const a = r.records.find((x) => x.from === `${JW_A}-12345678.mp4`)!;
     assert.equal(a.rename, "held");
     assert.equal(a.to, a.from);
     assert.deepEqual(a.changes, {});
     assert.equal(r.records.find((x) => x.from === "watch")!.rename, "renamed");
-    assert.deepEqual((await readdir(dataDir)).sort(), [`${JW_A}-12345678.mp4`, JW_B, "Plain12345a", YT].sort());
+    assert.deepEqual((await readdir(dataDir)).sort(), [`${JW_A}-12345678.mp4`, `${JW_B}-12345678.mp4`, "Plain12345a", YT].sort());
   });
 });
 
diff --git a/common/controller/waybackRefresh.ts b/common/controller/waybackRefresh.ts
@@ -19,8 +19,8 @@
 //                        `wayback-provenance`
 //
 // A record a live job names (a `.jobs/` meta, queued or running, whose writer
-// is alive) is skipped and reported; a live job over the whole channel holds
-// every record. Only what differs is written, so a second run writes nothing.
+// is alive, or one of an auto-queue lane's newest picks) is skipped and
+// reported; a live job over the whole channel holds every record. Only what differs is written, so a second run writes nothing.
 // No network.
 
 import path from "node:path";
@@ -38,6 +38,7 @@ import { reconcileVideoDirs } from "./reconcileVideoDirs";
 import { loadRoster, renameRosterEntries, writeRoster } from "./rosterStore";
 import { writerIsGone } from "../jobs/bootQueuedJobs";
 import type { JobMeta } from "../jobs/jobMeta";
+import type { AutoQueuePick } from "../jobs/autoQueueState";
 
 // What the operator found for a record: its title and the day it is of.
 export type WaybackTitle = { title?: string; upload_date?: string };
@@ -105,6 +106,7 @@ async function captureUrlOf(videoDir: string, info: Info | null): Promise<string
 async function liveJobs(
   paths: Paths,
   slug: string,
+  now: number,
 ): Promise<{ ids: Set<string>; channelWide: string[] }> {
   const ids = new Set<string>();
   const channelWide: string[] = [];
@@ -120,9 +122,28 @@ async function liveJobs(
     if (meta.videoId) ids.add(meta.videoId);
     else channelWide.push(`${meta.kind} ${meta.id}`);
   }
+  // The auto-queue lanes are jobs of no channel, and the record each is on
+  // is its newest pick (jobs/autoQueueState.ts). A lane runs a few workers, so
+  // the newest few recent picks of this channel are held.
+  if (paths.autoQueueStateFile) {
+    const read = await readJsonFile(paths.autoQueueStateFile);
+    const state = read.ok && read.value && typeof read.value === "object" ? (read.value as Record<string, unknown>) : {};
+    const since = now - LANE_PICK_HOLD_MS;
+    for (const kind of Object.values(state)) {
+      const picks = (kind as { picks?: unknown } | null)?.picks;
+      if (!Array.isArray(picks)) continue;
+      for (const pick of picks.slice(0, LANE_PICKS_HELD) as Partial<AutoQueuePick>[]) {
+        if (pick?.channelSlug === slug && typeof pick.videoId === "string" && (pick.at ?? 0) >= since) ids.add(pick.videoId);
+      }
+    }
+  }
   return { ids, channelWide };
 }
 
+// How many of a lane's newest picks may still be in flight, and for how long.
+const LANE_PICKS_HELD = 4;
+const LANE_PICK_HOLD_MS = 6 * 60 * 60 * 1000;
+
 // A title that is not one: absent, or the file's own name (what yt-dlp's
 // generic extractor titles a raw file with).
 function hasRealTitle(info: Info, names: string[]): boolean {
@@ -171,7 +192,7 @@ export async function refreshWaybackRecords(opts: {
   const channelDir = path.join(paths.channelsDir, opts.slug);
   const dataDir = path.join(channelDir, "data");
   const result: WaybackRefreshResult = { records: [], unmatchedTitles: [], failed: [] };
-  const jobs = await liveJobs(paths, opts.slug);
+  const jobs = await liveJobs(paths, opts.slug, (opts.now?.() ?? new Date()).getTime());
 
   // ─── Find the Wayback records ───
   type Found = { rec: WaybackRecordResult; info: Info | null; canonical: string | null };