commit 427b6f218041766fab69589d9d906f5b22c93ef8
parent b3b47e28964866e4e3189b8354a997740a7cd9a1
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 6 Oct 2026 09:07:41 -0400
wayback refresh: a lane pick whose outcome sidecar is newer has finished
A transcription or download pick is released once the record's transcribe-outcome.json /
download-outcome.json was written at or after the pick, so a finished pick does not hold
its record for the whole six-hour window.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
2 files changed, 27 insertions(+), 5 deletions(-)
diff --git a/common/controller/waybackRefresh.test.ts b/common/controller/waybackRefresh.test.ts
@@ -181,10 +181,18 @@ test("a record a live job names is held; a dead writer's job is not", async () =
await writeFile(
paths.autoQueueStateFile,
JSON.stringify({
- transcription: { picks: [{ at, leafId: "x", videoId: `${JW_B}-12345678.mp4`, channelSlug: SLUG }] },
+ transcription: {
+ picks: [
+ { at, leafId: "x", videoId: `${JW_B}-12345678.mp4`, channelSlug: SLUG },
+ // Finished: its outcome sidecar is newer than the pick (below).
+ { at: at - 1000, leafId: "x", videoId: "watch", channelSlug: SLUG },
+ ],
+ },
+ // Too old to be in flight.
download: { picks: [{ at: at - 7 * 3600 * 1000, leafId: "x", videoId: "watch", channelSlug: SLUG }] },
}),
);
+ await writeFile(path.join(dataDir, "watch", "transcribe-outcome.json"), "{}");
const r = await refreshWaybackRecords({ slug: SLUG, paths, titles, now: () => new Date(at + 60_000) });
assert.equal(r.records.find((x) => x.from === `${JW_B}-12345678.mp4`)!.rename, "held");
const a = r.records.find((x) => x.from === `${JW_A}-12345678.mp4`)!;
diff --git a/common/controller/waybackRefresh.ts b/common/controller/waybackRefresh.ts
@@ -24,7 +24,7 @@
// No network.
import path from "node:path";
-import { readdir, readFile } from "node:fs/promises";
+import { readdir, readFile, stat } from "node:fs/promises";
import { getPaths, type Paths } from "../lib/paths";
import { readJsonFile } from "../lib/jsonFile-server";
import { assertChannelTextReadable, readRelocationMarker } from "../lib/channelMedia";
@@ -129,11 +129,20 @@ async function liveJobs(
const read = await readJsonFile(paths.autoQueueStateFile);
const state = read.ok && read.value && typeof read.value === "object" ? (read.value as Record<string, unknown>) : {};
const since = now - LANE_PICK_HOLD_MS;
- for (const kind of Object.values(state)) {
- const picks = (kind as { picks?: unknown } | null)?.picks;
+ for (const [kind, kindState] of Object.entries(state)) {
+ const picks = (kindState as { picks?: unknown } | null)?.picks;
if (!Array.isArray(picks)) continue;
for (const pick of picks.slice(0, LANE_PICKS_HELD) as Partial<AutoQueuePick>[]) {
- if (pick?.channelSlug === slug && typeof pick.videoId === "string" && (pick.at ?? 0) >= since) ids.add(pick.videoId);
+ if (pick?.channelSlug !== slug || typeof pick.videoId !== "string") continue;
+ const at = pick.at ?? 0;
+ if (at < since) continue;
+ // A pick whose outcome sidecar was written since is finished.
+ const outcome = LANE_OUTCOME[kind];
+ if (outcome) {
+ const done = await stat(path.join(paths.channelsDir, slug, "data", pick.videoId, outcome)).catch(() => null);
+ if (done && done.mtimeMs >= at) continue;
+ }
+ ids.add(pick.videoId);
}
}
}
@@ -143,6 +152,11 @@ async function liveJobs(
// How many of a lane's newest picks may still be in flight, and for how long.
const LANE_PICKS_HELD = 4;
const LANE_PICK_HOLD_MS = 6 * 60 * 60 * 1000;
+// The sidecar a lane's run writes when it ends, by lane kind.
+const LANE_OUTCOME: Record<string, string> = {
+ transcription: "transcribe-outcome.json",
+ download: "download-outcome.json",
+};
// A title that is not one: absent, or the file's own name (what yt-dlp's
// generic extractor titles a raw file with).
