commit cf3c0abab3f35edc741d81f805482e96ea2932b4
parent d99651df6ad7549dcc3124111542f321224d0b48
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 25 Sep 2026 18:43:14 -0400
common: /jobs flags a stall by quiet time, not age
A running slot read `possibly-stalled` once its record had held the slot
10 min with no task in flight. A metadata scan never has a task in flight,
so a healthy one at ~10 videos/min read STUCK · POSSIBLY-STALLED on
2026-09-25.
JobRecord gains `progressAt`, stamped when the job moves: `noteProgress`
(the setProgress every runManagedFunction job gets) stamps it only when
the snapshot's numbers changed, and `recordTaskDuration` stamps it when a
sub-operation finishes. reconcileSlots measures quiet time from the later
of the start and that stamp. Stamped at the source rather than diffed in a
view-side map, so the first /jobs render after an hour away is as right as
the hundredth. jobRows.test.ts +2: an hour of advancing progress is never
stuck; progress frozen past 10 min is (both fail on the old rule).
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
4 files changed, 90 insertions(+), 4 deletions(-)
diff --git a/common/jobs/registry.ts b/common/jobs/registry.ts
@@ -129,8 +129,34 @@ export type JobRecord = {
// task-count average cannot express it — the corpus is 77k videos and 77k
// audio-hours, and those distribute completely differently.
completedTaskAudioSeconds?: number;
+ // When the job last MOVED (epoch ms): its progress snapshot changed
+ // (`noteProgress`) or a sub-operation finished (`recordTaskDuration`).
+ // Undefined until the first. What the /jobs stall heuristic measures quiet
+ // time from — a slow scan at 10 videos/min that holds a slot for an hour with
+ // no in-flight tasks is still moving (release 9, B4a).
+ progressAt?: number;
};
+// Store a progress snapshot on the record, stamping `progressAt` only when the
+// numbers changed — a runner that re-reports the same count is not moving.
+export function noteProgress(
+ record: JobRecord,
+ snap: JobProgress,
+ now: number = Date.now(),
+): void {
+ const prev = record.progress;
+ if (
+ !prev ||
+ prev.current !== snap.current ||
+ prev.target !== snap.target ||
+ prev.initial !== snap.initial ||
+ prev.remainingAudioSeconds !== snap.remainingAudioSeconds
+ ) {
+ record.progressAt = now;
+ }
+ record.progress = snap;
+}
+
export type QueueSnapshot = {
name: string;
running?: JobRecord;
@@ -360,6 +386,7 @@ class JobRegistry {
if (!job || durationMs < 0) return;
job.completedTaskCount = (job.completedTaskCount ?? 0) + 1;
job.completedTaskMs = (job.completedTaskMs ?? 0) + durationMs;
+ job.progressAt = Date.now();
if (typeof audioSeconds === "number" && audioSeconds > 0) {
job.completedTaskAudioSeconds =
(job.completedTaskAudioSeconds ?? 0) + audioSeconds;
diff --git a/common/jobs/streamCommand.ts b/common/jobs/streamCommand.ts
@@ -5,6 +5,7 @@ import { execa } from "execa";
import {
getRegistry,
newJobId,
+ noteProgress,
type JobProgress,
type JobRecord,
type JobStatus,
@@ -342,7 +343,7 @@ export async function runManagedFunction(
};
const setProgress = (snap: JobProgress) => {
- record.progress = snap;
+ noteProgress(record, snap);
};
const ctx: JobRunContext = {
diff --git a/common/views/jobRows.test.ts b/common/views/jobRows.test.ts
@@ -1,6 +1,6 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { newJobId, type JobRecord } from "../jobs/registry";
+import { newJobId, noteProgress, type JobRecord } from "../jobs/registry";
import type { JobListEntry } from "../jobs/listJobs";
import { createScheduler } from "../jobs/scheduler";
import {
@@ -399,3 +399,55 @@ test("fromInFlight: a note replaces the detail and drops the synthetic video id"
assert.equal(row.videoId, undefined);
assert.equal(row.id, "download:metadata-scan alpha");
});
+
+// THE STALL LABEL MEASURES QUIET, NOT AGE (release 9, B4a). A metadata scan
+// never has a task in flight and advances its count every few seconds; on
+// 2026-09-25 one at ~10 videos/min read STUCK · POSSIBLY-STALLED after 10 min.
+test("reconcileSlots: a long-running idle job whose progress advances is never stuck", () => {
+ const { scheduler, ids } = scheduledQueue();
+ const id = ids[0];
+ const start = 10_000_000;
+ const r = rec({
+ id,
+ queueKey: "qFacts",
+ kind: "metadata-scan",
+ status: "running",
+ startedAt: start,
+ tasks: [],
+ });
+ // One scanned video every 6 s for an hour, checked each minute.
+ for (let t = 0; t <= 60 * 60 * 1000; t += 6_000) {
+ noteProgress(
+ r,
+ { metric: "scans", initial: 0, target: 1_382, current: t / 6_000 },
+ start + t,
+ );
+ if (t % 60_000 === 0) {
+ const [fact] = reconcileSlots(scheduler.queues(), () => r, start + t);
+ assert.equal(fact.stuck, undefined, `flagged at ${t / 60_000} min`);
+ }
+ }
+});
+
+test("reconcileSlots: idle tasks and progress frozen past 10 min is possibly-stalled", () => {
+ const { scheduler, ids } = scheduledQueue();
+ const id = ids[0];
+ const start = 10_000_000;
+ const r = rec({
+ id,
+ queueKey: "qFacts",
+ status: "running",
+ startedAt: start,
+ tasks: [],
+ });
+ noteProgress(r, { metric: "scans", initial: 0, target: 100, current: 40 }, start + 20 * 60_000);
+ // The same count re-reported is not a move.
+ noteProgress(r, { metric: "scans", initial: 0, target: 100, current: 40 }, start + 25 * 60_000);
+ assert.equal(r.progressAt, start + 20 * 60_000);
+ const at = (min: number) =>
+ reconcileSlots(scheduler.queues(), () => r, start + min * 60_000)[0];
+ assert.equal(at(29).stuck, undefined);
+ const late = at(31);
+ assert.equal(late.stuck?.reason, "possibly-stalled");
+ assert.equal(late.heal, false);
+});
diff --git a/common/views/jobRows.ts b/common/views/jobRows.ts
@@ -23,7 +23,11 @@ import type { JobRowView, StuckReason } from "./jobRowView";
export const RECENT_MS = 30_000;
// A running slot is treated as "possibly stalled" (the soft case) once a
-// running/draining record has held its slot this long with no in-flight tasks.
+// running/draining record has held its slot this long with no in-flight tasks
+// AND no progress: the clock runs from the later of its start and its last move
+// (`JobRecord.progressAt`). Age alone flagged a healthy metadata scan at ~10
+// videos/min as STUCK on 2026-09-25 — it never has a task in flight, and it
+// advances its count every few seconds (release 9, B4a).
// Distinct from the hard cases (terminal-but-holding-slot / record-evicted),
// which are always safe to auto-heal; the soft case is only surfaced/force-
// releasable, never auto-healed.
@@ -78,8 +82,10 @@ export function reconcileSlots(
heal = true;
} else {
const ageMs = now - (rec.startedAt ?? rec.queuedAt);
+ const quietMs =
+ now - Math.max(rec.startedAt ?? rec.queuedAt, rec.progressAt ?? 0);
const idle = !rec.tasks || rec.tasks.length === 0;
- if (idle && ageMs > STUCK_AGE_MS) {
+ if (idle && quietMs > STUCK_AGE_MS) {
// Soft case: a live running/draining record sitting idle far too
// long. Force-releasable, but deliberately NOT auto-healed.
stuck = { reason: "possibly-stalled", heldMs: ageMs };
