commit ec15995d9424f236b933a1105b4342cb48aafccf
parent 33c9ae84365c6066ab879a7a4674066520a86653
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 1 Oct 2026 23:15:39 -0400
editor: both ops refresh-report forms answer once queued, with the ids --wait follows (D0 re-review R1)
{ slug } returns { ok, jobId, started } (404 for an unknown channel); { all }
returns { queued, jobIds, skipped } without waiting for the walks. ops-api and
dashboard-answers follow the job records instead of the response.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
3 files changed, 87 insertions(+), 45 deletions(-)
diff --git a/editor/app/api/ops/refresh-report/route.ts b/editor/app/api/ops/refresh-report/route.ts
@@ -1,11 +1,11 @@
import { NextResponse } from "next/server";
+import { refreshAllChannelSnapshotsAction } from "../../../channels/actions";
+import { channelExists } from "yt-dlp-transcript-common/controller/channels";
+import { requestRefreshReport } from "yt-dlp-transcript-common/jobs/snapshotScheduler";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import {
- refreshAllChannelSnapshotsAction,
- refreshChannelSnapshotAction,
-} from "../../../channels/actions";
-import {
- actionResponse,
OpsInputError,
+ opsFail,
ops,
optBool,
optString,
@@ -16,11 +16,17 @@ export const dynamic = "force-dynamic";
// POST { slug: string } | { all: true }
//
-// The single-channel form queues a refresh-report job (the one serial queue
-// every walk runs on, release 17) and returns `{ ok: true }` once snapshot.json
-// is on disk — it waits for the job, behind any other channel's. The
-// `all` form queues one refresh-report job per channel and returns the bulk
-// { queued, skipped } the /channels header button shows.
+// BOTH FORMS ANSWER ONCE QUEUED, with the job ids `--wait` follows — the ops
+// rule (_lib.ts: a job-starting route returns a jobId and never streams).
+// Every regeneration runs on the one serial refresh-report queue (release 17),
+// so holding the request until a report is on disk meant waiting for every
+// walk queued ahead of it: past five minutes, and fetch's own header timeout
+// ends the CLI with an error while the job goes on to succeed.
+//
+// `{ slug }` → `{ ok, jobId, started }`: the job this call queued, or the one
+// already queued for the channel (`started: false`; it has not started
+// reading, so it is as fresh). `{ all: true }` → the bulk `{ queued, jobIds,
+// skipped }` the /channels header button shows.
export async function POST(request: Request) {
return ops(request, ["slug", "all"], async (body) => {
const all = optBool(body, "all");
@@ -37,8 +43,17 @@ export async function POST(request: Request) {
}
// Re-read through reqSlug now that we know it is the single-channel form:
// the shape check belongs on the value that reaches a path.join.
- return actionResponse(
- await refreshChannelSnapshotAction(reqSlug(body, "slug")),
- );
+ const checked = reqSlug(body, "slug");
+ const paths = getPaths();
+ if (!(await channelExists(paths, checked))) {
+ return opsFail(`Channel "${checked}" not found`, 404);
+ }
+ const requested = await requestRefreshReport(paths, checked);
+ if (!requested.ok) return opsFail(requested.error);
+ return NextResponse.json({
+ ok: true,
+ jobId: requested.jobId,
+ started: requested.started,
+ });
});
}
diff --git a/editor/e2e/dashboard-answers.spec.ts b/editor/e2e/dashboard-answers.spec.ts
@@ -154,42 +154,46 @@ test("/ and /jobs answer while two large reports regenerate, one after the other
);
// "Update all reports", through the ops API: a refresh-report job per
- // channel on the serial queue. The call answers when both have finished, so
- // it is the clock for "while they regenerate".
+ // channel on the serial queue. The call answers once they are QUEUED, with
+ // their ids; "while they regenerate" lasts until both jobs' records say
+ // they have ended.
const startedAt = Date.now();
- let finishedAt: number | null = null;
- const refresh = request
- .post(`${baseUrl}/api/ops/refresh-report`, {
- headers: OPS_AUTH,
- data: { all: true },
- timeout: 400_000,
- })
- .then(async (res) => {
- finishedAt = Date.now();
- return {
- status: res.status(),
- body: (await res.json()) as { queued?: string[]; jobIds?: string[] },
- };
- });
+ const res = await request.post(`${baseUrl}/api/ops/refresh-report`, {
+ headers: OPS_AUTH,
+ data: { all: true },
+ timeout: 120_000,
+ });
+ expect(res.status()).toBe(200);
+ const done = { body: (await res.json()) as { queued?: string[]; jobIds?: string[] } };
+ expect([...(done.body.queued ?? [])].sort()).toEqual(BIG);
+ expect(done.body.jobIds).toHaveLength(2);
+ const ended = async (): Promise<boolean> => {
+ for (const id of done.body.jobIds ?? []) {
+ const m = await readFile(resolvePath(`test-transcripts/.jobs/${id}.meta.json`), "utf8")
+ .then((raw) => JSON.parse(raw) as JobMetaOnDisk)
+ .catch(() => null);
+ if (!m || m.status === "queued" || m.status === "running") return false;
+ }
+ return true;
+ };
+ let finishedAt: number | null = null;
const during: Sample[] = [];
while (finishedAt === null) {
+ if (Date.now() - startedAt > 360_000) throw new Error("the regenerations did not end in 6 min");
const tick = Date.now();
for (const path of ["/", "/jobs"]) {
const s = await timedGet(request, path);
- if (finishedAt === null) {
- during.push(s);
- console.log(`[dashboard-answers] +${tick - startedAt} ms ${path} ${s.ms} ms`);
- }
+ during.push(s);
+ console.log(`[dashboard-answers] +${tick - startedAt} ms ${path} ${s.ms} ms`);
}
- const wait = 2_000 - (Date.now() - tick);
- if (wait > 0 && finishedAt === null) {
- await Promise.race([new Promise((r) => setTimeout(r, wait)), refresh]);
+ if (await ended()) {
+ finishedAt = Date.now();
+ break;
}
+ const wait = 2_000 - (Date.now() - tick);
+ if (wait > 0) await new Promise((r) => setTimeout(r, wait));
}
- const done = await refresh;
- expect(done.status).toBe(200);
- expect([...(done.body.queued ?? [])].sort()).toEqual(BIG);
for (const slug of BIG) {
const snapshot = JSON.parse(
await readFile(
diff --git a/editor/e2e/ops-api.spec.ts b/editor/e2e/ops-api.spec.ts
@@ -501,23 +501,46 @@ test("refresh-report regenerates snapshot.json", async ({ request }) => {
await rm(resolvePath(SNAP), { force: true });
expect(await pathExists(SNAP)).toBe(false);
- // No page, no click: the route IS the refresh. It regenerates SYNCHRONOUSLY
- // (a filesystem scan, not a job), so { ok: true } means the file is there.
+ // No page, no click: the route IS the refresh. It answers once the
+ // regeneration is QUEUED (release 17: one serial refresh-report queue, and
+ // the ops rule — a job-starting route returns a jobId), so the job is
+ // followed to `done` before the file is read.
+ const doneJob = async (jobId: string | undefined) => {
+ expect(jobId).toBeTruthy();
+ await expect
+ .poll(
+ async () =>
+ (
+ await readJson<{ status: string }>(
+ `test-transcripts/.jobs/${jobId}.meta.json`,
+ ).catch(() => null)
+ )?.status ?? null,
+ { timeout: 30_000 },
+ )
+ .toBe("done");
+ };
const first = await ops(request, "refresh-report", { slug: SLUG });
- expect(first.body).toEqual({ ok: true });
+ expect(first.body.ok).toBe(true);
+ await doneJob(first.body.jobId);
const snapshot = await readJson<{ generatedAt: string; totals: { videos: number } }>(SNAP);
expect(snapshot.generatedAt).toBeTruthy();
expect(snapshot.totals.videos).toBeGreaterThanOrEqual(0);
- // Re-running REWRITES it. Polled through the action itself because two scans
- // of a six-video fixture can land in the same millisecond.
+ // Re-running REWRITES it. Polled because two scans of a six-video fixture
+ // can land in the same millisecond.
await expect
.poll(async () => {
- await ops(request, "refresh-report", { slug: SLUG });
+ const again = await ops(request, "refresh-report", { slug: SLUG });
+ await doneJob(again.body.jobId);
return (await readJson<{ generatedAt: string }>(SNAP)).generatedAt;
})
.not.toBe(snapshot.generatedAt);
+ // An unknown channel is a 404 naming it, and starts nothing.
+ const missing = await ops(request, "refresh-report", { slug: "no-such-channel" });
+ expect(missing.status).toBe(404);
+ expect(missing.body.error).toMatch(/no-such-channel/);
+
// The bulk form queues a job per channel and reports both lists.
const all = await ops(request, "refresh-report", { all: true });
expect(all.status).toBe(200);
