import { NextResponse } from "next/server"; import { MAX_CLIP_WINDOW_SECONDS, MAX_FETCH_MAX_HEIGHT, MIN_FETCH_MAX_HEIGHT, isFetchMaxHeight, } from "yt-dlp-transcript-common/lib/clipWindow"; import { isValidSiteId } from "yt-dlp-transcript-common/lib/site"; import type { FetchWindowsItem } from "yt-dlp-transcript-common/controller/fetchWindows"; import { fetchMissingEvidenceAction, fetchWindowsAction, } from "../../../channels/[slug]/videos/fetchWindowsAction"; import { OpsInputError, ops, opsFail, optBool, optString, reqSlug, reqString, reqVideoId, type OpsBody, } from "../_lib"; export const dynamic = "force-dynamic"; // POST { siteId, maxHeight?, dryRun? } // | { items: [{ slug, id, from, to, clipId?, reason?, pad?, webpageUrl? }, …], // requestedBy, manifest?, maxHeight?, dryRun? } // dryRun -> { ok, dryRun: true, groups: [{ platform, queueKey, items }], // cached, inFlight, unresolved[, onDisk, unfetchable] } // else -> { ok, dryRun: false, jobs: [{ platform, queueKey, jobId, items }], // jobIds, jobId (one job only), refused, cached, inFlight, unresolved // [, onDisk, unfetchable] } // // Fetch clip windows through the managed path, ONE PACED JOB PER PLATFORM // QUEUE (controller/fetchWindows.ts): the batch form of // /api/media/fetch-window. `siteId` fetches every window the site's published // reports cite and the disk does not hold (`unfetchable` lists the spans no // window can fill, with why; `onDisk` counts the ones already there). `items` // fetches an explicit list, and must say who asked (`requestedBy`). A window // already on disk is answered in `cached` and joins no job; a platform cooling // down or held is `refused` for its group, and the other groups still start. // Re-running the same body is the resume: fetched windows are cached. // // A big list goes in a file: `pnpm ops fetch-windows --file list.json`. const ITEM_KEYS = ["slug", "id", "from", "to", "clipId", "reason", "pad", "webpageUrl"]; function optItemString(e: OpsBody, key: string, where: string): string | undefined { const v = e[key]; if (v === undefined) return undefined; if (typeof v !== "string") throw new OpsInputError(`"${where}.${key}" must be a string`); return v.trim() || undefined; } function reqSeconds(e: OpsBody, key: string, where: string): number { const v = e[key]; if (typeof v !== "number" || !Number.isFinite(v) || v < 0) { throw new OpsInputError(`"${where}.${key}" must be finite, non-negative seconds`); } return v; } function reqWindowItems(body: OpsBody): FetchWindowsItem[] { const v = body.items; if (!Array.isArray(v) || v.length === 0) { throw new OpsInputError( `"items" must be a non-empty array of { "slug", "id", "from", "to" }`, ); } return v.map((entry, i) => { const where = `items[${i}]`; if (typeof entry !== "object" || entry === null || Array.isArray(entry)) { throw new OpsInputError(`"${where}" must be an object { "slug", "id", "from", "to" }`); } const e = entry as OpsBody; const extra = Object.keys(e).filter((k) => !ITEM_KEYS.includes(k)); if (extra.length) { throw new OpsInputError( `"${where}" has unknown key(s): ${extra.join(", ")} — accepted: ${ITEM_KEYS.join(", ")}`, ); } const from = reqSeconds(e, "from", where); const to = reqSeconds(e, "to", where); if (from >= to) { throw new OpsInputError(`"${where}": from (${from}) must be less than to (${to})`); } if (to - from > MAX_CLIP_WINDOW_SECONDS) { throw new OpsInputError( `"${where}": a window may be at most ${MAX_CLIP_WINDOW_SECONDS}s ` + `(asked for ${Math.round(to - from)}s)`, ); } const pad = e.pad; if (pad !== undefined && (typeof pad !== "number" || !Number.isFinite(pad))) { throw new OpsInputError(`"${where}.pad" must be a number`); } return { slug: reqSlug(e, "slug"), id: reqVideoId(e, "id"), from, to, clipId: optItemString(e, "clipId", where), reason: optItemString(e, "reason", where)?.slice(0, 400), ...(typeof pad === "number" ? { pad } : {}), webpageUrl: optItemString(e, "webpageUrl", where), }; }); } function optMaxHeight(body: OpsBody): number | undefined { const v = body.maxHeight; if (v === undefined || v === null) return undefined; if (!isFetchMaxHeight(v)) { throw new OpsInputError( `"maxHeight" must be a whole number of pixels from ` + `${MIN_FETCH_MAX_HEIGHT} to ${MAX_FETCH_MAX_HEIGHT}`, ); } return v; } export async function POST(request: Request) { return ops( request, ["siteId", "items", "requestedBy", "manifest", "maxHeight", "dryRun"], async (body) => { const maxHeight = optMaxHeight(body); const dryRun = optBool(body, "dryRun"); if (body.siteId !== undefined && body.items !== undefined) { throw new OpsInputError('send either "siteId" or "items", not both'); } let result: Awaited>; if (body.siteId !== undefined) { // A site's windows say who asked themselves (the reports, by site). for (const k of ["requestedBy", "manifest"]) { if (body[k] !== undefined) { throw new OpsInputError(`"${k}" goes with "items"; a site's windows name the site`); } } const siteId = reqString(body, "siteId"); if (!isValidSiteId(siteId)) { throw new OpsInputError( `"${siteId}" is not a valid site id (lowercase letters, digits and "-"; must start with a letter or digit)`, ); } result = await fetchMissingEvidenceAction(siteId, { dryRun, maxHeight }); } else if (body.items !== undefined) { const items = reqWindowItems(body); // WHO ASKED IS NOT OPTIONAL, as on the single route: a window nobody // can explain in six months is one nobody can clean up. const requestedBy = reqString(body, "requestedBy"); result = await fetchWindowsAction({ items, requestedBy, manifest: optString(body, "manifest")?.trim() || undefined, maxHeight, dryRun, }); } else { throw new OpsInputError('"siteId" (a site\'s missing evidence) or "items" (a list of windows) is required'); } if (!result.ok) return opsFail(result.error); if (result.dryRun) { const { jobs: _jobs, refused: _refused, ...plan } = result; return NextResponse.json(plan); } const { groups: _groups, ...run } = result; // EVERY JOB THAT WILL WRITE ONE OF THESE WINDOWS: the ones this call // started, then the ones already queued or running for a window it asked // for (`inFlight`, release 19 A5) — so --wait follows the windows asked // for, not only the jobs that happened to be new. const jobIds = [ ...new Set([ ...run.jobs.map((j) => j.jobId), ...run.inFlight.map((f) => f.jobId), ]), ]; return NextResponse.json({ ...run, jobIds, ...(jobIds.length === 1 ? { jobId: jobIds[0] } : {}), }); }, ); }