commit cb57555fac2ec278e24db1fb064d20aeaee4938a
parent 142106ad889f50dd05f1738b421fc42194d4e1fb
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 21 Sep 2026 02:36:27 -0400
full source: ask the editor for the whole recording, and find it again
`full: true` already existed on `/api/media/fetch-window`; nothing used it and
nothing could find the result afterwards. Three halves:
**The editor's.** `fetch-window.spec.ts` now pins the round trip: 202 + jobId,
the container in the saved-video store, `saved-video.json` carrying `origin`
(who asked — separate from `keepReason`, which stays override/pin so the
retention prune leaves the container alone), and a second identical ask
answering 200 cached with no new yt-dlp invocation. That last assertion is the
whole reason a tool routes through here instead of running yt-dlp.
**umtool's fetch.** `fetch-via-editor.mjs --full` posts `full: true` instead of
a span — the route reads `full === true` before it validates from/to, so sending
both would describe a request the editor does not have.
`editorFullSourceSteps` is the same script, same NDJSON events, a longer
timeout. There is deliberately no local twin: `UMTOOL_LOCAL_FETCH` is about a
machine with no editor, and such a machine has nowhere to put a full source that
anything else would find. `/api/report/fetch` takes `full: true` and SKIPS the
cached-window check — that check asks "does a file hold this span", and a full
source is not a span; the editor's own cached branch is what makes the second
ask free. The bench gets a plainly-labelled button, separate from the two
widen-one-edge ones because a recording can be gigabytes.
**Finding it again.** The store is on another path and possibly another drive,
so the only way back is the pointer. `clipWindowDirs` is now async and emits
`{video, dir: pointer.dir, duration}` beside the clips dir — exactly the shape
`rawCacheOf`'s SOURCE_MEDIA_RE branch handles, which counts the container as the
window [0, duration], so every clip of that video is fetched at once. The
duration comes from the cue doc and is read ONLY for a video that has a pointer:
`rawCacheOf` needs a finite number, the cue file is the archive's own record of
it, and it is the expensive input that module memoises against mtime — asking
for the handful of videos somebody fetched a full source for keeps the index
load as cheap as it was. No duration means no entry, never a guess.
Also here because the plan put it here: `/api/jobs/<id>/log` is DOCUMENTED as
deliberately ungated rather than gated. It leaks job log text by id — yt-dlp
output, slugs, video ids, corpus paths; not credentials, not media — and gating
it would switch every run panel's log off on a default install, because
`authorizeWorkerRequest` answers 503 when `WORKER_TOKEN` is unset and the
browser polls it same-origin with no token. The boundary is
`docker/guard-exposure.sh` and Caddy on 127.0.0.1. `followJob`'s authorization
header is gone: implying a gate that does not exist is worse than sending
nothing.
FACTS.md gains the whole slice.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
10 files changed, 453 insertions(+), 26 deletions(-)
diff --git a/editor/app/api/jobs/[id]/log/route.ts b/editor/app/api/jobs/[id]/log/route.ts
@@ -6,6 +6,31 @@ import { getRegistry } from "yt-dlp-transcript-common/jobs/registry";
export const dynamic = "force-dynamic";
+// A JOB'S LOG, AND IT IS DELIBERATELY UNGATED.
+//
+// WHAT IT LEAKS, stated plainly: the text of any job's log by id, plus its
+// status and queue position. That is yt-dlp output, channel slugs, video ids,
+// file paths inside the corpus, and whatever a controller chose to log. It is
+// not credentials (a step's env is never logged) and it is not media.
+//
+// WHY IT IS NOT BEHIND `WORKER_TOKEN` like /api/worker/* and /api/ops/*.
+// `authorizeWorkerRequest` answers 503 when the variable is UNSET — "this
+// surface is off until you opt in" — which is exactly right for a surface that
+// accepts instructions, and exactly wrong here: the browser polls this endpoint
+// same-origin, with no token, for every run panel in the app. Gating it would
+// switch every job log off on a default install, and the fix an operator would
+// reach for is to put a shared secret into the page that starts the job.
+//
+// THE BOUNDARY IS THE NETWORK, NOT THIS ROUTE. Nothing in the editor publishes
+// a port; Caddy is the single front door and binds 127.0.0.1, and
+// `docker/guard-exposure.sh` REFUSES TO START if a private app is bound
+// off-loopback with no auth in front of it (AGENTS.md, "The runtime
+// container"). An editor reachable by a stranger is a misconfiguration that
+// guard exists to make impossible, and one this route could not fix anyway.
+//
+// So: `scripts/archilyzer-ops.mjs`'s `followJob` sends no authorization header,
+// because there was never anything to send it to.
+
export async function GET(
request: Request,
{ params }: { params: Promise<{ id: string }> },
diff --git a/editor/e2e/fetch-window.spec.ts b/editor/e2e/fetch-window.spec.ts
@@ -323,3 +323,67 @@ test("a 429 fails the job and puts the platform in cooldown", async ({
expect(body.platform).toBe("youtube");
expect(body.cooldownMs).toBeGreaterThan(0);
});
+
+// THE WHOLE RECORDING, when a window will not do — a tool that needs to re-cut
+// freely, or a source whose windows would tile the entire runtime.
+//
+// It goes to the SAVED-VIDEO STORE, not to clips/: that is where big containers
+// already live, with a retention rule that leaves an explicitly-requested one
+// alone. `origin` is what records who asked — separate from `keepReason`, which
+// stays override/pin so `pruneSavedVideos` never evicts a container somebody
+// wanted.
+test("a full-source fetch lands in the saved-video store and is cached on a second ask", async ({
+ request,
+}) => {
+ test.setTimeout(90_000);
+ const ask = () =>
+ request.post(`${baseUrl}/api/media/fetch-window`, {
+ headers: AUTH,
+ data: {
+ channelSlug: SLUG,
+ videoId: VIDEO,
+ full: true,
+ requestedBy: "umtool",
+ manifest: "demo-report",
+ clipId: "c07",
+ reason: "the bench needs to re-cut this one freely",
+ },
+ });
+
+ const post = await ask();
+ expect(post.status(), await post.text()).toBe(202);
+ const queued = (await post.json()) as { jobId: string; file: string };
+ expect(queued.jobId).toBeTruthy();
+
+ const finished = await pollJob(request, queued.jobId);
+ expect(finished.status, JSON.stringify(finished)).toBe("done");
+
+ // THE POINTER IS THE RECORD. It lives in the video dir and names the store
+ // directory holding the container — the shape umtool's raw cache reads
+ // (`rawCacheOf`'s source-media branch).
+ const pointer = JSON.parse(
+ await readFile(resolvePath(rel(`data/${VIDEO}/saved-video.json`)), "utf8"),
+ ) as Record<string, unknown>;
+ expect(typeof pointer.dir).toBe("string");
+ expect(String(pointer.file)).toMatch(/^source-media\./);
+ expect(Number(pointer.bytes)).toBeGreaterThan(0);
+ // WHO ASKED, and it is not `keepReason`: that stays override/pin so the
+ // retention prune leaves the container alone, which cannot also carry a
+ // requester.
+ const origin = pointer.origin as Record<string, unknown>;
+ expect(origin?.requestedBy).toBe("umtool");
+ expect(origin?.manifest).toBe("demo-report");
+ expect(origin?.clipId).toBe("c07");
+
+ // THE SECOND ASK COSTS NOTHING. A cached full source answers 200 with the
+ // file rather than queueing a second download of the same container — which
+ // is the whole reason a tool routes through here instead of running yt-dlp.
+ const invBefore = await invocations();
+ const again = await ask();
+ expect(again.status()).toBe(200);
+ const cached = (await again.json()) as Record<string, unknown>;
+ expect(cached.cached).toBe(true);
+ expect(String(cached.file)).toContain(String(pointer.file));
+ expect(Number(cached.bytes)).toBe(Number(pointer.bytes));
+ expect(await invocations()).toBe(invBefore);
+});
diff --git a/plans/FACTS.md b/plans/FACTS.md
@@ -4524,3 +4524,146 @@ with `WORKER_TOKEN=test-worker-token` in `editor/package.json`'s `dev:test`, one
server for the whole suite, so no spec can observe the disabled state.
`editor/e2e/ops-api.spec.ts` covers 401 (missing and wrong); the 503 is
`authorizeWorkerRequest`'s own first branch, shared with `/api/worker/*`.
+
+---
+
+## Storage debts (verified 2026-09-21, branch `storage/debts-1`)
+
+The sweep FACTS asked for above ("a later sweep owes three things: the recursion
+or an explicit clips total, a count on the storage page, and an eviction rule").
+
+### `clips/` is counted now, and `totalMediaBytes` GREW
+
+- **`snapshot.totalMediaBytes` includes clip windows.** The `mediaBytes` loop in
+ `channelSnapshot.ts` recurses ONE level into `data/<id>/clips/` and nothing
+ else; the old `if (!st.isFile()) continue` sat under a comment saying "a video
+ dir is flat", which the clip-window feature made untrue. The definition did
+ not change — it was always "every byte under `data/<id>/`" — the number did.
+ Four readers see a slightly larger, correct figure: the /storage rows, the
+ /channels Size column, the free-up selection and the relocation preflight.
+ Nothing partitions on it.
+- **`totalClipsBytes` is the "of which", not a sibling.** Optional on the same
+ terms (absent means "unknown until Refresh report", never 0; a fresh snapshot
+ always writes it, 0 included). Split out because clips are the one part of a
+ channel's bytes that is a CACHE nothing prunes.
+- `LocationRollup.clipsBytes` and `StorageRow.clipsBytes` carry it to /storage;
+ the video page folds `listClipWindows`'s own per-window `bytes`, which costs
+ no I/O. There is no `unknownClipsBytes`: it would be the same set of channels
+ `unknownBytes` already counts.
+
+### Eviction is BY AGE, and that is load-bearing
+
+- `common/controller/evictClipWindows.ts` is the only thing that removes a
+ window. **Whether a window is still WANTED cannot be known from the editor**:
+ a umtool report's manifest cites spans, and those manifests live in a umtool
+ project — possibly on another machine, possibly not written yet. There is no
+ reference count and no honest way to invent one. Every surface says so (the
+ /storage card's caveat block, the job log's second line, the route header).
+ An evicted window is re-fetchable: the cost is a fetch, not data.
+- **The age is the MTIME, not the provenance sidecar's `fetchedAt`.** A window
+ whose sidecar was never written (a fetch interrupted after the media landed)
+ would otherwise be un-evictable forever — exactly backwards. `rsync -a`
+ preserves mtimes, so a relocated channel's windows keep their real ages.
+- `needsMedia: true` on the job kind covers a one-channel run;
+ `inspectChannelMedia` inside the controller covers the corpus-wide one, where
+ there is no slug for `runManagedFunction` to check. Without it the pass
+ reports a clean eviction of zero bytes about a platter full of windows.
+ `in-transition` is refused outright: the mover's verify pass compares trees.
+- The empty `clips/` is left behind on purpose — `rmdir` would race a fetch that
+ just created it, and an empty directory is invisible to every video-dir
+ enumerator anyway.
+
+### `assertRelocationRootPresent`
+
+- **Every absolute `mkdir` in both movers is `{recursive: true}`**, so a move
+ aimed at an unmounted root built it on the root filesystem and filled it. Two
+ checks: `stat(root)` is a directory, and — when the root belongs to a location
+ carrying a learned `volume.uuid` — the probe answers `available` with a KNOWN
+ identity whose uuid matches. An empty mountpoint directory passes the first
+ and fails the second, because findmnt -T reports the root filesystem's uuid.
+- **It FAILS OPEN on unknown identity** (a container has no block devices) and a
+ root nobody named as a location is STAT-ONLY. Called from
+ `relocationRootProblem` — so preview and action agree — and again immediately
+ before each copy phase's mkdir, because a job can sit in the queue for hours.
+ On the move-BACK paths the guard is on the LOCATION ROOT, not on `incoming`.
+- **The probe memo moved to `lib/storageVolumes.ts`** (re-exported from
+ `controller/storageLocations.ts`): the guard probes and is called per channel
+ by the bulk move and the re-point preflight, but the controller imports
+ `relocationRootProblem`, so reaching the memo through it would be a cycle.
+
+### `relocateChannelMedia` takes an `io` settings seam
+
+- The space check demands `bytes + resumeMarginGB` free, default 2 GB, and the
+ unit-test fixtures live under `os.tmpdir()` — a tmpfs sized from RAM. Eight
+ tests failed on this machine with "2.9 KB to move plus a 2 GB resume margin".
+ The tests now inject `minFreeDiskGB: 0`, which is production's own rule
+ (`minFreeDiskGB > 0 ? resumeMarginGB : 0`), not a faked margin.
+ `relocateSavedVideos` already had the seam, which is why none of its tests
+ were in the failing set.
+
+### `/api/jobs/<id>/log` is deliberately ungated
+
+- **It leaks job log text by id**, plus status and queue position: yt-dlp output,
+ slugs, video ids, corpus paths. Not credentials (a step's env is never logged)
+ and not media.
+- **Gating it would switch every run panel off on a default install.**
+ `authorizeWorkerRequest` answers 503 when `WORKER_TOKEN` is UNSET, and the
+ browser polls this endpoint same-origin with no token. The boundary is
+ `docker/guard-exposure.sh` plus Caddy binding 127.0.0.1, not this route.
+ `scripts/archilyzer-ops.mjs`'s `followJob` sent an authorization header to it;
+ that header is gone, because implying a gate that does not exist is worse than
+ sending nothing.
+
+### A fan-out now returns its job ids
+
+- `QueueOutcome.jobIds` is parallel to `queued`, and `queueResponse` emits it
+ plus a bare `jobId` when exactly one job started. Without it `pnpm ops
+ relocate --wait` found no id, concluded nothing had been started and returned
+ 0 — reporting success about a copy that had not begun. Both fields are
+ additive; `queued` keeps its meaning and spelling.
+
+### A full source reaches the build through the pointer, not through `clips/`
+
+- `full: true` puts the whole recording in the SAVED-VIDEO STORE, which is on
+ another path and possibly another drive — so the only way back to it is
+ `data/<id>/saved-video.json`. `clipWindowDirs` (`umtool/lib/projects/report.mjs`)
+ is now ASYNC and emits `{video, dir: pointer.dir, duration}` beside the clips
+ dir; `rawCacheOf`'s `SOURCE_MEDIA_RE` branch then counts the container as the
+ window `[0, duration]`.
+- **The duration comes from the cue doc, and ONLY for a video that has a
+ pointer.** `rawCacheOf` needs a finite duration; the cue file is the archive's
+ own record of it, and it is the expensive input that module memoises against
+ mtime. Reading it for the handful of videos somebody fetched a full source for
+ keeps the index load exactly as cheap as it was. No duration ⇒ no entry, never
+ a guess.
+- The bench's "fetch whole source via editor" SKIPS the cached-window check in
+ `/api/report/fetch`: that check asks "does a file hold this span", and a full
+ source is not a span. There is no local (`UMTOOL_LOCAL_FETCH`) twin — a
+ machine with no editor has nowhere to put a full source anything else finds.
+
+### `/review` lists what the machine decided
+
+- `common/views/review.ts` (`autoPausedRows`) over `settings.channelPriority`.
+ A channel a PERSON paused is not a row. Resume is `setChannelTierAction` — the
+ ONE priority writer — posting the `previousTier`, which is the whole content
+ of the record; a manual tier change is also what clears `autoPaused`.
+
+### The four nits that were numbers that could lie
+
+- **`volumeFreeBytes` believed an unmounted mountpoint**: the directory existing
+ is not the drive being there, so `getFreeBytes` reported the PARENT volume's
+ free space — the disk being emptied. `st.dev` vs the parent's is what a mount
+ is; asked only when the location has a `volume.uuid`.
+- **"Free up N GB" proposed channels mid-relocation**, which the bulk move
+ refuses by name — a guaranteed skip whose bytes inflated the promise.
+ `FreeUpSelection.moving` counts them separately from `unmeasured`.
+- **`bytesOnLocation` travelled without its caveat**; `unknownBytesOnLocation` /
+ `unknownBytesInPlace` now pair with it.
+- **"internal" was an available location id** in both the form regex and the
+ sanitizer. `INTERNAL_LOCATION_ID` moved to `lib/storageLocations.ts` so
+ `lib/settings.ts` can refuse it (lib may not import controller).
+- Also: `measureTree` in `buildStorage.ts` ran per render on a `force-dynamic`
+ page an AutoRefresh re-renders. `editor/app/storage/lib/measureStore.ts` caches
+ 60 s, 32 entries; `measureTree` is BANNED in
+ `noCorpusWalkInRenderPaths.test.ts` with an allow map naming that one file, and
+ the ban's match is word-bounded so `measureTreeCached` is distinguishable.
diff --git a/scripts/archilyzer-ops.mjs b/scripts/archilyzer-ops.mjs
@@ -163,12 +163,17 @@ function authHeaders() {
// Follow a job's log to its terminal state. Returns the status string.
// Deliberately polls the SAME endpoint the editor's own log panel does, so a
// job started here and a job started by a click are observed identically.
+// NO AUTH HEADER, and that is not an omission. `/api/jobs/<id>/log` is
+// deliberately ungated (see its route for what it does and does not leak, and
+// why gating it would switch every run panel's log off on a default install) —
+// the browser polls it same-origin with no token. Sending one here implied a
+// gate that does not exist, which is worse than sending nothing: the next
+// person to read this would conclude the endpoint was protected.
async function followJob(jobId, quiet) {
let from = 0;
for (;;) {
const res = await fetch(
`${baseUrl()}/api/jobs/${encodeURIComponent(jobId)}/log?from=${from}`,
- { headers: authHeaders() },
);
if (!res.ok) throw new Error(`log poll failed: HTTP ${res.status}`);
const payload = await res.json();
diff --git a/umtool/app/api/report/fetch/route.ts b/umtool/app/api/report/fetch/route.ts
@@ -2,6 +2,7 @@ import { getJob, jobView, runningJob, startJob } from "@/lib/jobs";
import {
FETCH_MAX_PAD,
editorFetchSteps,
+ editorFullSourceSteps,
fetchSteps,
localFetch,
} from "@/lib/report/driver.mjs";
@@ -36,6 +37,20 @@ export async function POST(request: Request) {
const padBefore = side(body.padBefore, pad);
const padAfter = side(body.padAfter, pad);
+ // THE WHOLE RECORDING, when widening a window one side at a time is the wrong
+ // shape of answer: a clip whose windows would tile the entire runtime, or a
+ // bench session that wants to re-cut freely without a fetch per attempt.
+ //
+ // It SKIPS the cached-window check below, and that is not an oversight. That
+ // check asks "does a file already hold this span"; a full source is not a
+ // span, and a window that happens to cover the clip is not a reason to refuse
+ // the request — the operator asked for the recording, not for these seconds.
+ // The editor's own cached branch is what makes the second ask free.
+ //
+ // No local twin (see editorFullSourceSteps): a machine with no editor has
+ // nowhere to put a full source that anything else would find.
+ const wantFull = body.full === true;
+
const r = await resolveClip(projectId, clipId);
if ("error" in r) return Response.json({ error: r.error }, { status: r.status });
@@ -51,11 +66,13 @@ export async function POST(request: Request) {
// corpus counts as cached. Without it this route would queue a job to
// download bytes that are on the disk, and the bench would show a fetch that
// moved nothing.
- const windows = (await windowsFor(
- r.project,
- r.clip,
- await projectCache(r.project, r.manifest),
- )) as { from: number; to: number }[];
+ const windows = wantFull
+ ? []
+ : ((await windowsFor(
+ r.project,
+ r.clip,
+ await projectCache(r.project, r.manifest),
+ )) as { from: number; to: number }[]);
// CLAMPED AT 0, because the FETCH clamps at 0 — build-video.mjs's fetchClip
// and fetch-via-editor.mjs both do `Math.max(0, start - padBefore)`, since a
// recording has no seconds before its first. Asking here whether a window
@@ -104,15 +121,26 @@ export async function POST(request: Request) {
// corpus where the next report reuses them instead of paying again. Running
// yt-dlp from here has none of that, which is why it is now the opt-out
// (UMTOOL_LOCAL_FETCH=1) rather than the default.
- const steps = localFetch()
- ? fetchSteps(r.project, clipId, { padBefore, padAfter })
- : editorFetchSteps(r.project, clipId, { padBefore, padAfter });
- const job = startJob(`fetch ${projectId}/${clipId}`, steps, {
- project: r.project.id,
- });
+ const steps = wantFull
+ ? editorFullSourceSteps(r.project, clipId)
+ : localFetch()
+ ? fetchSteps(r.project, clipId, { padBefore, padAfter })
+ : editorFetchSteps(r.project, clipId, { padBefore, padAfter });
+ const job = startJob(
+ wantFull
+ ? `fetch whole source ${projectId}/${clipId}`
+ : `fetch ${projectId}/${clipId}`,
+ steps,
+ {
+ project: r.project.id,
+ },
+ );
// Which side this run is growing, so the caller can say so rather than
- // guessing from a file name.
- return Response.json({ job: jobView(job), padBefore, padAfter }, { status: 202 });
+ // guessing from a file name. `full` says it grew neither.
+ return Response.json(
+ { job: jobView(job), padBefore, padAfter, full: wantFull },
+ { status: 202 },
+ );
}
export async function GET(request: Request) {
diff --git a/umtool/components/projects/ClipBench.tsx b/umtool/components/projects/ClipBench.tsx
@@ -879,6 +879,62 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
[data.project, clip.id, refresh, windows],
);
+ // ---- the whole recording --------------------------------------------------
+ //
+ // WHEN WIDENING ONE SIDE AT A TIME IS THE WRONG SHAPE OF ANSWER: a clip whose
+ // windows would tile the entire runtime, or a session that wants to re-cut
+ // freely without a fetch per attempt. It posts `full: true` through the same
+ // route and the same script (`fetch-via-editor.mjs --full`), so the events,
+ // the job view and the polling below are identical.
+ //
+ // The container goes to the editor's SAVED-VIDEO STORE, not to clips/ — that
+ // is where big files already live, with a retention rule that leaves an
+ // explicitly-requested one alone — and `clipWindowDirs` reads the pointer
+ // back, so the build treats it as the window [0, duration] and every clip of
+ // that video is fetched at once.
+ //
+ // IT IS A DELIBERATE, SEPARATE BUTTON and not a smarter default: a whole
+ // recording can be gigabytes and hours of somebody's bandwidth. The label
+ // says what it is asking for.
+ const fetchWholeSource = useCallback(async () => {
+ setBusy("asking the editor for the whole recording…");
+ setNote(null);
+ const r = await fetch("/api/report/fetch", {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ project: data.project, clip: clip.id, full: true }),
+ });
+ if (!r.ok) {
+ const j = (await r.json()) as { error?: string };
+ setBusy(null);
+ setNote(j.error ?? "could not start the fetch");
+ return;
+ }
+ const { job } = (await r.json()) as { job: { id: string } };
+ // A whole recording is minutes, not the window fetch's seconds — same
+ // 500 ms poll, a longer patience. Giving up here loses nothing: the job
+ // keeps running on the editor and the next ask finds it cached.
+ for (let i = 0; i < 3600; i += 1) {
+ await new Promise((res) => setTimeout(res, 500));
+ const s = await fetch(`/api/report/fetch?job=${job.id}`, { cache: "no-store" });
+ const sj = (await s.json()) as { job: { state: string; error: string | null } | null };
+ if (!sj.job || sj.job.state === "running") continue;
+ setBusy(null);
+ if (sj.job.state === "failed") {
+ setNote(`the fetch failed: ${sj.job.error ?? "unknown"}`);
+ } else {
+ await refresh();
+ setNote(
+ "the whole recording is in the saved-video store — every clip of " +
+ "this video now cuts from it without another fetch",
+ );
+ }
+ return;
+ }
+ setBusy(null);
+ setNote("the fetch is still running; reload to pick it up");
+ }, [data.project, clip.id, refresh]);
+
// ---- re-rendering this one clip -------------------------------------------
//
// The `preview` preset is one clip and nothing else, and the segment is
@@ -1382,6 +1438,17 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
fetch to +{nextAfter} s →
</button>
)}
+ {/* THE WHOLE RECORDING, when neither edge is the answer. Last,
+ and plainly labelled: it can be gigabytes. */}
+ <button
+ type="button"
+ data-fetch-full=""
+ className={buttonVariants({ size: "sm" })}
+ disabled={!!busy}
+ onClick={() => void fetchWholeSource()}
+ >
+ fetch whole source via editor
+ </button>
</div>
)}
diff --git a/umtool/lib/projects/report.mjs b/umtool/lib/projects/report.mjs
@@ -81,6 +81,17 @@ const DEAD_ORIGIN =
export const isDeadOrigin = (o) => !o || typeof o !== "string" || DEAD_ORIGIN.test(o);
const stat0 = (p) => stat(p).then((s) => s, () => null);
+const readJson0 = (p) =>
+ readFile(p, "utf8").then(
+ (t) => {
+ try {
+ return JSON.parse(t);
+ } catch {
+ return null;
+ }
+ },
+ () => null,
+ );
export const manifestPath = (dir) => path.join(dir, MANIFEST_NAME);
@@ -318,7 +329,7 @@ export async function readAvailability(dir) {
* window one report paid for is a window the next one does not. One entry per
* distinct (channel, video) the timeline cites; rawCacheOf reads each once.
*/
-export const clipWindowDirs = (m, channelsDir) => {
+export const clipWindowDirs = async (m, channelsDir) => {
const seen = new Set();
const out = [];
for (const e of clipsOf(m)) {
@@ -327,10 +338,35 @@ export const clipWindowDirs = (m, channelsDir) => {
const key = `${chan}/${e.video}`;
if (seen.has(key)) continue;
seen.add(key);
- out.push({
- video: e.video,
- dir: path.join(channelsDir ?? GLOBAL_CHANNELS_DIR(), chan, "data", e.video, "clips"),
- });
+ const videoDir = path.join(channelsDir ?? GLOBAL_CHANNELS_DIR(), chan, "data", e.video);
+ out.push({ video: e.video, dir: path.join(videoDir, "clips") });
+
+ // AND THE WHOLE SOURCE, when somebody asked the editor for one.
+ //
+ // `full: true` puts the entire recording in the saved-video STORE, not in
+ // clips/ -- that is where big containers already live, with a retention
+ // rule that leaves an explicitly-requested one alone. The store is on
+ // another path (and possibly another drive), so the only way back to it is
+ // the pointer the editor leaves in the video dir. `pointer.dir` is exactly
+ // the shape rawCacheOf's SOURCE_MEDIA_RE branch handles.
+ //
+ // THE DURATION IS READ FROM THE CUES, and ONLY for a video that actually
+ // has a pointer. rawCacheOf needs a finite duration to treat the container
+ // as the window [0, duration]; the cue doc is the archive's own record of
+ // how long the recording is. Reading it is the expensive thing this module
+ // memoises against mtime (see below) -- so it is asked for the handful of
+ // videos somebody fetched a full source for, and never for the rest, which
+ // keeps the index load exactly as cheap as it was.
+ const pointer = await readJson0(path.join(videoDir, "saved-video.json"));
+ if (!pointer?.dir) continue;
+ const cues = await readCues(path.join(videoDir, "transcript.cues.json"));
+ const duration = Number(cues?.duration);
+ // No duration is no entry rather than a guess: without one the cache
+ // cannot say what the container holds, and claiming a span a file may not
+ // cover is the one failure mode worse than a cache miss.
+ if (Number.isFinite(duration) && duration > 0) {
+ out.push({ video: e.video, dir: pointer.dir, duration });
+ }
}
return out;
};
@@ -1050,7 +1086,7 @@ export async function readClipDetail(dir, { manifest = null } = {}) {
// ONE listing of out/clips-raw for the whole project. This used to be two
// readdirs PER CLIP -- the padded lookup and the full list -- so a forty-clip
// cut paid eighty directory reads to draw one page.
- const raw = await rawCacheOf(dir, { extraDirs: clipWindowDirs(m, channelsDir) });
+ const raw = await rawCacheOf(dir, { extraDirs: await clipWindowDirs(m, channelsDir) });
const segDirs = segmentDirs(path.join(dir, "out"));
const segLists = await Promise.all(segDirs.map((d) => readdir(d).catch(() => [])));
const segWhich = segLists.findIndex((l) => l.length);
diff --git a/umtool/lib/report/driver.mjs b/umtool/lib/report/driver.mjs
@@ -247,6 +247,52 @@ export function editorFetchSteps(project, clipId, pad) {
];
}
+/**
+ * THE WHOLE RECORDING, asked of the editor.
+ *
+ * Same script, same events, one extra flag — so the job runner and the bench's
+ * progress readout cannot tell it from a window fetch, which is what keeps
+ * "which one ran" an operator's decision rather than a fork in every consumer.
+ *
+ * There is no local twin on purpose. A full source is the expensive ask, and
+ * the reason to route it through the editor (cookie policy, per-platform
+ * sleeps, the 429 cooldown, and a container that lands in the store where the
+ * next report reuses it) is strongest exactly here. UMTOOL_LOCAL_FETCH is about
+ * a machine with no editor; such a machine has nowhere to put a full source
+ * that anything else would find.
+ *
+ * The timeout is the window fetch's doubled: a whole recording can be hours of
+ * media, and the job keeps running on the editor if we give up — nothing is
+ * lost, and the next ask finds it cached.
+ *
+ * @param {{ dir: string }} project
+ * @param {string} clipId
+ * @returns {import("../trim").Step[]}
+ */
+export function editorFullSourceSteps(project, clipId) {
+ return [
+ {
+ cwd: PIPELINE_DIR,
+ // EMPTY — see editorFetchSteps: jobView() echoes a step's env to the
+ // browser, and WORKER_TOKEN is already in process.env.
+ env: {},
+ label: `ask the editor for the whole source behind ${clipId}`,
+ argv: [
+ "node",
+ script("fetch-via-editor.mjs"),
+ path.join(project.dir, "video.manifest.json"),
+ "--fetch-only",
+ clipId,
+ "--full",
+ "--progress",
+ "ndjson",
+ ],
+ ndjson: true,
+ timeoutMs: 30 * 60_000,
+ },
+ ];
+}
+
/** Whether this instance fetches locally with yt-dlp instead of asking. */
export const localFetch = () => process.env.UMTOOL_LOCAL_FETCH === "1";
diff --git a/umtool/lib/report/serve.mjs b/umtool/lib/report/serve.mjs
@@ -46,7 +46,7 @@ export async function projectCache(project, manifest = null) {
const m = manifest ?? (await readManifest(project.dir));
const shadowExists = await hasShadowChannels(project.dir);
const channelsDir = channelsDirFor(project.dir, m, { shadowExists });
- return rawCacheOf(project.dir, { extraDirs: clipWindowDirs(m, channelsDir) });
+ return rawCacheOf(project.dir, { extraDirs: await clipWindowDirs(m, channelsDir) });
}
/**
diff --git a/umtool/report-to-video/fetch-via-editor.mjs b/umtool/report-to-video/fetch-via-editor.mjs
@@ -57,9 +57,16 @@ function die(message) {
const manifestPath = argv.find((a) => !a.startsWith("-") && a.endsWith(".json"));
const clipId = flag("--fetch-only") ?? flag("--clip");
if (!manifestPath || !clipId) {
- die("usage: fetch-via-editor.mjs <manifest.json> --fetch-only <clipId> [--pad-before N] [--pad-after N] [--progress ndjson]");
+ die("usage: fetch-via-editor.mjs <manifest.json> --fetch-only <clipId> [--full] [--pad-before N] [--pad-after N] [--progress ndjson]");
}
+// THE WHOLE RECORDING INSTEAD OF A WINDOW. For a clip whose windows would tile
+// the entire runtime, or a bench session that wants to re-cut freely without a
+// fetch per attempt. The editor puts it in the SAVED-VIDEO STORE (not clips/),
+// where the retention rule leaves an explicitly-requested container alone, and
+// the pointer it writes beside the video is what `clipWindowDirs` reads back.
+const wantFull = argv.includes("--full");
+
const editorUrl = (process.env.ARCHILYZER_EDITOR_URL ?? DEFAULT_EDITOR).replace(/\/+$/, "");
const token = process.env.WORKER_TOKEN ?? "";
if (!token) {
@@ -140,22 +147,28 @@ async function ask(url, init) {
}
}
-EMIT("fetch", { id: entry.id, video: entry.video, from, to, cached: false });
+EMIT("fetch", {
+ id: entry.id,
+ video: entry.video,
+ ...(wantFull ? { full: true } : { from, to }),
+ cached: false,
+});
const res = await ask(`${editorUrl}/api/media/fetch-window`, {
method: "POST",
headers,
+ // `full` REPLACES the span rather than joining it: the route reads
+ // `full === true` before it validates from/to, and sending both would
+ // describe a request the editor does not have.
body: JSON.stringify({
channelSlug,
videoId: entry.video,
webpageUrl: entry.webpageUrl ?? undefined,
- from,
- to,
+ ...(wantFull ? { full: true } : { from, to, pad: Math.max(padBefore, padAfter) }),
requestedBy: "umtool",
manifest: manifestId,
clipId: entry.id,
reason: String(reason).slice(0, 400),
- pad: Math.max(padBefore, padAfter),
}),
});
