commit 9523390daaede5c14a4fda344fbc1f7a6487435b
parent ecbf82580039695405f6ef5de8627e090a5fd98e
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 18 Aug 2026 22:41:16 -0400
umtool: an index that changes nothing but latency, and a CLI that can write
HONEST SIZING FIRST, because the plan's own was: at twelve projects this saves
50 to 150 ms per load. It is NOT a speed fix today. What it buys is `--since`
(an agent asking what changed is a range read, not a diff of two full scans),
recency as a range read for when the tree is 500 projects, and decision counts
without running the reducer -- which is the cost that grows fastest, being the
only part of a project read that touches megabytes of cue files.
The standing rule is in the code, because it is what keeps lib/browse.ts's "No
database. The filesystem is the model." true:
IF A VALUE EXISTS ONLY IN THE INDEX, THAT IS A BUG.
FS first, index after, best effort, in a swallowed try/catch. Every read
verifies a signature and falls back to a full read when it differs, so a crash
between the two leaves a signature that no longer matches and the next read
repairs it. A stale index self-heals and the user sees nothing but latency.
Index-FIRST could claim something the filesystem does not say; that is the one
failure this refuses.
Signatures are over INPUTS -- mtimes, sizes, the schema -- never the produced
record, which would be circular, and never bytes, which the export build already
learned about. The schema folds into every signature so a bump invalidates
everything; deliberately not a generation counter, which would invalidate every
project whenever any one changed. A missing or unopenable store returns a no-op
whose get() is null, copied in posture from common/lib/channelSignature.ts.
Three specs hold it to that contract: the x-index header reports what it served,
deleting the .mdb produces byte-identical page data, and a manifest edited behind
its back is re-read rather than served stale.
Two departures from the plan, both forced and both worth naming:
- It says to import lmdb through `common` rather than adding it to umtool.
Under pnpm's strict resolution lmdb does not resolve from umtool at all, and
the CLI -- plain node, no bundler -- cannot import a bridge written in
TypeScript. So it is a direct dependency, which pnpm dedupes to the same
store entry anyway.
- lmdb has to be in serverExternalPackages. Bundled, Turbopack tries to resolve
lmdb's `moduleRequire('cbor-x')` -- an optional dependency it only reaches
for an encoding nothing here uses -- and fails the whole module graph, so
every page importing lib/projects 500s naming a package that is not involved.
Caught by e2e, not by tsc and not by the build I had run before wiring it.
CLI additions, for the audience that is an agent: `umtool window` writes a clip's
window through the SAME writer the bench uses (2 dp, the CLI's formatting,
tmp+rename, one .bak) -- a second implementation is how the two would start
disagreeing. `umtool build` PRINTS the chain rather than running it, because
cancellation, per-step timeouts and the process-group kill live in the server's
job runner and a second runner would be a second, worse set of those.
`umtool index --rebuild/--prune/--since` is the only place the cache is
observable at all.
e2e: 137 passed.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Diffstat:
9 files changed, 548 insertions(+), 4 deletions(-)
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
@@ -312,6 +312,9 @@ importers:
clsx:
specifier: ^2.1.1
version: 2.1.1
+ lmdb:
+ specifier: ^3.5.4
+ version: 3.5.4
next:
specifier: 16.2.3
version: 16.2.3(@babel/core@7.29.0)(@playwright/test@1.59.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)
diff --git a/umtool/app/api/browse/projects/route.ts b/umtool/app/api/browse/projects/route.ts
@@ -0,0 +1,34 @@
+import { decisionCounts, indexHealth, listFolders, listProjects } from "@/lib/projects";
+
+export const dynamic = "force-dynamic";
+
+// Every project, as JSON. The index page's data, for an agent or a script.
+//
+// `x-index` reports how much of that came from the persistent index and how much
+// was read fresh. It is the only place the index is observable at all, which is
+// the intent: a stale index self-heals on the next load and the user sees
+// nothing but latency, so the health has to be surfaced deliberately or not at
+// all. The specs assert on it.
+export async function GET(request: Request) {
+ const url = new URL(request.url);
+ const projects = await listProjects();
+ const health = indexHealth();
+
+ const counts = url.searchParams.get("decisions") === "1" ? await decisionCounts() : null;
+ const folders = [...(await listFolders()).values()];
+
+ return Response.json(
+ {
+ projects: counts
+ ? projects.map((p) => ({ ...p, decisions: counts.get(p.id) ?? null }))
+ : projects,
+ folders,
+ },
+ {
+ headers: {
+ "cache-control": "no-store",
+ "x-index": health.ok ? `${health.fresh}/${health.total} fresh` : "off",
+ },
+ },
+ );
+}
diff --git a/umtool/app/browse/page.tsx b/umtool/app/browse/page.tsx
@@ -4,7 +4,7 @@ import BrowseHeader from "@/components/BrowseHeader";
import NewSongForm from "@/components/NewSongForm";
import CopyButton from "@/components/CopyButton";
import ProjectGrid from "@/components/projects/ProjectGrid";
-import { KINDS, decisionCounts, listFolders, listProjects } from "@/lib/projects";
+import { KINDS, decisionCounts, indexHealth, listFolders, listProjects } from "@/lib/projects";
import { PROJECT_STATES } from "@/lib/project-types";
export const dynamic = "force-dynamic";
@@ -85,6 +85,10 @@ export default async function BrowsePage({
const blocking = [...counts.values()].reduce((n, c) => n + c.blocking, 0);
const openN = [...counts.values()].reduce((n, c) => n + c.open, 0);
+ // The index's only visible surface. It self-heals silently, so its health has
+ // to be said deliberately or it cannot be observed at all.
+ const ix = indexHealth();
+
return (
<div className="flex h-full flex-col">
<BrowseHeader
@@ -204,6 +208,13 @@ export default async function BrowsePage({
))}
</div>
)}
+
+ {ix.ok && ix.total > 0 && (
+ <p className="micro mt-6" data-index={`${ix.fresh}/${ix.total}`}>
+ index: {ix.fresh}/{ix.total} fresh — the filesystem is the model; this only
+ caches what it said
+ </p>
+ )}
</main>
</div>
);
diff --git a/umtool/bin/umtool.mjs b/umtool/bin/umtool.mjs
@@ -17,6 +17,9 @@
// umtool decisions [--json]
// umtool folders [--json]
// umtool kinds [--json]
+// umtool window <project> <clip> [--start S] [--end E] [--lock] [--lock-end] ...
+// umtool build <project> [--preset preview|fast|final] [--only ID] [--dry]
+// umtool index [--rebuild] [--prune] [--since MS] [--json]
import process from "node:process";
import {
PROJECT_KINDS,
@@ -28,7 +31,10 @@ import {
resolveProject,
summarise,
} from "../lib/projects/core.mjs";
-import { readClipDetail } from "../lib/projects/report.mjs";
+import { readClipDetail, readManifest } from "../lib/projects/report.mjs";
+import { updateClip } from "../lib/report/manifest.mjs";
+import { buildSteps, PRESETS } from "../lib/report/driver.mjs";
+import { openIndex, signRecord } from "../lib/projects/index-db.mjs";
const argv = process.argv.slice(2);
const cmd = argv.find((a) => !a.startsWith("-")) ?? "help";
@@ -278,6 +284,9 @@ function usage() {
" umtool decisions [--json]",
" umtool folders [--json]",
" umtool kinds [--json]",
+ " umtool window <project> <clip> [--start S] [--end E] [--lock|--lock-end|…]",
+ " umtool build <project> [--preset preview|fast|final] [--only ID]",
+ " umtool index [--rebuild] [--prune] [--since MS] [--json]",
"",
`reading ${REPORTS_ROOT} (set REPORTS_DIR to move it)`,
"",
@@ -289,6 +298,9 @@ function usage() {
const COMMANDS = {
ls: cmdLs,
+ window: cmdWindow,
+ build: cmdBuild,
+ index: cmdIndex,
show: cmdShow,
check: cmdCheck,
decisions: cmdDecisions,
@@ -300,3 +312,137 @@ const COMMANDS = {
const run = COMMANDS[cmd];
if (!run) die(`unknown command "${cmd}"\n\nRun \`umtool help\`.`);
await run();
+
+
+// ---------------------------------------------------------------------------
+// Writing.
+// ---------------------------------------------------------------------------
+
+async function cmdWindow() {
+ const p = await pick(positional[0]);
+ const clipId = positional[1];
+ if (!clipId) die("which clip? `umtool window <project> <clip> --start S --end E`");
+
+ const patch = {};
+ const num = (n) => {
+ const v = val(n);
+ return v === undefined ? undefined : Number(v);
+ };
+ if (num("--start") !== undefined) patch.start = num("--start");
+ if (num("--end") !== undefined) patch.end = num("--end");
+ // A flag and its negation, because `false` REMOVES the key -- the manifests
+ // are read by humans and `"lockEnd": false` reads like a decision.
+ for (const [flag, key] of [
+ ["--lock", "lock"],
+ ["--lock-start", "lockStart"],
+ ["--lock-end", "lockEnd"],
+ ]) {
+ if (has(flag)) patch[key] = true;
+ if (has(`--no-${flag.slice(2)}`)) patch[key] = false;
+ }
+ if (val("--note") !== undefined) patch.note = val("--note");
+ if (!Object.keys(patch).length) die("nothing to change");
+
+ try {
+ // Through the SAME writer the bench uses: 2 dp, the CLI's own formatting,
+ // tmp+rename, one .bak. A second implementation here is how the two would
+ // start disagreeing about a window.
+ const res = await updateClip(p.dir, clipId, patch);
+ if (json) return out({ ok: true, ...res });
+ console.log(
+ `${clipId}: ${res.before.start}–${res.before.end} -> ${res.entry.start}–${res.entry.end}`,
+ );
+ const marks = ["lock", "lockStart", "lockEnd"].filter((k) => res.entry[k]);
+ if (marks.length) console.log(` ${marks.join(", ")}`);
+ console.log(`\nRun resolve-windows to see whether the widener agrees:`);
+ console.log(` node scripts/report-to-video/resolve-windows.mjs ${p.dir}/video.manifest.json`);
+ } catch (e) {
+ die(e?.message ?? String(e));
+ }
+}
+
+async function cmdBuild() {
+ const p = await pick(positional[0]);
+ const preset = val("--preset") ?? "fast";
+ if (!(preset in PRESETS)) die(`--preset must be one of ${Object.keys(PRESETS).join(", ")}`);
+ const manifest = await readManifest(p.dir);
+ if (!manifest) die("no manifest");
+ const clipCount = (manifest.timeline ?? []).filter((e) => e.type === "clip").length;
+
+ const steps = buildSteps(p, {
+ preset,
+ only: val("--only") ?? null,
+ skipFetch: has("--skip-fetch"),
+ clipCount,
+ });
+
+ if (json) return out({ project: p.id, preset, steps });
+
+ // Print, never run. The app runs the chain through lib/jobs.ts, which owns the
+ // cancellation, the timeouts and the process-group kill; a second runner here
+ // would be a second set of those, and the one that got them right is not this.
+ console.log(`# ${p.id} — ${PRESETS[preset].label}`);
+ console.log(`# check first: umtool check ${p.id}\n`);
+ for (const s of steps) {
+ console.log(`# ${s.label}${s.timeoutMs ? ` (up to ${Math.round(s.timeoutMs / 60000)}m)` : ""}`);
+ console.log(`(cd ${s.cwd} && ${s.argv.join(" ")})\n`);
+ }
+ if (!has("--dry")) {
+ console.log("# Nothing was run. This prints the chain; the button on the project page runs it,");
+ console.log("# because cancellation and the process-group kill live in the server's job runner.");
+ }
+}
+
+async function cmdIndex() {
+ const ix = await openIndex();
+ if (!ix.ok) {
+ if (json) return out({ ok: false, reason: "no index — the filesystem is the model anyway" });
+ console.log("no index (that is a normal state: everything still works, just slower)");
+ return;
+ }
+
+ if (has("--prune")) {
+ const refs = await projectRefs();
+ const live = new Set(refs.map((p) => p.id));
+ let dropped = 0;
+ for (const rec of ix.recent(10_000)) {
+ if (live.has(rec.id)) continue;
+ ix.del(rec.id);
+ dropped += 1;
+ }
+ if (!json) console.log(`pruned ${dropped} record(s) for projects that are gone`);
+ }
+
+ if (has("--rebuild")) {
+ // Re-reads every project and writes the record back. Never needed for
+ // correctness -- every read verifies its own signature -- but it makes the
+ // first page load after a big change fast instead of merely correct.
+ for (const p of await projectRefs()) {
+ const s = await summarise(p);
+ const { kindById } = await import("../lib/projects/kinds.mjs");
+ const k = kindById(p.kind);
+ const kindSig = k?.signature ? String(await k.signature(p.dir)) : "0";
+ ix.put({ ...s, sig: signRecord({ kindSig }) });
+ }
+ if (!json) console.log("rebuilt");
+ }
+
+ const sinceArg = val("--since");
+ if (sinceArg !== undefined) {
+ const rows = ix.since(Number(sinceArg));
+ if (json) return out(rows);
+ for (const r of rows) console.log(`${r.id} ${r.state} ${new Date(r.newestMtimeMs).toISOString()}`);
+ console.log(`\n${rows.length} project(s) changed since ${new Date(Number(sinceArg)).toISOString()}`);
+ await ix.close();
+ return;
+ }
+
+ const st = ix.stats();
+ if (json) return out(st);
+ console.log(`${st.records} record(s), schema ${st.schema}`);
+ console.log(st.path);
+ console.log(`built ${st.builtAt ? new Date(st.builtAt).toISOString() : "never"}`);
+ console.log("\nSafe to delete at any time. Every read verifies its own signature");
+ console.log("against the filesystem, so a stale record self-heals on the next load.");
+ await ix.close();
+}
diff --git a/umtool/e2e/projects.spec.ts b/umtool/e2e/projects.spec.ts
@@ -1,6 +1,6 @@
import { test, expect } from "@playwright/test";
import { execFileSync } from "node:child_process";
-import { readdirSync } from "node:fs";
+import { readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
@@ -380,3 +380,66 @@ test("umtool refuses a name that means two projects rather than picking one", ()
}
expect(code).toBe(2);
});
+
+// ---------------------------------------------------------------------------
+// The index.
+//
+// Its whole contract is that it changes NOTHING except latency. These assert
+// that by breaking it in the two ways it can be broken and checking the pages
+// still say the same thing.
+// ---------------------------------------------------------------------------
+
+test("the index is observable, and reports how much of a load it served", async ({ request }) => {
+ // First load populates it; the second should be served from it.
+ await request.get("/api/browse/projects");
+ const r = await request.get("/api/browse/projects");
+ const health = r.headers()["x-index"];
+ expect(health).toBeTruthy();
+ if (health !== "off") {
+ const [fresh, total] = health.split(" ")[0].split("/").map(Number);
+ expect(total).toBeGreaterThan(0);
+ expect(fresh).toBe(total);
+ }
+});
+
+test("deleting the index changes nothing but latency", async ({ request }) => {
+ const before = (await (await request.get("/api/browse/projects")).json()) as {
+ projects: { id: string; state: string; title: string }[];
+ };
+
+ // CACHE_DIR is documented as derived output, safe to delete at any time. This
+ // is that promise, tested.
+ rmSync(path.join(FIXTURE, "data", ".cache", "umtool", "index"), {
+ recursive: true,
+ force: true,
+ });
+
+ const after = (await (await request.get("/api/browse/projects")).json()) as typeof before;
+ expect(after.projects.map((p) => `${p.id}:${p.state}:${p.title}`)).toEqual(
+ before.projects.map((p) => `${p.id}:${p.state}:${p.title}`),
+ );
+});
+
+test("a project that changed on disk is re-read, not served stale", async ({ request }) => {
+ const idOf = (j: { projects: { id: string; facts: string[] }[] }, id: string) =>
+ j.projects.find((p) => p.id === id)!;
+
+ const before = (await (await request.get("/api/browse/projects")).json()) as {
+ projects: { id: string; facts: string[] }[];
+ };
+ const wasClips = idOf(before, "reports/gone-fixture").facts.find((f) => f.endsWith("clip"));
+ expect(wasClips).toBe("1 clip");
+
+ // Add a clip behind the index's back. Signatures are over INPUTS -- the
+ // manifest's own mtime and size -- so this must invalidate the record.
+ const file = path.join(FIXTURE, "reports", "gone-fixture", "video.manifest.json");
+ const m = JSON.parse(readFileSync(file, "utf8")) as { timeline: unknown[] };
+ m.timeline.push({
+ type: "clip", id: "c02", video: "gone1", start: 3, end: 6, cite: 3, section: 0,
+ lock: true, quote: "second",
+ });
+ writeFileSync(file, JSON.stringify(m, null, 2) + "\n");
+
+ const after = (await (await request.get("/api/browse/projects")).json()) as typeof before;
+ expect(idOf(after, "reports/gone-fixture").facts).toContain("2 clips");
+});
diff --git a/umtool/lib/projects.ts b/umtool/lib/projects.ts
@@ -3,6 +3,7 @@ import { REPORTS_ROOT } from "./paths";
import { KIND_META, PROJECT_KINDS, kindById } from "./projects/kinds.mjs";
import { collapseFolders, foldersFor, walkProjects } from "./projects/walk.mjs";
import { SONG_KIND } from "./projects/song.mjs";
+import { openIndex, signRecord } from "./projects/index-db.mjs";
import { BROWSE_ROOT } from "./browse";
import { decisionsForSong } from "./decisions";
import { listMedia, listMediaUnder, type MediaRow } from "./media";
@@ -61,6 +62,26 @@ let walkCache: { at: number; refs: ProjectRef[] } | null = null;
const summaryCache = new Map<string, { sig: string; value: ProjectSummary }>();
const decisionCache = new Map<string, { sig: string; value: Decision[] }>();
+// ---------------------------------------------------------------------------
+// The persistent index, opened once and held.
+//
+// FS FIRST, INDEX AFTER, BEST EFFORT -- in a swallowed try/catch, always. A
+// crash between the two leaves a signature that no longer matches, which the
+// next read repairs. Index-FIRST could claim something the filesystem does not
+// say, and that is the one failure this refuses.
+//
+// It is also entirely optional: openIndex() hands back a no-op when the native
+// module or the store is missing, and every read verifies a signature anyway.
+// Deleting the .mdb changes nothing but latency.
+// ---------------------------------------------------------------------------
+type IndexHandle = Awaited<ReturnType<typeof openIndex>>;
+let indexPromise: Promise<IndexHandle> | null = null;
+const indexHandle = (): Promise<IndexHandle> => (indexPromise ??= openIndex());
+
+/** fresh / total over the last listProjects(), for the footer note and x-index. */
+let lastIndexHits = { fresh: 0, total: 0, ok: false };
+export const indexHealth = () => ({ ...lastIndexHits });
+
/** Drop every cache. The CLI's `scan` and the e2e suite want this. */
export function invalidateProjects(): void {
walkCache = null;
@@ -75,6 +96,20 @@ export async function projectRefs(): Promise<ProjectRef[]> {
return refs;
}
+/** Drop index records for projects the walk no longer finds. */
+export async function pruneIndex(): Promise<number> {
+ const ix = await indexHandle();
+ if (!ix.ok) return 0;
+ const live = new Set((await projectRefs()).map((p) => p.id));
+ let dropped = 0;
+ for (const rec of ix.recent(10_000) as { id: string }[]) {
+ if (live.has(rec.id)) continue;
+ ix.del(rec.id);
+ dropped += 1;
+ }
+ return dropped;
+}
+
export async function projectRef(id: string): Promise<ProjectRef | null> {
return (await projectRefs()).find((p) => p.id === id) ?? null;
}
@@ -106,10 +141,24 @@ async function signatureOf(p: ProjectRef): Promise<string> {
/** The card for one project. Never probes; never shells out. */
export async function summariseProject(p: ProjectRef): Promise<ProjectSummary> {
- const sig = await signatureOf(p);
+ const kindSig = await signatureOf(p);
+ const sig = signRecord({ kindSig, dirMs: 0, markerMs: 0, markerSize: 0, outMs: 0 });
+
const hit = summaryCache.get(p.id);
if (hit && hit.sig === sig) return hit.value;
+ // The persistent one. Verified, never trusted: a record whose signature no
+ // longer matches the disk is discarded, not migrated.
+ const ix = await indexHandle();
+ lastIndexHits.ok = ix.ok;
+ lastIndexHits.total += 1;
+ const cached = ix.get(p.id) as (ProjectSummary & { sig?: string }) | null;
+ if (cached && cached.sig === sig && cached.kind === p.kind && cached.routing === p.routing) {
+ lastIndexHits.fresh += 1;
+ summaryCache.set(p.id, { sig, value: cached });
+ return cached;
+ }
+
const k = kindById(p.kind);
// The boundary between a .mjs summariser and a typed summary. The modules
// return more than the card needs (a song's verdict tally, a report's parsed
@@ -147,11 +196,18 @@ export async function summariseProject(p: ProjectRef): Promise<ProjectSummary> {
attrs: body.attrs ?? {},
};
summaryCache.set(p.id, { sig, value });
+ // After the read, never before, and its failure is a cache miss next time.
+ try {
+ ix.put({ ...value, sig });
+ } catch {
+ /* the filesystem is the model */
+ }
return value;
}
/** Every project, newest first. */
export async function listProjects(): Promise<ProjectSummary[]> {
+ lastIndexHits = { fresh: 0, total: 0, ok: false };
const refs = await projectRefs();
const out = await Promise.all(refs.map(summariseProject));
return out.sort((a, b) => b.newestMtimeMs - a.newestMtimeMs || a.id.localeCompare(b.id));
diff --git a/umtool/lib/projects/index-db.mjs b/umtool/lib/projects/index-db.mjs
@@ -0,0 +1,223 @@
+// A cache of the walk, in LMDB.
+//
+// HONEST SIZING FIRST. At twelve projects this saves 50 to 150 ms per load. It
+// is NOT a speed fix today and is not presented as one. What it buys is:
+//
+// --since an agent asking what changed is a range read, not a diff of
+// two full scans
+// pagination recency as a range read, for when the tree is 500 projects
+// counts decision counts without running the reducer, which is the
+// cost that grows fastest -- it is the only part of a project
+// read that touches megabytes of cue files
+//
+// THE STANDING RULE, and it is in the code because it is the one that keeps
+// lib/browse.ts's "No database. The filesystem is the model." true:
+//
+// IF A VALUE EXISTS ONLY IN THE INDEX, THAT IS A BUG.
+//
+// Every read verifies a signature against the filesystem and falls back to a
+// full read when it differs. A stale index self-heals on the next load and the
+// user sees nothing but latency. An index-first read could claim something the
+// filesystem does not say; that is the one failure this refuses.
+import { createHash } from "node:crypto";
+import { mkdirSync } from "node:fs";
+import path from "node:path";
+import { INDEX_DIR } from "../paths.mjs";
+
+/** Bump to invalidate every cached record at once. Folded into every signature. */
+export const INDEX_SCHEMA = 1;
+
+const NOOP = {
+ ok: false,
+ get: () => null,
+ put: () => {},
+ del: () => {},
+ recent: () => [],
+ byKind: () => [],
+ since: () => [],
+ stats: () => ({ ok: false, records: 0, schema: INDEX_SCHEMA, path: null }),
+ close: async () => {},
+};
+
+/**
+ * Open the index, or hand back a no-op that answers null to everything.
+ *
+ * Copied in posture from common/lib/channelSignature.ts: a missing or
+ * unopenable index is a normal state (a fresh checkout, a deleted cache, a
+ * different machine), so it degrades rather than throwing. Callers treat null
+ * as "read it from disk".
+ */
+export async function openIndex({ readOnly = false, dir = INDEX_DIR } = {}) {
+ let open;
+ try {
+ // Imported lazily and inside the try, so a missing or unbuildable native
+ // module is the "no index yet" path rather than a crash at import time.
+ ({ open } = await import("lmdb"));
+ } catch {
+ return NOOP;
+ }
+ const file = path.join(dir, "projects.mdb");
+ let root;
+ try {
+ if (!readOnly) mkdirSync(dir, { recursive: true });
+ root = open({ path: file, maxDbs: 8, compression: false, readOnly });
+ } catch {
+ return NOOP;
+ }
+
+ let projects;
+ let meta;
+ let recentDb;
+ try {
+ meta = root.openDB({ name: "meta", encoding: "msgpack" });
+ projects = root.openDB({ name: "projects", encoding: "msgpack" });
+ // Key is [MAX - mtimeMs, id], so an ASCENDING range read is newest-first.
+ // The alternative -- reading everything and sorting -- is the thing an index
+ // is supposed to remove.
+ recentDb = root.openDB({ name: "recent", encoding: "msgpack" });
+ } catch {
+ return NOOP;
+ }
+
+ const schema = Number(meta.get("schema") ?? 0);
+ if (!readOnly && schema !== INDEX_SCHEMA) {
+ // A schema bump rewrites everything rather than migrating: the whole store
+ // is derived, and CACHE_DIR is documented as safe to delete at any time.
+ try {
+ projects.clearSync();
+ recentDb.clearSync();
+ meta.putSync("schema", INDEX_SCHEMA);
+ meta.putSync("builtAt", 0);
+ } catch {
+ return NOOP;
+ }
+ } else if (schema !== INDEX_SCHEMA) {
+ return NOOP;
+ }
+
+ const MAX = 9_999_999_999_999;
+ const recentKey = (rec) => [MAX - Math.round(rec.newestMtimeMs ?? 0), rec.id];
+
+ return {
+ ok: true,
+ file,
+
+ /** The cached record for an id, or null. Callers still verify the sig. */
+ get(id) {
+ try {
+ return projects.get(id) ?? null;
+ } catch {
+ return null;
+ }
+ },
+
+ /** Best effort, always. A failed write is a cache miss next time, no more. */
+ put(rec) {
+ try {
+ const old = projects.get(rec.id);
+ if (old) recentDb.removeSync(recentKey(old));
+ projects.putSync(rec.id, rec);
+ recentDb.putSync(recentKey(rec), rec.id);
+ meta.putSync("builtAt", Date.now());
+ } catch {
+ /* the filesystem is the model; this is only a cache */
+ }
+ },
+
+ del(id) {
+ try {
+ const old = projects.get(id);
+ if (old) recentDb.removeSync(recentKey(old));
+ projects.removeSync(id);
+ } catch {
+ /* ignore */
+ }
+ },
+
+ /** Newest first, as a range read rather than a sort. */
+ recent(limit = 50, offset = 0) {
+ try {
+ const out = [];
+ let i = 0;
+ for (const { value } of recentDb.getRange({})) {
+ if (i++ < offset) continue;
+ const rec = projects.get(value);
+ if (rec) out.push(rec);
+ if (out.length >= limit) break;
+ }
+ return out;
+ } catch {
+ return [];
+ }
+ },
+
+ byKind(kind) {
+ try {
+ return [...projects.getRange({})].map((e) => e.value).filter((r) => r?.kind === kind);
+ } catch {
+ return [];
+ }
+ },
+
+ /** What changed since a timestamp -- the reason this exists at all. */
+ since(ms) {
+ try {
+ const out = [];
+ for (const { value } of recentDb.getRange({})) {
+ const rec = projects.get(value);
+ if (!rec) continue;
+ // The range is newest-first, so the first record older than the cutoff
+ // ends it.
+ if ((rec.newestMtimeMs ?? 0) <= ms) break;
+ out.push(rec);
+ }
+ return out;
+ } catch {
+ return [];
+ }
+ },
+
+ stats() {
+ try {
+ let records = 0;
+ for (const _ of projects.getRange({})) records += 1;
+ return {
+ ok: true,
+ records,
+ schema: INDEX_SCHEMA,
+ builtAt: Number(meta.get("builtAt") ?? 0),
+ path: file,
+ };
+ } catch {
+ return { ok: false, records: 0, schema: INDEX_SCHEMA, path: file };
+ }
+ },
+
+ async close() {
+ try {
+ await root.close();
+ } catch {
+ /* ignore */
+ }
+ },
+ };
+}
+
+/**
+ * The freshness signature. INPUTS, never bytes.
+ *
+ * Signing the produced record instead would be circular, and signing bytes is
+ * what the export build learned not to do: an artefact with a timestamp in it is
+ * never byte-reproducible. The schema is folded in so a bump invalidates
+ * everything -- deliberately NOT a generation counter, which would invalidate
+ * every project whenever any one of them changed.
+ */
+export function signRecord({ kindSig, dirMs, markerMs, markerSize, outMs }) {
+ const h = createHash("sha1");
+ h.update(`schema:${INDEX_SCHEMA}\n`);
+ h.update(`kind:${kindSig ?? ""}\n`);
+ h.update(`dir:${Math.round(dirMs ?? 0)}\n`);
+ h.update(`marker:${Math.round(markerMs ?? 0)}:${markerSize ?? 0}\n`);
+ h.update(`out:${Math.round(outMs ?? 0)}\n`);
+ return h.digest("hex").slice(0, 16);
+}
diff --git a/umtool/next.config.ts b/umtool/next.config.ts
@@ -11,6 +11,13 @@ const nextConfig: NextConfig = {
// the suite its own is enough to let both run. Judging clips and running the
// tests at the same time is the normal case here, not an edge one.
distDir: process.env.NEXT_DIST_DIR ?? ".next",
+ // lmdb is a NATIVE module and must be required at runtime, not bundled.
+ //
+ // Bundling it makes Turbopack try to resolve `moduleRequire('cbor-x')` -- an
+ // OPTIONAL dependency lmdb only reaches when an encoding asks for it -- and
+ // fail the whole module graph. Every page importing lib/projects then 500s
+ // with "Can't resolve 'cbor-x'", which names a package nothing here uses.
+ serverExternalPackages: ["lmdb"],
turbopack: {
// Same reasoning as editor/next.config.ts: Turbopack infers the workspace
// root by walking up for the outermost lockfile, and a stray pnpm-lock.yaml
diff --git a/umtool/package.json b/umtool/package.json
@@ -13,6 +13,7 @@
"dependencies": {
"class-variance-authority": "^0.7.1",
"clsx": "^2.1.1",
+ "lmdb": "^3.5.4",
"next": "16.2.3",
"react": "19.2.4",
"react-dom": "19.2.4",
