// A cache of the walk, in LMDB. // // HONEST SIZING FIRST. At twelve projects this saves 50 to 150 ms per load. It // is NOT a speed fix today and is not presented as one. What it buys is: // // --since an agent asking what changed is a range read, not a diff of // two full scans // pagination recency as a range read, for when the tree is 500 projects // counts decision counts without running the reducer, which is the // cost that grows fastest -- it is the only part of a project // read that touches megabytes of cue files // // THE STANDING RULE, and it is in the code because it is the one that keeps // lib/browse.ts's "No database. The filesystem is the model." true: // // IF A VALUE EXISTS ONLY IN THE INDEX, THAT IS A BUG. // // Every read verifies a signature against the filesystem and falls back to a // full read when it differs. A stale index self-heals on the next load and the // user sees nothing but latency. An index-first read could claim something the // filesystem does not say; that is the one failure this refuses. import { createHash } from "node:crypto"; import { mkdirSync } from "node:fs"; import path from "node:path"; import { INDEX_DIR } from "../paths.mjs"; /** Bump to invalidate every cached record at once. Folded into every signature. */ // 2: `finalBytes` and the source attrs (sources-checked-at, sources-dead, cue-gaps) // joined the record for the dashboard. export const INDEX_SCHEMA = 2; const NOOP = { ok: false, get: () => null, put: () => {}, del: () => {}, recent: () => [], byKind: () => [], since: () => [], stats: () => ({ ok: false, records: 0, schema: INDEX_SCHEMA, path: null }), close: async () => {}, }; /** * Open the index, or hand back a no-op that answers null to everything. * * Copied in posture from common/lib/channelSignature.ts: a missing or * unopenable index is a normal state (a fresh checkout, a deleted cache, a * different machine), so it degrades rather than throwing. Callers treat null * as "read it from disk". */ export async function openIndex({ readOnly = false, dir = INDEX_DIR } = {}) { let open; try { // Imported lazily and inside the try, so a missing or unbuildable native // module is the "no index yet" path rather than a crash at import time. ({ open } = await import("lmdb")); } catch { return NOOP; } const file = path.join(dir, "projects.mdb"); let root; try { if (!readOnly) mkdirSync(dir, { recursive: true }); root = open({ path: file, maxDbs: 8, compression: false, readOnly }); } catch { return NOOP; } let projects; let meta; let recentDb; try { meta = root.openDB({ name: "meta", encoding: "msgpack" }); projects = root.openDB({ name: "projects", encoding: "msgpack" }); // Key is [MAX - mtimeMs, id], so an ASCENDING range read is newest-first. // The alternative -- reading everything and sorting -- is the thing an index // is supposed to remove. recentDb = root.openDB({ name: "recent", encoding: "msgpack" }); } catch { return NOOP; } const schema = Number(meta.get("schema") ?? 0); if (!readOnly && schema !== INDEX_SCHEMA) { // A schema bump rewrites everything rather than migrating: the whole store // is derived, and CACHE_DIR is documented as safe to delete at any time. try { projects.clearSync(); recentDb.clearSync(); meta.putSync("schema", INDEX_SCHEMA); meta.putSync("builtAt", 0); } catch { return NOOP; } } else if (schema !== INDEX_SCHEMA) { return NOOP; } const MAX = 9_999_999_999_999; const recentKey = (rec) => [MAX - Math.round(rec.newestMtimeMs ?? 0), rec.id]; return { ok: true, file, /** The cached record for an id, or null. Callers still verify the sig. */ get(id) { try { return projects.get(id) ?? null; } catch { return null; } }, /** Best effort, always. A failed write is a cache miss next time, no more. */ put(rec) { try { const old = projects.get(rec.id); if (old) recentDb.removeSync(recentKey(old)); projects.putSync(rec.id, rec); recentDb.putSync(recentKey(rec), rec.id); meta.putSync("builtAt", Date.now()); } catch { /* the filesystem is the model; this is only a cache */ } }, del(id) { try { const old = projects.get(id); if (old) recentDb.removeSync(recentKey(old)); projects.removeSync(id); } catch { /* ignore */ } }, /** Newest first, as a range read rather than a sort. */ recent(limit = 50, offset = 0) { try { const out = []; let i = 0; for (const { value } of recentDb.getRange({})) { if (i++ < offset) continue; const rec = projects.get(value); if (rec) out.push(rec); if (out.length >= limit) break; } return out; } catch { return []; } }, byKind(kind) { try { return [...projects.getRange({})].map((e) => e.value).filter((r) => r?.kind === kind); } catch { return []; } }, /** What changed since a timestamp -- the reason this exists at all. */ since(ms) { try { const out = []; for (const { value } of recentDb.getRange({})) { const rec = projects.get(value); if (!rec) continue; // The range is newest-first, so the first record older than the cutoff // ends it. if ((rec.newestMtimeMs ?? 0) <= ms) break; out.push(rec); } return out; } catch { return []; } }, stats() { try { let records = 0; for (const _ of projects.getRange({})) records += 1; return { ok: true, records, schema: INDEX_SCHEMA, builtAt: Number(meta.get("builtAt") ?? 0), path: file, }; } catch { return { ok: false, records: 0, schema: INDEX_SCHEMA, path: file }; } }, async close() { try { await root.close(); } catch { /* ignore */ } }, }; } /** * The freshness signature. INPUTS, never bytes. * * Signing the produced record instead would be circular, and signing bytes is * what the export build learned not to do: an artefact with a timestamp in it is * never byte-reproducible. The schema is folded in so a bump invalidates * everything -- deliberately NOT a generation counter, which would invalidate * every project whenever any one of them changed. */ export function signRecord({ kindSig, dirMs, markerMs, markerSize, outMs }) { const h = createHash("sha1"); h.update(`schema:${INDEX_SCHEMA}\n`); h.update(`kind:${kindSig ?? ""}\n`); h.update(`dir:${Math.round(dirMs ?? 0)}\n`); h.update(`marker:${Math.round(markerMs ?? 0)}:${markerSize ?? 0}\n`); h.update(`out:${Math.round(outMs ?? 0)}\n`); return h.digest("hex").slice(0, 16); }