import path from "node:path"; import { REPORTS_ROOT } from "./paths"; import { KIND_META, PROJECT_KINDS, kindById } from "./projects/kinds.mjs"; import { collapseFolders, foldersFor, walkProjects } from "./projects/walk.mjs"; import { SONG_KIND } from "./projects/song.mjs"; import { openIndex, signRecord } from "./projects/index-db.mjs"; import { BROWSE_ROOT } from "./browse"; import { decisionsForSong, noteDecisions } from "./decisions"; import { listMedia, listMediaUnder, type MediaRow } from "./media"; import type { Decision } from "./decisions"; import type { FolderNode, ProjectKindMeta, ProjectRef, ProjectState, ProjectSummary, } from "./project-types"; export type { FolderNode, ProjectKindMeta, ProjectRef, ProjectState, ProjectSummary }; export { PROJECT_STATES, STATE_LABEL } from "./project-types"; // --------------------------------------------------------------------------- // The app's view of the registry. // // SERVER ONLY. lib/projects/kinds.mjs reaches the disk through its per-kind // modules, so importing it from a client component drags `node:fs` into the // browser bundle -- which this repo has already been bitten by once: it passed // `tsc --noEmit` and then 500'd every page. A client component that wants kind // labels or state names imports lib/project-types.ts and is handed the rest as // props. `pnpm build`, not typecheck, is what catches a regression here. // --------------------------------------------------------------------------- export const KINDS: ProjectKindMeta[] = KIND_META(); /** * Which kinds emit which decisions, and the union of all of them. * * Assembled rather than hand-written, so a kind's vocabulary lives with the * kind. A closed union in lib/decisions.ts would put every kind's words in one * shared file -- exactly the coupling the registry exists to remove. */ /** Emitted by the walk rather than by any kind: how a project is reachable. */ export const ROUTING_DECISION_KINDS = ["shadowed-name", "unroutable-name", "ambiguous-project"]; export const ALL_DECISION_KINDS: string[] = [ ...new Set([...KINDS.flatMap((k) => k.decisionKinds), ...ROUTING_DECISION_KINDS]), ].sort(); // --------------------------------------------------------------------------- // Caching. // // The WALK is memoised for a second: it is 25 ms on the real tree, and a page // render asks for it two or three times. // // A SUMMARY is memoised against its own kind's signature -- a tuple of mtimes // and sizes, never the bytes -- so it survives for as long as the project has // not changed and is thrown away the moment it has. That is the same rule the // export build's incremental signatures follow, and it is the reason no index // is needed yet at twelve projects. // --------------------------------------------------------------------------- let walkCache: { at: number; refs: ProjectRef[] } | null = null; const summaryCache = new Map(); const decisionCache = new Map(); // --------------------------------------------------------------------------- // The persistent index, opened once and held. // // FS FIRST, INDEX AFTER, BEST EFFORT -- in a swallowed try/catch, always. A // crash between the two leaves a signature that no longer matches, which the // next read repairs. Index-FIRST could claim something the filesystem does not // say, and that is the one failure this refuses. // // It is also entirely optional: openIndex() hands back a no-op when the native // module or the store is missing, and every read verifies a signature anyway. // Deleting the .mdb changes nothing but latency. // --------------------------------------------------------------------------- type IndexHandle = Awaited>; let indexPromise: Promise | null = null; const indexHandle = (): Promise => (indexPromise ??= openIndex()); /** fresh / total over the last listProjects(), for the footer note and x-index. */ let lastIndexHits = { fresh: 0, total: 0, ok: false }; export const indexHealth = () => ({ ...lastIndexHits }); /** Drop every cache. The CLI's `scan` and the e2e suite want this. */ export function invalidateProjects(): void { walkCache = null; summaryCache.clear(); decisionCache.clear(); } export async function projectRefs(): Promise { if (walkCache && Date.now() - walkCache.at < 1000) return walkCache.refs; const refs = (await walkProjects(REPORTS_ROOT)) as ProjectRef[]; walkCache = { at: Date.now(), refs }; return refs; } /** Drop index records for projects the walk no longer finds. */ export async function pruneIndex(): Promise { const ix = await indexHandle(); if (!ix.ok) return 0; const live = new Set((await projectRefs()).map((p) => p.id)); let dropped = 0; for (const rec of ix.recent(10_000) as { id: string }[]) { if (live.has(rec.id)) continue; ix.del(rec.id); dropped += 1; } return dropped; } export async function projectRef(id: string): Promise { return (await projectRefs()).find((p) => p.id === id) ?? null; } type KindSummary = { title?: string; subtitle?: string | null; state?: string; newestMtimeMs?: number; facts?: string[]; flags?: string[]; posterRel?: string | null; haystack?: string; attrs?: Record; }; type Ctx = ProjectRef & { root: string }; const ctxFor = (p: ProjectRef): Ctx => ({ ...p, root: REPORTS_ROOT }); async function signatureOf(p: ProjectRef): Promise { const k = kindById(p.kind); if (!k?.signature) return "0"; try { return String(await k.signature(p.dir)); } catch { return "0"; } } /** The card for one project. Never probes; never shells out. */ export async function summariseProject(p: ProjectRef): Promise { const kindSig = await signatureOf(p); const sig = signRecord({ kindSig, dirMs: 0, markerMs: 0, markerSize: 0, outMs: 0 }); const hit = summaryCache.get(p.id); if (hit && hit.sig === sig) return hit.value; // The persistent one. Verified, never trusted: a record whose signature no // longer matches the disk is discarded, not migrated. const ix = await indexHandle(); lastIndexHits.ok = ix.ok; lastIndexHits.total += 1; const cached = ix.get(p.id) as (ProjectSummary & { sig?: string }) | null; if (cached && cached.sig === sig && cached.kind === p.kind && cached.routing === p.routing) { lastIndexHits.fresh += 1; summaryCache.set(p.id, { sig, value: cached }); return cached; } const k = kindById(p.kind); // The boundary between a .mjs summariser and a typed summary. The modules // return more than the card needs (a song's verdict tally, a report's parsed // manifest) and their `state` infers as string, so it is narrowed once here // rather than asserted at every read. let body: KindSummary = {}; if (k?.summarise) { try { body = await k.summarise(ctxFor(p)); } catch { // A project that cannot be read is still a project. Saying so beats // dropping it from a list whose whole job is to be complete. body = { flags: ["could not be read"] }; } } const flags = [...(body.flags ?? [])]; // Routing failures are kind-independent, so they are added here rather than // in any kind's summariser. if (p.routing === "shadowed") flags.push(`/browse/${p.id.split("/")[0]} is a tool page`); if (p.routing === "unroutable") flags.push("name will not route"); if (p.ambiguousWith) flags.push(`two kinds match: ${p.ambiguousWith.join(", ")}`); const value: ProjectSummary = { ...p, badge: k?.badge ?? p.kind, title: body.title ?? p.name, subtitle: body.subtitle ?? null, state: (body.state as ProjectState) ?? "draft", newestMtimeMs: body.newestMtimeMs ?? 0, facts: body.facts ?? [], flags, posterRel: body.posterRel ?? null, haystack: `${body.haystack ?? ""} ${p.id} ${p.kind} ${p.template}`.toLowerCase(), attrs: body.attrs ?? {}, }; summaryCache.set(p.id, { sig, value }); // After the read, never before, and its failure is a cache miss next time. try { ix.put({ ...value, sig }); } catch { /* the filesystem is the model */ } return value; } /** Every project, newest first. */ export async function listProjects(): Promise { lastIndexHits = { fresh: 0, total: 0, ok: false }; const refs = await projectRefs(); const out = await Promise.all(refs.map(summariseProject)); return out.sort((a, b) => b.newestMtimeMs - a.newestMtimeMs || a.id.localeCompare(b.id)); } /** The folder tree, collapsed for display. URLs are never collapsed. */ export async function listFolders(): Promise> { return collapseFolders(foldersFor(await projectRefs())) as Map; } // --------------------------------------------------------------------------- // Decisions. // // The dispatch is here rather than in lib/decisions.ts because the song reducer // IS lib/decisions.ts -- it reads verdicts, plans, recipes, loudness and the // accepted cover set, all of which live in TypeScript beside readSong(). Naming // it from this side keeps the import graph acyclic and keeps the one kind-id // literal the app needs inside the registry's own module. // --------------------------------------------------------------------------- export async function decisionsForProject(p: ProjectRef): Promise { const sig = await signatureOf(p); const hit = decisionCache.get(p.id); if (hit && hit.sig === sig) return hit.value; let out: Decision[] = []; try { if (p.kind === SONG_KIND) { // Keyed by BASENAME while the song routes still are -- and the test for // "is this song reachable by basename" has to be the same one songIds() // uses, which is BROWSE_ROOT. Writing the production path out by hand here // meant the fixture (whose songs live elsewhere) silently produced no song // decisions at all, and the inbox looked clean because it was empty. out = p.dir === path.join(BROWSE_ROOT, p.name) ? await decisionsForSong(p.name) : []; } else { const k = kindById(p.kind); if (k?.decisions) { const summary = k.summarise ? await k.summarise(ctxFor(p)) : null; out = (await k.decisions(ctxFor(p), summary)) as Decision[]; } } } catch { out = []; } // Routing failures are kind-independent, so they are added here rather than // asked of every kind. Both of these used to be SILENT: a shadowed project was // listed with a link that rendered a tool page, and an unroutable one simply // did not appear. A thing that cannot be opened has to say so. if (p.routing === "shadowed") { out.push({ kind: "shadowed-name", project: p.id, target: p.id.split("/")[0], why: `/browse/${p.id.split("/")[0]} is a tool page and always wins the route — rename the directory, or open it from here`, href: `/browse/at?path=${encodeURIComponent(p.id)}`, severity: "blocking", at: Date.now(), }); } if (p.routing === "unroutable") { out.push({ kind: "unroutable-name", project: p.id, target: p.name, why: "this name cannot be a URL segment, so the project has no address of its own", href: `/browse/at?path=${encodeURIComponent(p.id)}`, severity: "info", at: Date.now(), }); } if (p.ambiguousWith) { out.push({ kind: "ambiguous-project", project: p.id, target: p.ambiguousWith.join(" + "), why: "two kinds match this directory — it is read as the first, which is a bug rather than a choice", href: `/browse/${p.id}`, severity: "blocking", at: Date.now(), }); } const title = (await summariseProject(p)).title; const value = out.map((d) => ({ ...d, project: p.id, projectTitle: title, projectKind: p.kind })); decisionCache.set(p.id, { sig, value }); return value; } const RANK: Record = { blocking: 0, open: 1, info: 2 }; /** Every open decision, every project, worst first. */ export async function openDecisions(): Promise { const refs = await projectRefs(); const per = await Promise.all(refs.map(decisionsForProject)); // Open notes on articles and report videos (lib/decisions.ts noteDecisions): // an article is not a project, so they are added here, not per project. const notes = await noteDecisions(); return [...per.flat(), ...notes] .sort((a, b) => RANK[a.severity] - RANK[b.severity] || b.at - a.at); } /** blocking + open counts per project id, for the index's chips. */ export async function decisionCounts(): Promise> { const refs = await projectRefs(); const out = new Map(); await Promise.all( refs.map(async (p) => { const ds = await decisionsForProject(p); out.set(p.id, { blocking: ds.filter((d) => d.severity === "blocking").length, open: ds.filter((d) => d.severity === "open").length, }); }), ); return out; } // --------------------------------------------------------------------------- // Routing. // --------------------------------------------------------------------------- export type Resolved = | { project: ProjectRef; rest: string[]; folder: null; via?: "alias" } | { project: null; rest: []; folder: FolderNode } | { project: null; rest: []; folder: null; ambiguous: ProjectRef[] } | null; /** * Resolve a `/browse/...` path to the LONGEST prefix that is a project, and * hand the remaining segments to that kind's view. * * Longest-prefix rather than "first project found" because a project id is a * path, and `a/b` being a project must not stop `a/b/c` from being one too. */ export async function resolveProjectPath(segments: string[]): Promise { const refs = await projectRefs(); const id = segments.join("/"); let best: ProjectRef | null = null; for (const p of refs) { if (p.routing !== "ok") continue; if (id === p.id || id.startsWith(`${p.id}/`)) { if (!best || p.id.length > best.id.length) best = p; } } if (best) { const rest = id === best.id ? [] : id.slice(best.id.length + 1).split("/"); return { project: best, rest, folder: null }; } const folders = await listFolders(); const folder = folders.get(id); if (folder) return { project: null, rest: [], folder }; // ---- the basename alias ------------------------------------------------- // // A project id is a PATH, so a song's canonical URL is now // /browse/quartering-uh-song/videos/yoshi. But /browse/yoshi and // /browse/yoshi/wide are the URLs that exist -- in every decision href, in // the e2e suite, and in whatever anybody has open in a tab. A URL has to mean // the same thing in six weeks, so the bare name keeps working as an alias. // // ONLY when it is unique. Two projects sharing a basename is precisely why an // id is a path in the first place, and picking one of them would be the guess // this whole design refuses to make -- so it reports the collision instead. const [first, ...restSegs] = segments; if (first) { const named = refs.filter((p) => p.name === first && p.routing === "ok"); if (named.length === 1) { return { project: named[0], rest: restSegs, folder: null, via: "alias" }; } if (named.length > 1) { return { project: null, rest: [], folder: null, ambiguous: named }; } } return null; } /** Kind metadata by id, for a page that has a summary and wants its label. */ export const kindMetaOf = (id: string): ProjectKindMeta | null => KINDS.find((k) => k.id === id) ?? null; export { PROJECT_KINDS }; // --------------------------------------------------------------------------- // Media, grouped by the project that owns it. // // The mix picker was a single newest-first list with a cap, and the cap was // saturated: measured against the real tree, four of the six report deliverables // fell off the end of a 600-entry list, and per-song cuts never appeared at all. // Raising the cap only moves the cliff. // // So coverage becomes a property of the enumeration. Each project is asked for // its own files -- with its own small cap, so one busy project cannot push every // other project off the end -- and whatever is left over (the corpus, the render // scratch) becomes one final group. `` renders it with zero client JS // and the control stays a native select. // --------------------------------------------------------------------------- export type MediaGroup = { project: string; label: string; kind: string; files: MediaRow[]; }; const PER_PROJECT = 40; export async function mediaGroups(): Promise<{ groups: MediaGroup[]; other: MediaRow[] }> { const refs = await projectRefs(); const groups: MediaGroup[] = []; for (const p of refs) { const files = await listMediaUnder(p.dir, 2, PER_PROJECT); if (!files.length) continue; const summary = await summariseProject(p); groups.push({ project: p.id, label: `${p.id} (${summary.badge})`, kind: p.kind, files }); } groups.sort((a, b) => a.project.localeCompare(b.project)); // Longest-prefix ownership, so a file inside a project never also appears in // `other` -- and so nothing needs a second walk to work out who owns what. const dirs = refs.map((p) => p.dir + path.sep); const owned = (abs: string) => dirs.some((d) => abs.startsWith(d)); const other = (await listMedia(600)).filter((f) => !owned(f.path)); return { groups, other }; }