commit a18beb03ae1c204a9624219e4d3e33d764f250f4
parent 77f8e6e1a9bb05ab6aeddfe8bdababdc63a02901
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 9 Oct 2026 10:19:32 -0400
Merge umtool/articles (every site's articles + media in umtool, operator notes an agent reads back, a better video editor)
/sites, the article reader with anchored notes, cited evidence, workspace
files; notes.json beside report.json / the project manifest, `umtool notes`
for agents; timed notes on cuts, takes and article videos; row and take
notes; the generated-manifest guard with edit notes; structural timeline
edits with auto snapshots and undo.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
101 files changed, 9264 insertions(+), 74 deletions(-)
diff --git a/AGENTS.md b/AGENTS.md
@@ -140,6 +140,21 @@ better), whose clock is not the same.
This tooling is on `main` as of 2026-08-20.
+# Operator notes on articles and videos
+
+```sh
+umtool notes --all --open # from the checkout: node umtool/bin/umtool.mjs notes --all --open
+```
+
+The operator leaves notes in umtool on articles (`/sites/<site>/<report>`) and on
+report-video projects (rows, takes, moments in a cut). They live in a `notes.json`
+beside the report's `report.json` (never published) or beside the project's
+`video.manifest.json`. `umtool notes <site>/<report>` prints each open note with its
+anchor resolved and the **source** file to edit — a draft, not the generated
+`report.json` or manifest. Act, regenerate, then `umtool notes reply <id> "<what
+changed>" --resolve`. Never hand-edit `notes.json`. See
+[umtool/docs/notes.md](umtool/docs/notes.md).
+
# The runtime container
`docker compose up -d` stands up a working archive: the editor plus Caddy, with
diff --git a/REPORT.md b/REPORT.md
@@ -2,7 +2,7 @@
<!-- GENERATED by common/bin/file-schemas-docs.ts from the schemas and their *_FIELD_DOCS records — do not edit by hand. -->
-One cited report, format `"archilyzer-report"`, version 1, persisted to `transcripts/sites/<siteId>/reports/<reportId>/report.json` beside its `stills/` and `sources/<sourceId>/`; a relative path in it is relative to that directory. A site's `reports` list in `site.json` is the published, ordered list — see [SITE.md](SITE.md); a report directory it does not name is a draft. The schema is `common/lib/report/schema.ts`; its citations and sources are the citation model's — see [CITATIONS.md](CITATIONS.md).
+One cited report, format `"archilyzer-report"`, version 1, persisted to `transcripts/sites/<siteId>/reports/<reportId>/report.json` beside its `stills/` and `sources/<sourceId>/`; a relative path in it is relative to that directory. A site's `reports` list in `site.json` is the published, ordered list — see [SITE.md](SITE.md); a report directory it does not name is a draft. The schema is `common/lib/report/schema.ts`; its citations and sources are the citation model's — see [CITATIONS.md](CITATIONS.md). A `notes.json` beside it holds the operator's notes on the article (umtool, `umtool notes`; [umtool/docs/notes.md](umtool/docs/notes.md)) and is never published.
A **fact-check** (`"kind": "factcheck"`) is sections (chapters) of claims, each with a verdict and its findings. A **sweep** (`"kind": "sweep"`) is sections with no verdicts, or bodies that cite inline. Markdown fields (`summary`, a section's `body`, a claim's `findings`) cite with `[label](cite:<id>)`.
diff --git a/common/lib/report/docs.ts b/common/lib/report/docs.ts
@@ -32,7 +32,9 @@ export function renderReportMarkdown(): string {
"that directory. A site's `reports` list in `site.json` is the published, " +
"ordered list — see [SITE.md](SITE.md); a report directory it does not name is a " +
"draft. The schema is `common/lib/report/schema.ts`; its citations and sources " +
- "are the citation model's — see [CITATIONS.md](CITATIONS.md).",
+ "are the citation model's — see [CITATIONS.md](CITATIONS.md). A `notes.json` " +
+ "beside it holds the operator's notes on the article (umtool, `umtool notes`; " +
+ "[umtool/docs/notes.md](umtool/docs/notes.md)) and is never published.",
);
out.push("");
out.push(
diff --git a/common/publish/composeReports.test.ts b/common/publish/composeReports.test.ts
@@ -54,6 +54,8 @@ const VIDEOS = "demo-channel";
const SOCIAL = "demo-social";
const REPORT = "demo-report";
const NOW_FLOOR = new Date().toISOString();
+// What a report's notes.json says: if it shows up in anything built, a note leaked.
+const NOTES_SENTINEL = "operator-note-never-published-7f3a";
const writeJson = (file: string, value: unknown) => {
mkdirSync(path.dirname(file), { recursive: true });
@@ -172,6 +174,8 @@ function seedSite(siteId: string, extra: Record<string, unknown> = {}, withRepor
writeJson(path.join(dir, "report.json"), report());
writeText(path.join(dir, "stills", "a01.png"), "png-bytes");
writeText(path.join(dir, "sources", "s0", "page.html"), "<p>saved copy, never published</p>");
+ // umtool's operator notes live beside report.json and are never published.
+ writeJson(path.join(dir, "notes.json"), { format: "umtool-notes", version: 1, notes: [{ text: NOTES_SENTINEL }] });
seedMedia(siteId);
}
@@ -786,3 +790,27 @@ test("reports export: no browser skips the PDF with a note; no zip fails the pac
assert.equal(await exportMain({ siteId: "cited", formats: ["zip"], zipBin: path.join(ROOT, "no-such-zip") }, quiet), 1);
assert.equal(await exportMain({ siteId: "cited", formats: ["html", "md"] }, quiet), 0);
});
+
+test("a report's notes.json (umtool's operator notes) is never published, exported or committed to its history", async () => {
+ const { reportHistoryGitDir } = await import("./reportHistory");
+ const { execFileSync } = await import("node:child_process");
+ const notes = path.join(paths.sitesDir, "cited", "reports", REPORT, "notes.json");
+ assert.ok(existsSync(notes));
+ await exportSiteReports({ siteId: "cited", paths, now: () => new Date("2026-10-08T08:00:00Z"), openPdfPrinter: fakePrinter([]), onLog: () => {} });
+ await compose("cited");
+ const leaks = (root: string) =>
+ filesUnder(root).filter((f) => f.endsWith("notes.json") || readFileSync(path.join(root, f)).includes(NOTES_SENTINEL));
+ assert.deepEqual(leaks(paths.exportPublicDir), []);
+ assert.deepEqual(leaks(reportExportDir(paths, "cited", REPORT)), []);
+ const gitDir = reportHistoryGitDir(paths, "cited", REPORT);
+ const revs = execFileSync("git", ["--git-dir", gitDir, "rev-list", "--all"], { encoding: "utf8" }).trim().split("\n").filter(Boolean);
+ assert.ok(revs.length > 0);
+ for (const rev of revs) {
+ const names = execFileSync("git", ["--git-dir", gitDir, "ls-tree", "-r", "--name-only", rev], { encoding: "utf8" }).trim().split("\n");
+ assert.deepEqual(names.filter((n) => n.includes("notes")), [], rev);
+ for (const n of names) {
+ const blob = execFileSync("git", ["--git-dir", gitDir, "cat-file", "blob", `${rev}:${n}`]);
+ assert.ok(!blob.includes(NOTES_SENTINEL), `${rev}:${n}`);
+ }
+ }
+});
diff --git a/common/publish/reportMedia.ts b/common/publish/reportMedia.ts
@@ -42,6 +42,7 @@
// a channel whose media tier is not reachable (lib/channelMedia.ts) is
// reported as unreachable, with the reason, rather than as missing.
+import type { Dirent } from "node:fs";
import { readdir, rm } from "node:fs/promises";
import path from "node:path";
import { readJsonFile, writeJsonAtomic } from "../lib/jsonFile-server";
@@ -55,7 +56,7 @@ import { readChannelConfig } from "../controller/channels";
import { momentKey, momentOf, momentProblem, type Moment } from "../lib/citations/moments";
import type { CitationPad } from "../lib/citations/schema";
import { parseReport } from "../lib/report/validate";
-import type { Report } from "../lib/report/schema";
+import { isReportId, type Report } from "../lib/report/schema";
import { isPermanentlyGone } from "../lib/availability";
import { loadAvailability } from "../lib/availability-server";
import { MAX_CLIP_WINDOW_SECONDS } from "../lib/clipWindow";
@@ -93,6 +94,23 @@ export function siteReportFile(paths: Paths, siteId: string, reportId: string):
return path.join(siteReportDir(paths, siteId, reportId), "report.json");
}
+// Every report directory of a site, published or draft: a directory under
+// `sites/<siteId>/reports/` whose name is a report id. Anything else there (a
+// stray file, a bad name) is not a report. Sorted. Whether one is PUBLISHED is
+// the site's `reports` list (site.json), not anything on disk.
+export async function listReportDirs(paths: Paths, siteId: string): Promise<string[]> {
+ let entries: Dirent[];
+ try {
+ entries = await readdir(path.join(siteDir(paths, siteId), "reports"), { withFileTypes: true });
+ } catch {
+ return [];
+ }
+ return entries
+ .filter((e) => e.isDirectory() && isReportId(e.name))
+ .map((e) => e.name)
+ .sort((a, b) => a.localeCompare(b));
+}
+
export type ReportMediaEntry =
| EvidenceMedia
| {
diff --git a/editor/app/sites/lib/reportListServer.ts b/editor/app/sites/lib/reportListServer.ts
@@ -1,12 +1,8 @@
import "server-only";
-import type { Dirent } from "node:fs";
-import { readdir } from "node:fs/promises";
-import path from "node:path";
import type { Paths } from "yt-dlp-transcript-common/lib/paths";
import { readJsonFile } from "yt-dlp-transcript-common/lib/jsonFile-server";
-import { isReportId } from "yt-dlp-transcript-common/lib/report/schema";
-import { siteDir, type Site } from "yt-dlp-transcript-common/lib/site";
-import { siteReportFile } from "yt-dlp-transcript-common/publish/reportMedia";
+import type { Site } from "yt-dlp-transcript-common/lib/site";
+import { listReportDirs, siteReportFile } from "yt-dlp-transcript-common/publish/reportMedia";
import {
publishableReportExports,
readReportExportManifest,
@@ -18,24 +14,11 @@ import { listAllJobs, type JobListEntry } from "yt-dlp-transcript-common/jobs/li
import { readJobMeta } from "yt-dlp-transcript-common/jobs/jobMeta";
import { reportRows, type ReportFileRead, type ReportRow } from "./reportList";
-// The report directories under `sites/<siteId>/reports/`: a directory whose
-// name is a report id. Anything else there (a stray file, a bad name) is not a
-// report and is not listed.
-async function reportDirIds(paths: Paths, siteId: string): Promise<string[]> {
- let entries: Dirent[];
- try {
- entries = await readdir(path.join(siteDir(paths, siteId), "reports"), { withFileTypes: true });
- } catch {
- return [];
- }
- return entries.filter((e) => e.isDirectory() && isReportId(e.name)).map((e) => e.name);
-}
-
// Every report of the site, published (in order) then drafts, each read and
// validated.
export async function readSiteReportRows(paths: Paths, site: Site): Promise<ReportRow[]> {
const published = site.reports ?? [];
- const dirIds = await reportDirIds(paths, site.siteId);
+ const dirIds = await listReportDirs(paths, site.siteId);
const ids = [...new Set([...published, ...dirIds])];
const reads = new Map<string, ReportFileRead>(
await Promise.all(
diff --git a/package.json b/package.json
@@ -22,7 +22,7 @@
"e2e": "node scripts/worktree.mjs run -- pnpm --filter editor run e2e",
"wt": "node scripts/worktree.mjs",
"e2e:sharded": "node scripts/run-sharded-e2e.mjs",
- "test:scripts": "node --test scripts/*.test.mjs umtool/report-to-video/*.test.mjs umtool/lib/report/*.test.mjs",
+ "test:scripts": "node --test scripts/*.test.mjs umtool/report-to-video/*.test.mjs umtool/lib/report/*.test.mjs umtool/lib/annotations/*.test.mjs umtool/lib/articles/*.test.mjs",
"lint": "pnpm --filter export run lint",
"ops": "node scripts/archilyzer-ops.mjs"
},
diff --git a/umtool/app/api/notes/context/route.ts b/umtool/app/api/notes/context/route.ts
@@ -0,0 +1,32 @@
+import { errorResponse, targetFrom } from "@/lib/annotations/server";
+import { digest } from "@/lib/annotations/digest.mjs";
+import { readNotes } from "@/lib/annotations/store.mjs";
+
+export const dynamic = "force-dynamic";
+
+// The agent brief: the same markdown `umtool notes <target>` prints, as
+// text/plain, so "Copy agent brief" and `curl` hand over the words an agent
+// would read on the command line.
+//
+// /api/notes/context?article=<site>/<report>[&status=open|resolved|all]
+// /api/notes/context?project=<id>[&status=…]
+export async function GET(request: Request) {
+ try {
+ const url = new URL(request.url);
+ const target = await targetFrom({ article: url.searchParams.get("article"), project: url.searchParams.get("project") });
+ const s = url.searchParams.get("status");
+ const status = s === "resolved" || s === "all" ? s : "open";
+ const read = await readNotes(target.file);
+ const cli = `umtool notes ${target.id}`;
+ const body =
+ !read.doc && !read.error
+ ? `No notes on ${target.id}.\n`
+ : `${await digest(
+ { kind: target.kind as "article" | "video-project", id: target.id, file: target.file, doc: read.doc, ...(read.error ? { error: read.error } : {}) },
+ { status, projectDir: "dir" in target ? target.dir : undefined },
+ )}\n\nRead these again with \`${cli}\` (from the repo checkout).\n`;
+ return new Response(body, { headers: { "content-type": "text/plain; charset=utf-8", "cache-control": "no-store" } });
+ } catch (err) {
+ return errorResponse(err);
+ }
+}
diff --git a/umtool/app/api/notes/route.ts b/umtool/app/api/notes/route.ts
@@ -0,0 +1,51 @@
+import { errorResponse, targetFrom } from "@/lib/annotations/server";
+import { readNotes } from "@/lib/annotations/store.mjs";
+import { writeNote } from "@/lib/annotations/targets.mjs";
+
+export const dynamic = "force-dynamic";
+
+// Notes on an article or a report-video project (lib/annotations/, docs/notes.md).
+//
+// GET ?article=<site>/<report> | ?project=<id>
+// { subject, file, token, doc, source, error? } -- doc null when
+// there are no notes; `source` is what a first write would record.
+// POST same params, body { token, op: { op: "add" | "edit" | "status" |
+// "reply" | "delete" | "delete-reply" | "source", ... } }
+// { doc, token, note }. 409 when `token` is stale or the file on
+// disk is not a notes doc; 400 for a refused op or target.
+//
+// Every write from here is the OPERATOR's. An agent writes through
+// `umtool notes`, which stamps "agent"; there is no way to claim to be one here.
+const NO_STORE = { "cache-control": "no-store" };
+
+function params(request: Request) {
+ const url = new URL(request.url);
+ return { article: url.searchParams.get("article"), project: url.searchParams.get("project") };
+}
+
+export async function GET(request: Request) {
+ try {
+ const target = await targetFrom(params(request));
+ const read = await readNotes(target.file);
+ const source = read.doc?.source ?? (await target.source());
+ return Response.json(
+ { subject: target.subject, file: target.file, token: read.token, doc: read.doc, source: source ?? null, ...(read.error ? { error: read.error } : {}) },
+ { headers: NO_STORE },
+ );
+ } catch (err) {
+ return errorResponse(err);
+ }
+}
+
+export async function POST(request: Request) {
+ try {
+ const target = await targetFrom(params(request));
+ const body = (await request.json().catch(() => null)) as { token?: unknown; op?: unknown } | null;
+ if (!body || typeof body.op !== "object" || body.op === null) return Response.json({ error: "body needs an op" }, { status: 400 });
+ const token = typeof body.token === "string" ? body.token : null;
+ const out = await writeNote(target, body.op as Record<string, unknown>, { by: "operator", token });
+ return Response.json(out, { headers: NO_STORE });
+ } catch (err) {
+ return errorResponse(err);
+ }
+}
diff --git a/umtool/app/api/report/chrome/route.ts b/umtool/app/api/report/chrome/route.ts
@@ -1,3 +1,4 @@
+import { withEditNotes } from "@/lib/report/guard";
import { ChromeRefused, StaleToken, manifestToken, updateChrome } from "@/lib/report/manifest.mjs";
import { resolveReport } from "@/lib/report/serve.mjs";
import { DECK_DEFAULTS, deckOn, resolveDeck, validateChrome } from "umtool-report-to-video/deck";
@@ -53,15 +54,18 @@ export async function PUT(request: Request) {
if ("error" in r) return Response.json({ error: r.error }, { status: r.status });
try {
- const res = await updateChrome(r.project.dir, (body.chrome ?? null) as Record<string, unknown> | null, {
- token: body.token === undefined ? null : String(body.token),
- });
+ const { result: res, editNotes } = await withEditNotes(r.project, () =>
+ updateChrome(r.project.dir, (body.chrome ?? null) as Record<string, unknown> | null, {
+ token: body.token === undefined ? null : String(body.token),
+ }),
+ );
return Response.json(
{
ok: true,
chrome: res.chrome,
deck: res.chrome ? resolveDeck({ ...(r.manifest.render ?? {}), chrome: res.chrome }) : null,
token: res.token,
+ editNotes,
},
{ headers: noStore },
);
diff --git a/umtool/app/api/report/claim/route.ts b/umtool/app/api/report/claim/route.ts
@@ -1,3 +1,4 @@
+import { withEditNotes } from "@/lib/report/guard";
import { StaleToken, manifestToken, updateClaim } from "@/lib/report/manifest.mjs";
import { resolveClaim } from "@/lib/report/serve.mjs";
import { readClaimDetail } from "@/lib/projects/report.mjs";
@@ -50,11 +51,16 @@ export async function PUT(request: Request) {
}
try {
- const { entry, token } = await updateClaim(r.project.dir, claimId, patch, {
- token: body.token === undefined ? null : String(body.token),
- });
+ const {
+ result: { entry, token },
+ editNotes,
+ } = await withEditNotes(r.project, () =>
+ updateClaim(r.project.dir, claimId, patch, {
+ token: body.token === undefined ? null : String(body.token),
+ }),
+ );
const detail = await readClaimDetail(r.project.dir, claimId);
- return Response.json({ claim: entry, gaps: detail?.gaps ?? [], token });
+ return Response.json({ claim: entry, gaps: detail?.gaps ?? [], token, editNotes });
} catch (err) {
// A stale token is a 409 and never a silent overwrite.
if (err instanceof StaleToken) {
diff --git a/umtool/app/api/report/cut/route.ts b/umtool/app/api/report/cut/route.ts
@@ -1,3 +1,4 @@
+import { withEditNotes } from "@/lib/report/guard";
import { StaleToken, updateClip } from "@/lib/report/manifest.mjs";
import { cuesInWindow } from "@/lib/projects/report.mjs";
import { resolveClip } from "@/lib/report/serve.mjs";
@@ -63,14 +64,16 @@ export async function POST(request: Request) {
}
try {
- const res = await updateClip(
- project.dir,
- clipId,
- { cutStart: hit.cutStart, cutEnd: hit.cutEnd },
- { token: body.token === undefined ? null : String(body.token) },
+ const { result: res, editNotes } = await withEditNotes(project, () =>
+ updateClip(
+ project.dir,
+ clipId,
+ { cutStart: hit.cutStart, cutEnd: hit.cutEnd },
+ { token: body.token === undefined ? null : String(body.token) },
+ ),
);
return Response.json(
- { ok: true, entry: res.entry, token: res.token, score: hit.score, matched: hit.matched },
+ { ok: true, entry: res.entry, token: res.token, score: hit.score, matched: hit.matched, editNotes },
{ headers: { "cache-control": "no-store" } },
);
} catch (e) {
diff --git a/umtool/app/api/report/moment/route.ts b/umtool/app/api/report/moment/route.ts
@@ -0,0 +1,26 @@
+import { projectRef } from "@/lib/projects";
+import { resolveMomentOnFile } from "@/lib/report/moments.mjs";
+
+export const dynamic = "force-dynamic";
+
+// GET ?project=<id>&file=<project-relative mp4>&t=<seconds>[&duration=<seconds>]
+//
+// What is on screen at `t` of a rendered cut (lib/report/moments.mjs): the
+// entry, its onscreen title and quote, the source second and its archive link,
+// and `approx` when the file and the build's schedule disagree. A timed note
+// stores this at write time. `{ entry: null }` when the file has no schedule
+// (a preview made without a build) -- the note then keeps `t` alone.
+export async function GET(request: Request) {
+ const url = new URL(request.url);
+ const project = await projectRef(url.searchParams.get("project") ?? "");
+ if (!project) return Response.json({ error: "no such project" }, { status: 404 });
+ const file = url.searchParams.get("file") ?? "";
+ if (!file || file.startsWith("/") || file.split("/").some((s) => s === ".." || s === "")) {
+ return Response.json({ error: "file must be relative to the project" }, { status: 400 });
+ }
+ const t = Number(url.searchParams.get("t"));
+ if (!Number.isFinite(t) || t < 0) return Response.json({ error: "t must be seconds ≥ 0" }, { status: 400 });
+ const d = Number(url.searchParams.get("duration"));
+ const r = await resolveMomentOnFile(project.dir, file, t, Number.isFinite(d) && d > 0 ? d : null);
+ return Response.json(r, { headers: { "cache-control": "no-store" } });
+}
diff --git a/umtool/app/api/report/onscreen/route.ts b/umtool/app/api/report/onscreen/route.ts
@@ -1,3 +1,4 @@
+import { withEditNotes } from "@/lib/report/guard";
import { StaleToken, manifestToken, updateOnscreen } from "@/lib/report/manifest.mjs";
import { postRows, scheduleForPreview } from "@/lib/report/onscreen.mjs";
import { resolveReport } from "@/lib/report/serve.mjs";
@@ -86,12 +87,14 @@ export async function PUT(request: Request) {
if ("error" in r) return Response.json({ error: r.error }, { status: r.status });
try {
- const res = await updateOnscreen(
- r.project.dir,
- body.onscreen as Record<string, { title?: string; subtitle?: string; claim?: Claim | null } | null>,
- { token: body.token === undefined ? null : String(body.token) },
+ const { result: res, editNotes } = await withEditNotes(r.project, () =>
+ updateOnscreen(
+ r.project.dir,
+ body.onscreen as Record<string, { title?: string; subtitle?: string; claim?: Claim | null } | null>,
+ { token: body.token === undefined ? null : String(body.token) },
+ ),
);
- return Response.json({ ok: true, onscreen: res.onscreen, claims: res.claims, token: res.token }, { headers: noStore });
+ return Response.json({ ok: true, onscreen: res.onscreen, claims: res.claims, token: res.token, editNotes }, { headers: noStore });
} catch (e) {
if (e instanceof StaleToken) {
return Response.json(
diff --git a/umtool/app/api/report/posts/route.ts b/umtool/app/api/report/posts/route.ts
@@ -1,3 +1,4 @@
+import { withEditNotes } from "@/lib/report/guard";
import { PostsRefused, StaleToken, updatePosts } from "@/lib/report/manifest.mjs";
import { resolveReport } from "@/lib/report/serve.mjs";
@@ -28,12 +29,14 @@ export async function PUT(request: Request) {
if ("error" in r) return Response.json({ error: r.error }, { status: r.status });
try {
- const res = await updatePosts(
- r.project.dir,
- body.posts as Record<string, { attachTo?: string | null; hide?: boolean }>,
- { token: body.token === undefined ? null : String(body.token) },
+ const { result: res, editNotes } = await withEditNotes(r.project, () =>
+ updatePosts(
+ r.project.dir,
+ body.posts as Record<string, { attachTo?: string | null; hide?: boolean }>,
+ { token: body.token === undefined ? null : String(body.token) },
+ ),
);
- return Response.json({ ok: true, posts: res.posts, token: res.token }, { headers: noStore });
+ return Response.json({ ok: true, posts: res.posts, token: res.token, editNotes }, { headers: noStore });
} catch (e) {
if (e instanceof StaleToken) {
return Response.json(
diff --git a/umtool/app/api/report/timeline/route.ts b/umtool/app/api/report/timeline/route.ts
@@ -0,0 +1,134 @@
+import { invalidateProjects } from "@/lib/projects";
+import { withEditNotes } from "@/lib/report/guard";
+import {
+ StaleToken,
+ StructureRefused,
+ duplicateEntry,
+ insertEntry,
+ manifestToken,
+ moveEntry,
+ removeEntry,
+ removePost,
+ undoStructural,
+ updateFactcheck,
+ updateTeaser,
+ upsertPost,
+} from "@/lib/report/manifest.mjs";
+import { resolveReport } from "@/lib/report/serve.mjs";
+import { deckOn } from "umtool-report-to-video/deck";
+import { VERDICTS, resolveFactcheck } from "umtool-report-to-video/factcheck";
+
+export const dynamic = "force-dynamic";
+
+// The STRUCTURE of the cut: what is in the timeline and in what order, a
+// teaser's lines, the posts, the fact-check's labels (lib/report/manifest.mjs,
+// "STRUCTURE"). The window route deliberately cannot move an entry; this is
+// the different button it points at.
+//
+// GET ?project=<id> the token, the timeline's ids in order, the
+// teasers, the posts and the fact-check block
+// POST { project, token, op, ... } one op:
+// move { id, at?, toIndex } toIndex is the index AFTER the move
+// remove { id, at? }
+// duplicate { id, at? }
+// insert { afterId: id | null, at?, entry: "<channel>/<video>@<start>-<end>" | { type, … } }
+// teaser { id, at?, patch: { lines?, beat?, dip?, tail?, tailWait? } }
+// post { post: { id, platform, date, text, url, … } } add or replace
+// post-remove { id }
+// factcheck { factcheck: { verdicts?, stamp?, tally? } | null }
+// undo {} restore the newest auto snapshot
+//
+// Every op snapshots the manifest first (`revisions/<stamp>-auto-before-<op>`),
+// is checked by the build's own validators, and on a GENERATED manifest leaves
+// an `edit` note for the agent (lib/report/guard.ts). A stale token is a 409.
+
+type Body = Record<string, unknown>;
+const noStore = { "cache-control": "no-store" };
+
+export async function GET(request: Request) {
+ const url = new URL(request.url);
+ const r = await resolveReport(url.searchParams.get("project") ?? "");
+ if ("error" in r) return Response.json({ error: r.error }, { status: r.status });
+ const entries = (r.manifest.timeline ?? []) as Record<string, unknown>[];
+ const timeline = entries.map((e) => ({ id: String(e.id), type: String(e.type ?? "entry") }));
+ // What the structure editors edit: each teaser as written, the posts, and
+ // the fact-check block with every default filled (the form's placeholders).
+ const teasers = entries
+ .map((e, at) => ({ e, at }))
+ .filter(({ e }) => e.type === "teaser")
+ .map(({ e, at }) => ({ at, id: String(e.id), lines: e.lines ?? [], beat: e.beat ?? null, dip: e.dip ?? null, tail: e.tail ?? null, tailWait: e.tailWait ?? null }));
+ const render = (r.manifest.render ?? {}) as Record<string, unknown>;
+ return Response.json(
+ {
+ timeline,
+ teasers,
+ posts: Array.isArray(r.manifest.posts) ? r.manifest.posts : [],
+ deckOn: deckOn(render),
+ factcheck: (render.chrome as { factcheck?: unknown } | undefined)?.factcheck ?? null,
+ factcheckResolved: resolveFactcheck(render),
+ verdicts: VERDICTS,
+ generatedBy: r.manifest.generatedBy ?? null,
+ token: await manifestToken(r.project.dir),
+ },
+ { headers: noStore },
+ );
+}
+
+const atOf = (v: unknown) => (v === undefined || v === null || v === "" ? null : Number(v));
+
+export async function POST(request: Request) {
+ let body: Body;
+ try {
+ body = await request.json();
+ } catch {
+ return Response.json({ error: "expected JSON" }, { status: 400 });
+ }
+ const r = await resolveReport(String(body.project ?? ""));
+ if ("error" in r) return Response.json({ error: r.error }, { status: r.status });
+ const dir = r.project.dir;
+ // A string, or no guard at all: `String(null)` would be the token "null",
+ // which matches no file and refuses every write.
+ const token = typeof body.token === "string" ? body.token : null;
+ const id = String(body.id ?? "");
+ const at = atOf(body.at);
+
+ const run = (): Promise<Record<string, unknown>> => {
+ switch (body.op) {
+ case "move":
+ return moveEntry(dir, id, Number(body.toIndex), { token, at });
+ case "remove":
+ return removeEntry(dir, id, { token, at });
+ case "duplicate":
+ return duplicateEntry(dir, id, { token, at });
+ case "insert":
+ return insertEntry(dir, body.afterId ? String(body.afterId) : null, body.entry, { token, at });
+ case "teaser":
+ return updateTeaser(dir, id, body.patch as Record<string, unknown>, { token, at });
+ case "post":
+ return upsertPost(dir, body.post as Record<string, unknown>, { token });
+ case "post-remove":
+ return removePost(dir, id, { token });
+ case "factcheck":
+ return updateFactcheck(dir, (body.factcheck ?? null) as Record<string, unknown> | null, { token });
+ case "undo":
+ return undoStructural(dir, { token });
+ default:
+ throw new Error("op must be move, remove, duplicate, insert, teaser, post, post-remove, factcheck or undo");
+ }
+ };
+
+ try {
+ const { result, editNotes } = await withEditNotes(r.project, run);
+ // revisions/ changed: the project page lists them.
+ invalidateProjects();
+ return Response.json({ ok: true, ...result, editNotes }, { headers: noStore });
+ } catch (e) {
+ if (e instanceof StaleToken) {
+ return Response.json({ error: e.message, expected: e.expected, got: e.got, stale: true }, { status: 409 });
+ }
+ if (e instanceof StructureRefused) {
+ return Response.json({ error: e.message, errors: e.errors }, { status: 400 });
+ }
+ return Response.json({ error: e instanceof Error ? e.message : String(e) }, { status: 400 });
+ }
+}
diff --git a/umtool/app/api/report/window/route.ts b/umtool/app/api/report/window/route.ts
@@ -1,3 +1,4 @@
+import { withEditNotes } from "@/lib/report/guard";
import { StaleToken, updateClip } from "@/lib/report/manifest.mjs";
import { resolveClip } from "@/lib/report/serve.mjs";
@@ -63,11 +64,13 @@ export async function PUT(request: Request) {
if (!Object.keys(patch).length) return Response.json({ error: "nothing to change" }, { status: 400 });
try {
- const res = await updateClip(r.project.dir, clipId, patch, {
- token: body.token === undefined ? null : String(body.token),
- });
+ const { result: res, editNotes } = await withEditNotes(r.project, () =>
+ updateClip(r.project.dir, clipId, patch, {
+ token: body.token === undefined ? null : String(body.token),
+ }),
+ );
return Response.json(
- { ok: true, entry: res.entry, before: res.before, token: res.token },
+ { ok: true, entry: res.entry, before: res.before, token: res.token, editNotes },
{ headers: { "cache-control": "no-store" } },
);
} catch (e) {
diff --git a/umtool/app/api/sites/evidence/route.ts b/umtool/app/api/sites/evidence/route.ts
@@ -0,0 +1,19 @@
+import { citationEvidence } from "@/lib/articles/evidence";
+import { readReportFile, siteById } from "@/lib/articles/sites";
+
+export const dynamic = "force-dynamic";
+
+// GET ?site&report&cite -- one citation's evidence (lib/articles/evidence.ts):
+// its quote and record, the transcript around it, and what can play it.
+export async function GET(request: Request) {
+ const url = new URL(request.url);
+ const site = url.searchParams.get("site") ?? "";
+ const reportId = url.searchParams.get("report") ?? "";
+ const cite = url.searchParams.get("cite") ?? "";
+ if (!siteById(site)) return Response.json({ error: `no site ${site}` }, { status: 404 });
+ const read = await readReportFile(site, reportId);
+ if (!read.report) return Response.json({ error: read.problems[0]?.message ?? "no report" }, { status: 404 });
+ const ev = await citationEvidence(site, read.report, cite);
+ if (!ev) return Response.json({ error: `no citation ${cite}` }, { status: 404 });
+ return Response.json(ev, { headers: { "cache-control": "no-store" } });
+}
diff --git a/umtool/app/api/sites/media/route.ts b/umtool/app/api/sites/media/route.ts
@@ -0,0 +1,94 @@
+import { readFile, realpath, stat } from "node:fs/promises";
+import path from "node:path";
+import { reportMediaDir, reportMediaIndexFile, siteReportDir } from "yt-dlp-transcript-common/publish/reportMedia";
+import { rangeResponse } from "@/lib/report/serve.mjs";
+import { CHANNELS_DIR, REPORTS_ROOT, SITES_DIR, inside } from "@/lib/paths";
+import { sitesPaths } from "@/lib/articles/sites";
+
+export const dynamic = "force-dynamic";
+
+// Article media, READ-ONLY, with byte ranges (lib/report/serve.mjs
+// rangeResponse, the one the cut player uses -- a <video> will not seek a
+// stream it was not given a 206 for).
+//
+// ?site&report&file a file in the report's directory: video.mp4,
+// poster.jpg, stills/… -- relative, no `..`, a media type,
+// and its real path under SITES_DIR (or REPORTS_ROOT: a
+// generator may link a take's preview in)
+// ?site&moment the site's PREPARED evidence clip for a moment, found
+// through report-media/index.json -- never a client path
+// ?corpus=<abs> a clip window, saved video, audio or post capture in
+// the corpus: lexically under CHANNELS_DIR (its real
+// path is on whatever drive the media tier links to)
+const TYPES: Record<string, string> = {
+ ".mp4": "video/mp4",
+ ".m4v": "video/mp4",
+ ".webm": "video/webm",
+ ".mkv": "video/x-matroska",
+ ".mov": "video/quicktime",
+ ".m4a": "audio/mp4",
+ ".mp3": "audio/mpeg",
+ ".opus": "audio/ogg",
+ ".ogg": "audio/ogg",
+ ".wav": "audio/wav",
+ ".jpg": "image/jpeg",
+ ".jpeg": "image/jpeg",
+ ".png": "image/png",
+ ".webp": "image/webp",
+ ".gif": "image/gif",
+};
+const SEGMENT = /^[a-z0-9][a-z0-9-]{0,63}$/;
+
+async function serve(request: Request, abs: string): Promise<Response> {
+ const type = TYPES[path.extname(abs).toLowerCase()];
+ if (!type) return new Response("not a media file", { status: 400 });
+ const st = await stat(/* turbopackIgnore: true */ abs).catch(() => null);
+ if (!st?.isFile()) return new Response("not found", { status: 404 });
+ return rangeResponse(request, {
+ abs,
+ size: st.size,
+ headers: { "content-type": type, "accept-ranges": "bytes", "cache-control": "private, no-store" },
+ });
+}
+
+async function reportFile(site: string, report: string, rel: string): Promise<string | null> {
+ if (!SEGMENT.test(site) || !SEGMENT.test(report)) return null;
+ if (!rel || rel.includes("\0") || rel.startsWith("/") || rel.split("/").some((s) => s === ".." || s === "" || s === ".")) return null;
+ const dir = siteReportDir(sitesPaths(), site, report);
+ const abs = path.join(/* turbopackIgnore: true */ dir, rel);
+ if (!inside(dir, abs)) return null;
+ const real = await realpath(/* turbopackIgnore: true */ abs).catch(() => null);
+ if (!real) return null;
+ const roots = await Promise.all([SITES_DIR, REPORTS_ROOT].map((r) => realpath(/* turbopackIgnore: true */ r).catch(() => r)));
+ return roots.some((r) => inside(r, real)) ? real : null;
+}
+
+async function preparedFile(site: string, moment: string): Promise<string | null> {
+ if (!SEGMENT.test(site)) return null;
+ try {
+ const index = JSON.parse(await readFile(/* turbopackIgnore: true */ reportMediaIndexFile(sitesPaths(), site), "utf8"));
+ const entry = index?.moments?.[moment];
+ if (!entry || typeof entry.file !== "string") return null;
+ const dir = reportMediaDir(sitesPaths(), site);
+ const abs = path.resolve(/* turbopackIgnore: true */ dir, entry.file);
+ return inside(dir, abs) ? abs : null;
+ } catch {
+ return null;
+ }
+}
+
+export async function GET(request: Request) {
+ const url = new URL(request.url);
+ const q = (k: string) => url.searchParams.get(k) ?? "";
+ if (url.searchParams.has("corpus")) {
+ const abs = path.resolve(/* turbopackIgnore: true */ q("corpus"));
+ if (abs !== q("corpus") || !inside(CHANNELS_DIR, abs)) return new Response("outside the corpus", { status: 400 });
+ return serve(request, abs);
+ }
+ if (url.searchParams.has("moment")) {
+ const abs = await preparedFile(q("site"), q("moment"));
+ return abs ? serve(request, abs) : new Response("no prepared clip for that moment", { status: 404 });
+ }
+ const abs = await reportFile(q("site"), q("report"), q("file"));
+ return abs ? serve(request, abs) : new Response("no such report file", { status: 404 });
+}
diff --git a/umtool/app/api/sites/workspace/route.ts b/umtool/app/api/sites/workspace/route.ts
@@ -0,0 +1,23 @@
+import { readFile } from "node:fs/promises";
+import { workspaceFile } from "@/lib/articles/workspace.mjs";
+
+export const dynamic = "force-dynamic";
+
+// GET ?ws=<name under REPORTS_ROOT>&rel=<listed file> -- one workspace file,
+// READ-ONLY, only one that lib/articles/workspace.mjs lists. HTML is served
+// under a CSP sandbox (no scripts, no same-origin) for the page's sandboxed
+// iframe; markdown and JSON as plain text.
+export async function GET(request: Request) {
+ const url = new URL(request.url);
+ const f = await workspaceFile(url.searchParams.get("ws") ?? "", url.searchParams.get("rel") ?? "");
+ if (!f) return new Response("not a workspace file", { status: 404 });
+ const body = await readFile(/* turbopackIgnore: true */ f.real);
+ const html = f.rel.endsWith(".html");
+ return new Response(body, {
+ headers: {
+ "content-type": html ? "text/html; charset=utf-8" : f.rel.endsWith(".json") ? "application/json; charset=utf-8" : "text/plain; charset=utf-8",
+ "cache-control": "no-store",
+ ...(html ? { "content-security-policy": "sandbox; default-src 'none'; img-src data:; style-src 'unsafe-inline'" } : {}),
+ },
+ });
+}
diff --git a/umtool/app/browse/decisions/page.tsx b/umtool/app/browse/decisions/page.tsx
@@ -122,7 +122,9 @@ export default async function DecisionsPage({
<section key={id} data-project={id}>
<h2 className="mb-1.5 flex items-baseline gap-2">
<Link
- href={`/browse/${id}`}
+ // An article's notes are listed under its page's path
+ // (lib/decisions.ts noteDecisions), not a project's.
+ href={id.startsWith("sites/") ? `/${id}` : `/browse/${id}`}
className="font-mono text-[13px] text-[var(--color-text)] hover:text-[var(--color-sel)]"
>
{id}
diff --git a/umtool/app/sites/[site]/[report]/evidence/page.tsx b/umtool/app/sites/[site]/[report]/evidence/page.tsx
@@ -0,0 +1,60 @@
+import { notFound } from "next/navigation";
+import { reportCitationNumbers } from "yt-dlp-transcript-common/lib/report/uses";
+import BrowseHeader from "@/components/BrowseHeader";
+import EvidenceWalk, { type WalkItem } from "@/components/articles/EvidenceWalk";
+import { readArticleNotes, readReportFile, siteById } from "@/lib/articles/sites";
+import { corpusNotesFile } from "@/lib/paths";
+import { linkedProjects } from "@/lib/articles/links.mjs";
+import { sourceFor } from "@/lib/articles/sources.mjs";
+import type { NotesRead } from "@/lib/annotations/types";
+
+export const dynamic = "force-dynamic";
+
+// Every citation of one article, one per screen, in the article's own
+// numbering. `?c=<citationId>` opens on that citation.
+export default async function EvidenceWalkPage({
+ params,
+ searchParams,
+}: {
+ params: Promise<{ site: string; report: string }>;
+ searchParams: Promise<{ c?: string }>;
+}) {
+ const { site: siteId, report: reportId } = await params;
+ const { c } = await searchParams;
+ const site = siteById(siteId);
+ if (!site) notFound();
+ const read = await readReportFile(site.siteId, reportId);
+ if (!read.report) notFound();
+ const r = read.report;
+ const numbers = reportCitationNumbers(r);
+ const items: WalkItem[] = [...numbers.entries()]
+ .sort((a, b) => a[1] - b[1])
+ .map(([id, number]) => ({ id, number, kind: r.citations?.[id]?.kind ?? "?", quote: r.citations?.[id]?.quote ?? "" }));
+ const at = Math.max(0, items.findIndex((i) => i.id === c));
+ const [notes, source] = await Promise.all([readArticleNotes(site.siteId, reportId), sourceFor(site.siteId, reportId)]);
+ const links = await linkedProjects(site.siteId, reportId, { workspace: source?.workspace ?? null });
+ const initialNotes: NotesRead = {
+ subject: { kind: "article", site: site.siteId, report: reportId },
+ file: corpusNotesFile(site.siteId, reportId) ?? "",
+ token: notes.token,
+ doc: notes.doc,
+ source: notes.doc?.source ?? null,
+ };
+ return (
+ <div className="flex h-full flex-col">
+ <BrowseHeader
+ active="sites"
+ crumbs={[
+ { href: "/sites", label: "sites" },
+ { href: `/sites/${site.siteId}`, label: site.siteId },
+ { href: `/sites/${site.siteId}/${reportId}`, label: reportId },
+ { label: "evidence" },
+ ]}
+ note={`${items.length} citations`}
+ />
+ <EvidenceWalk site={site.siteId} report={reportId} items={items} initial={at} initialNotes={initialNotes}
+ videoProjects={links.linked.map((p: { id: string; title: string; generatedBy: string | null }) => ({ id: p.id, title: p.title, generatedBy: p.generatedBy }))}
+ />
+ </div>
+ );
+}
diff --git a/umtool/app/sites/[site]/[report]/page.tsx b/umtool/app/sites/[site]/[report]/page.tsx
@@ -0,0 +1,180 @@
+import path from "node:path";
+import Link from "next/link";
+import { notFound } from "next/navigation";
+import { readRevisionHead, REPORT_HISTORY_GIT_DIRNAME } from "yt-dlp-transcript-common/publish/reportHistory";
+import { siteReportDir } from "yt-dlp-transcript-common/publish/reportMedia";
+import BrowseHeader from "@/components/BrowseHeader";
+import ArticleReader from "@/components/articles/ArticleReader";
+import WorkspacePanel from "@/components/articles/WorkspacePanel";
+import { badgeVariants } from "@/components/ui/badge";
+import { articleView } from "@/lib/articles/article";
+import { articleWorkspaceListing, openWorkspaceFile } from "@/lib/articles/files";
+import { linkedProjects } from "@/lib/articles/links.mjs";
+import { readArticleNotes, readReportFile, siteById, sitesPaths } from "@/lib/articles/sites";
+import { sourceFor } from "@/lib/articles/sources.mjs";
+import { corpusNotesFile } from "@/lib/paths";
+import type { NotesRead } from "@/lib/annotations/types";
+
+export const dynamic = "force-dynamic";
+
+// One article: the reader with its notes (default), or `?tab=source` -- the
+// workspace files it was written from, its draft marked.
+//
+// ?status=open|resolved|all the notes filter it opens with
+// ?note=<id> the note it opens on (the decisions inbox links here)
+// ?ws=&rel= a workspace file open on the source tab
+
+type Search = { tab?: string; status?: string; note?: string; ws?: string; rel?: string };
+
+export default async function ArticlePage({
+ params,
+ searchParams,
+}: {
+ params: Promise<{ site: string; report: string }>;
+ searchParams: Promise<Search>;
+}) {
+ const { site: siteId, report: reportId } = await params;
+ const sp = await searchParams;
+ const site = siteById(siteId);
+ if (!site) notFound();
+ const read = await readReportFile(site.siteId, reportId);
+ if (!read.report && read.problems[0]?.message === "no report.json") notFound();
+
+ const [notes, source] = await Promise.all([readArticleNotes(site.siteId, reportId), sourceFor(site.siteId, reportId)]);
+ const links = await linkedProjects(site.siteId, reportId, { workspace: source?.workspace ?? null });
+ const gitDir = path.join(/* turbopackIgnore: true */ siteReportDir(sitesPaths(), site.siteId, reportId), REPORT_HISTORY_GIT_DIRNAME);
+ const head = await readRevisionHead(gitDir).catch(() => null);
+ const published = (site.reports ?? []).includes(reportId);
+ const r = read.report;
+ const tab = sp.tab === "source" ? "source" : "article";
+ const base = `/sites/${site.siteId}/${reportId}`;
+ const media = (file: string) =>
+ `/api/sites/media?site=${encodeURIComponent(site.siteId)}&report=${encodeURIComponent(reportId)}&file=${encodeURIComponent(file)}`;
+
+ const meta = (
+ <div data-article-meta className="flex flex-wrap items-center gap-x-3 gap-y-1 text-[12px] text-[var(--color-dim)]">
+ <Link href={`/sites/${site.siteId}`} className="hover:text-[var(--color-text)]">
+ {site.siteTitle || site.siteId}
+ </Link>
+ <span data-status={published ? "published" : "draft"} className={badgeVariants({ variant: published ? "on" : "neutral", size: "sm" })}>
+ {published ? "published" : "draft"}
+ </span>
+ {r?.published && <span>published {r.published}</span>}
+ {r?.updated && <span>updated {r.updated}</span>}
+ {head && <span data-revisions={head.revision}>{head.revision} revision{head.revision === 1 ? "" : "s"}</span>}
+ {links.linked.map((p: { id: string }) => (
+ <Link key={p.id} href={`/browse/${p.id}`} data-project-link={p.id} className="font-mono text-[var(--color-sel)] hover:underline">
+ {p.id}
+ </Link>
+ ))}
+ {links.possible.map((p: { id: string }) => (
+ <Link key={p.id} href={`/browse/${p.id}`} className="font-mono hover:underline" title="shares the slug; not linked">
+ {p.id} (possible)
+ </Link>
+ ))}
+ <nav className="ml-auto flex gap-2" aria-label="article views">
+ <Link href={base} aria-current={tab === "article" ? "page" : undefined} className={tab === "article" ? "text-[var(--color-sel)]" : "hover:text-[var(--color-text)]"}>
+ article
+ </Link>
+ <Link href={`${base}?tab=source`} aria-current={tab === "source" ? "page" : undefined} className={tab === "source" ? "text-[var(--color-sel)]" : "hover:text-[var(--color-text)]"}>
+ source
+ </Link>
+ <Link href={`${base}/evidence`} className="hover:text-[var(--color-text)]">
+ evidence
+ </Link>
+ </nav>
+ {source && (
+ <div className="w-full font-mono text-[11px]" title={source.how}>
+ {source.draft && <span>draft {source.draft}</span>}
+ {source.generator && <span className="ml-3">generator {source.generator}</span>}
+ </div>
+ )}
+ </div>
+ );
+
+ const header = (
+ <BrowseHeader
+ active="sites"
+ crumbs={[{ href: "/sites", label: "sites" }, { href: `/sites/${site.siteId}`, label: site.siteId }, { label: reportId }]}
+ note={`${notes.doc?.notes.filter((n) => n.status === "open").length ?? 0} open notes`}
+ />
+ );
+
+ if (!r) {
+ return (
+ <div className="flex h-full flex-col">
+ {header}
+ <main className="deck-main flex-1 p-4">
+ <div className="mx-auto max-w-[72ch] space-y-3">
+ {meta}
+ <p className="text-[13px] text-[var(--color-bad)]">report.json does not read as a report:</p>
+ <ul className="list-disc pl-5 text-[12px] text-[var(--color-dim)]">
+ {read.problems.slice(0, 20).map((p, i) => (
+ <li key={i}>{p.message}</li>
+ ))}
+ </ul>
+ </div>
+ </main>
+ </div>
+ );
+ }
+
+ if (tab === "source") {
+ const ws = await articleWorkspaceListing(source?.workspace);
+ const opened =
+ sp.ws && sp.rel
+ ? await openWorkspaceFile(sp.ws, sp.rel)
+ : ws && source?.draft
+ ? await openWorkspaceFile(ws.name, path.relative(ws.dir, source.draft))
+ : null;
+ const draftRel = ws && source?.draft ? `${ws.name}/${path.relative(ws.dir, source.draft)}` : null;
+ return (
+ <div className="flex h-full flex-col">
+ {header}
+ <main className="deck-main flex-1 p-4">
+ <div className="space-y-4">
+ {meta}
+ <WorkspacePanel
+ workspaces={ws ? [ws] : []}
+ opened={opened}
+ highlight={draftRel}
+ hrefFor={(w, rel) => `${base}?${new URLSearchParams({ tab: "source", ws: w, rel })}`}
+ />
+ </div>
+ </main>
+ </div>
+ );
+ }
+
+ const { view, error } = await articleView(r);
+ const initialNotes: NotesRead = {
+ subject: { kind: "article", site: site.siteId, report: reportId },
+ file: corpusNotesFile(site.siteId, reportId) ?? "",
+ token: notes.token,
+ doc: notes.doc,
+ source: notes.doc?.source ?? source ?? null,
+ ...(notes.error ? { error: notes.error } : {}),
+ };
+ const status = sp.status === "resolved" || sp.status === "all" ? sp.status : "open";
+ const video = r.video
+ ? { file: r.video.src, src: media(r.video.src), poster: r.video.poster ? media(r.video.poster) : undefined, caption: r.video.caption }
+ : null;
+
+ return (
+ <div className="flex h-full flex-col">
+ {header}
+ <ArticleReader
+ site={site.siteId}
+ report={reportId}
+ view={view}
+ viewError={error}
+ initialNotes={initialNotes}
+ initialFilter={status}
+ initialNote={sp.note ?? null}
+ meta={meta}
+ video={video}
+ videoProjects={links.linked.map((p: { id: string; title: string; generatedBy: string | null }) => ({ id: p.id, title: p.title, generatedBy: p.generatedBy }))}
+ />
+ </div>
+ );
+}
diff --git a/umtool/app/sites/[site]/page.tsx b/umtool/app/sites/[site]/page.tsx
@@ -0,0 +1,123 @@
+import Link from "next/link";
+import { notFound } from "next/navigation";
+import BrowseHeader from "@/components/BrowseHeader";
+import ArticleTable, { articleHref } from "@/components/articles/ArticleTable";
+import SiteChips from "@/components/articles/SiteChips";
+import WorkspacePanel from "@/components/articles/WorkspacePanel";
+import { badgeVariants } from "@/components/ui/badge";
+import { readSiteRow, siteById } from "@/lib/articles/sites";
+import { openWorkspaceFile, siteWorkspaceListings, takeTally } from "@/lib/articles/files";
+import { videoProjects } from "@/lib/articles/links.mjs";
+
+export const dynamic = "force-dynamic";
+
+// One site: its articles, the videos they play, the umtool projects those
+// videos were cut in (with their takes), and the workspace files the articles
+// were written from. `?ws=&rel=` opens a workspace file below.
+
+export default async function SitePage({
+ params,
+ searchParams,
+}: {
+ params: Promise<{ site: string }>;
+ searchParams: Promise<{ ws?: string; rel?: string }>;
+}) {
+ const { site: siteId } = await params;
+ const sp = await searchParams;
+ const site = siteById(siteId);
+ if (!site) notFound();
+ const row = await readSiteRow(site);
+
+ const projects = await videoProjects();
+ const linkedIds = [...new Set(row.articles.flatMap((a) => a.projects.linked.map((p) => p.id)))];
+ const linked = await Promise.all(
+ linkedIds.map(async (id) => {
+ const p = projects.find((x: { id: string }) => x.id === id) as { id: string; dir: string };
+ return { id: p.id, tally: await takeTally(p.dir), articles: row.articles.filter((a) => a.projects.linked.some((l) => l.id === id)) };
+ }),
+ );
+ const videos = row.articles.filter((a) => a.hasVideo);
+ const workspaces = await siteWorkspaceListings(site.siteId, row.articles.map((a) => a.id));
+ const opened = sp.ws && sp.rel ? await openWorkspaceFile(sp.ws, sp.rel) : null;
+ const hrefFor = (ws: string, rel: string) =>
+ `/sites/${site.siteId}?${new URLSearchParams({ ws, rel }).toString()}#files`;
+ const media = (id: string, file: string) =>
+ `/api/sites/media?site=${encodeURIComponent(site.siteId)}&report=${encodeURIComponent(id)}&file=${file}`;
+
+ return (
+ <div className="flex h-full flex-col">
+ <BrowseHeader
+ active="sites"
+ crumbs={[{ href: "/sites", label: "sites" }, { label: row.title }]}
+ note={`${row.published} published · ${row.drafts} drafts · ${row.openNotes} open notes`}
+ />
+ <main className="deck-main flex-1 space-y-6 p-4">
+ <div className="flex flex-wrap items-baseline gap-2">
+ <h1 className="text-[16px] font-semibold text-[var(--color-text)]">{row.title}</h1>
+ <span className="font-mono text-[11px] text-[var(--color-dim)]">{row.siteId}</span>
+ <SiteChips site={row} />
+ </div>
+
+ <section data-section="articles">
+ <h2 className="micro mb-1.5">articles</h2>
+ <ArticleTable articles={row.articles} />
+ </section>
+
+ <section data-section="videos">
+ <h2 className="micro mb-1.5">report videos {videos.length}</h2>
+ {videos.length === 0 ? (
+ <p className="text-[12px] text-[var(--color-dim)]">none</p>
+ ) : (
+ <div className="grid grid-cols-[repeat(auto-fill,minmax(280px,1fr))] gap-3">
+ {videos.map((a) => (
+ <figure key={a.id} data-report-video={a.id} className="rounded border border-[var(--color-line)] bg-[var(--color-panel)] p-2">
+ <video
+ controls
+ preload="none"
+ src={media(a.id, "video.mp4")}
+ poster={a.hasPoster ? media(a.id, "poster.jpg") : undefined}
+ className="aspect-video w-full rounded bg-black"
+ />
+ <figcaption className="mt-1 text-[12px]">
+ <Link href={articleHref(a)} className="text-[var(--color-text)] hover:text-[var(--color-sel)]">
+ {a.title}
+ </Link>
+ </figcaption>
+ </figure>
+ ))}
+ </div>
+ )}
+ </section>
+
+ <section data-section="projects">
+ <h2 className="micro mb-1.5">video projects {linked.length}</h2>
+ {linked.length === 0 ? (
+ <p className="text-[12px] text-[var(--color-dim)]">none linked</p>
+ ) : (
+ <ul className="space-y-1">
+ {linked.map((p) => (
+ <li key={p.id} data-video-project={p.id} className="flex flex-wrap items-baseline gap-2 rounded border border-[var(--color-line)] bg-[var(--color-panel)] px-3 py-1.5 text-[12px]">
+ <Link href={`/browse/${p.id}`} className="font-mono text-[var(--color-sel)] hover:underline">
+ {p.id}
+ </Link>
+ <span className="text-[var(--color-dim)]">{p.articles.map((a) => a.title).join(", ")}</span>
+ <Link href={`/browse/${p.id}/takes`} className="ml-auto text-[var(--color-dim)] hover:text-[var(--color-text)]" data-takes={p.tally.takes}>
+ {p.tally.takes} takes
+ </Link>
+ <span className={badgeVariants({ variant: "neutral", size: "sm" })}>like {p.tally.like}</span>
+ <span className={badgeVariants({ variant: "neutral", size: "sm" })}>maybe {p.tally.maybe}</span>
+ <span className={badgeVariants({ variant: "neutral", size: "sm" })}>no {p.tally.no}</span>
+ </li>
+ ))}
+ </ul>
+ )}
+ </section>
+
+ <section data-section="files" id="files">
+ <h2 className="micro mb-1.5">workspace files</h2>
+ <WorkspacePanel workspaces={workspaces} opened={opened} hrefFor={hrefFor} />
+ </section>
+ </main>
+ </div>
+ );
+}
diff --git a/umtool/app/sites/page.tsx b/umtool/app/sites/page.tsx
@@ -0,0 +1,87 @@
+import Link from "next/link";
+import BrowseHeader from "@/components/BrowseHeader";
+import ArticleTable from "@/components/articles/ArticleTable";
+import SiteChips from "@/components/articles/SiteChips";
+import { badgeVariants } from "@/components/ui/badge";
+import { listSiteRows, type ArticleRow } from "@/lib/articles/sites";
+
+export const dynamic = "force-dynamic";
+
+// Every site's articles, private sites first. Zero client JS: the filters are
+// links that change searchParams (/browse/decisions' idiom), so a filtered
+// view is one pasteable URL.
+
+type Search = { site?: string; status?: string; notes?: string };
+
+export default async function SitesPage({ searchParams }: { searchParams: Promise<Search> }) {
+ const sp = await searchParams;
+ const all = await listSiteRows();
+
+ const keep = (a: ArticleRow) =>
+ (!sp.status || a.status === sp.status) && (sp.notes !== "open" || a.openNotes > 0);
+ const sites = all
+ .filter((s) => !sp.site || s.siteId === sp.site)
+ .map((s) => ({ ...s, shown: s.articles.filter(keep) }))
+ .filter((s) => s.shown.length > 0 || (!sp.status && !sp.notes));
+
+ const articles = all.flatMap((s) => s.articles);
+ const open = articles.reduce((n, a) => n + a.openNotes, 0);
+ const qs = (next: Partial<Search>) => {
+ const p = new URLSearchParams();
+ for (const [k, v] of Object.entries({ ...sp, ...next })) if (v) p.set(k, String(v));
+ const s = p.toString();
+ return `/sites${s ? `?${s}` : ""}`;
+ };
+
+ return (
+ <div className="flex h-full flex-col">
+ <BrowseHeader active="sites" crumbs={[{ label: "sites" }]} note={`${all.length} sites · ${articles.length} articles · ${open} open notes`} />
+ <main className="deck-main flex-1 p-4">
+ <div className="mb-3 flex flex-wrap items-center gap-1.5">
+ <span className="micro">site</span>
+ <Chip href={qs({ site: "" })} on={!sp.site} label={`all ${all.length}`} />
+ {all.map((s) => (
+ <Chip key={s.siteId} href={qs({ site: s.siteId })} on={sp.site === s.siteId} label={s.siteId} />
+ ))}
+ <span className="micro ml-3">status</span>
+ <Chip href={qs({ status: "" })} on={!sp.status} label="all" />
+ <Chip href={qs({ status: "published" })} on={sp.status === "published"} label={`published ${articles.filter((a) => a.status === "published").length}`} />
+ <Chip href={qs({ status: "draft" })} on={sp.status === "draft"} label={`draft ${articles.filter((a) => a.status === "draft").length}`} />
+ <span className="micro ml-3">notes</span>
+ <Chip href={qs({ notes: "" })} on={sp.notes !== "open"} label="all" />
+ <Chip href={qs({ notes: "open" })} on={sp.notes === "open"} label={`open ${articles.filter((a) => a.openNotes > 0).length}`} />
+ </div>
+
+ {sites.length === 0 ? (
+ <p className="text-[12px] text-[var(--color-dim)]">{all.length === 0 ? "no sites" : "nothing matches that filter"}</p>
+ ) : (
+ <div className="space-y-6">
+ {sites.map((s) => (
+ <section key={s.siteId} data-site={s.siteId}>
+ <h2 className="mb-1.5 flex flex-wrap items-baseline gap-2">
+ <Link href={`/sites/${s.siteId}`} className="text-[14px] font-semibold text-[var(--color-text)] hover:text-[var(--color-sel)]">
+ {s.title}
+ </Link>
+ <span className="font-mono text-[11px] text-[var(--color-dim)]">{s.siteId}</span>
+ <SiteChips site={s} />
+ <span className="micro" data-counts={`${s.published}/${s.drafts}`}>
+ {s.published} published · {s.drafts} draft{s.drafts === 1 ? "" : "s"}
+ </span>
+ </h2>
+ <ArticleTable articles={s.shown} />
+ </section>
+ ))}
+ </div>
+ )}
+ </main>
+ </div>
+ );
+}
+
+function Chip({ href, on, label }: { href: string; on: boolean; label: string }) {
+ return (
+ <Link href={href} aria-current={on ? "true" : undefined} className={badgeVariants({ variant: on ? "on" : "neutral" })}>
+ {label}
+ </Link>
+ );
+}
diff --git a/umtool/bin/umtool.mjs b/umtool/bin/umtool.mjs
@@ -36,6 +36,9 @@
// umtool diff <project> <snapshot> what changed since that snapshot
// umtool export <project> --format toc-bbcode|toc-markdown|description|chapters [--variant V]
// umtool check-sources [<project>…] prints the re-check chain
+// umtool notes [<site>/<report> | <project> | --all] [--open|--resolved|--all-status] [--json]
+// umtool notes reply <id> "<text>" [--resolve] | resolve | wontfix | reopen <id>
+// the operator's notes, and the agent's answers
import process from "node:process";
import {
PROJECT_KINDS,
@@ -65,6 +68,7 @@ import { buildSteps, checkSourcesSteps, PRESETS } from "../lib/report/driver.mjs
import { openIndex, signRecord } from "../lib/projects/index-db.mjs";
import { pipelineProcessesFor } from "../lib/report/busy.mjs";
import { probeTools } from "../lib/tools.mjs";
+import { notesCommand } from "../lib/annotations/cli.mjs";
import { scaffoldReportVideo } from "../lib/projects/scaffold.mjs";
import { CACHE_DIR, INDEX_DIR, MEDIA_ROOT, MEDIA_TIERED, OLD_CACHE_DIR } from "../lib/paths.mjs";
import {
@@ -698,6 +702,8 @@ function usage() {
" umtool diff <project> <snapshot> what changed since that snapshot",
" umtool export <project> --format toc-bbcode|toc-markdown|description|chapters [--variant V]",
" umtool check-sources [<project>…] prints the re-check chain (never-checked/old when no args)",
+ " umtool notes [<site>/<report>|<project>|--all] the operator's notes, as markdown (open ones)",
+ " umtool notes reply <id> \"<text>\" [--resolve] answer one; resolve | wontfix | reopen <id>",
"",
`reading ${REPORTS_ROOT} (set REPORTS_DIR to move it)`,
"",
@@ -725,6 +731,7 @@ const COMMANDS = {
diff: cmdDiff,
export: cmdExport,
"check-sources": cmdCheckSources,
+ notes: async () => process.exit(await notesCommand(argv.slice(argv.indexOf("notes") + 1))),
help: usage,
};
diff --git a/umtool/components/AppNav.tsx b/umtool/components/AppNav.tsx
@@ -6,7 +6,8 @@ import NavGroup from "./NavGroup";
// is waiting, the two benches that are not a project (mix, find), and the song
// piles folded under one entry.
//
-// SEVEN visible entries, and the cap is still NINE. A tenth wraps the header on
+// SEVEN visible entries (home, browse, decisions, sites, mix, find, song ▸),
+// and the cap is still NINE. A tenth wraps the header on
// a laptop, and a nav that wraps stops reading as one row of places and starts
// reading as a list. The next tool goes UNDER one of these, not beside them --
// which is exactly what happened to the four judging piles and the sources
@@ -30,6 +31,9 @@ export default function AppNav({ active }: { active: string }) {
// The worklist across every project, not a sixth pile. It sits beside
// browse because that is where every decision it names gets settled.
{ href: "/browse/decisions", label: "decisions" },
+ // Every site's articles -- published and drafts -- with their notes, their
+ // evidence and the workspace they were written in.
+ { href: "/sites", label: "sites" },
{ href: "/mix", label: "mix" },
// Every occurrence of a word across the corpus. It sits with browse because
// what it retrieves is raw material for a build, not a pile to judge.
diff --git a/umtool/components/articles/AddToVideo.tsx b/umtool/components/articles/AddToVideo.tsx
@@ -0,0 +1,78 @@
+"use client";
+
+import Link from "next/link";
+import { useState } from "react";
+import type { Evidence } from "@/lib/articles/evidence";
+
+// "Add to video": the cited span as a new clip at the END of a linked
+// report-video project's timeline, through the structure route
+// (/api/report/timeline, op insert). That route snapshots the manifest first
+// (Undo on the project page restores it) and, on a GENERATED manifest, leaves
+// an `edit` note so the agent ports the clip into the generator's inputs.
+
+export type VideoProjectLink = { id: string; title: string; generatedBy: string | null };
+
+export default function AddToVideo({ ev, projects }: { ev: Evidence; projects: VideoProjectLink[] }) {
+ const [project, setProject] = useState(projects[0]?.id ?? "");
+ const [state, setState] = useState<{ busy: boolean; done?: { id: string; project: string }; error?: string }>({ busy: false });
+ if (!projects.length || !ev.record || ev.start === undefined || ev.end === undefined) return null;
+ const chosen = projects.find((p) => p.id === project) ?? projects[0];
+
+ const add = async () => {
+ setState({ busy: true });
+ try {
+ const g = await fetch(`/api/report/timeline?project=${encodeURIComponent(chosen.id)}`, { cache: "no-store" });
+ const gj = await g.json();
+ if (!g.ok) throw new Error(gj.error ?? g.statusText);
+ const last = gj.timeline.length ? gj.timeline[gj.timeline.length - 1] : null;
+ const entry: Record<string, unknown> = {
+ type: "clip",
+ channel: ev.record!.channel,
+ video: ev.record!.id,
+ start: ev.start,
+ end: ev.end,
+ quote: ev.quote,
+ };
+ if (/^[A-Za-z0-9_-]{1,60}$/.test(ev.cite)) entry.id = `cite-${ev.cite}`;
+ const r = await fetch("/api/report/timeline", {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ project: chosen.id, token: gj.token, op: "insert", afterId: last?.id ?? null, at: last ? gj.timeline.length - 1 : null, entry }),
+ });
+ const j = await r.json();
+ if (!r.ok) throw new Error(j.error ?? r.statusText);
+ setState({ busy: false, done: { id: j.id ?? j.result?.id ?? "the clip", project: chosen.id } });
+ } catch (err) {
+ setState({ busy: false, error: err instanceof Error ? err.message : String(err) });
+ }
+ };
+
+ return (
+ <div data-add-to-video className="flex flex-wrap items-center gap-2 border-t border-[var(--color-line)] pt-2">
+ {projects.length > 1 ? (
+ <select value={chosen.id} onChange={(e) => setProject(e.target.value)} aria-label="video project" className="rounded border border-[var(--color-line)] bg-transparent px-1 py-0.5 font-mono text-[11px]">
+ {projects.map((p) => (
+ <option key={p.id} value={p.id}>
+ {p.id}
+ </option>
+ ))}
+ </select>
+ ) : (
+ <span className="font-mono text-[11px] text-[var(--color-dim)]">{chosen.id}</span>
+ )}
+ <button type="button" onClick={add} disabled={state.busy} className="rounded border border-[var(--color-line)] px-2 py-0.5 hover:border-[var(--color-sel)] disabled:opacity-50">
+ Add to video
+ </button>
+ {chosen.generatedBy && <span className="text-[11px] text-[var(--color-dim)]">generated; leaves an edit note</span>}
+ {state.done && (
+ <span role="status" className="text-[11px]">
+ added <span className="font-mono">{state.done.id}</span> —{" "}
+ <Link href={`/browse/${state.done.project}`} className="underline">
+ open
+ </Link>
+ </span>
+ )}
+ {state.error && <span role="alert" className="text-[11px] text-[var(--color-bad)]">{state.error}</span>}
+ </div>
+ );
+}
diff --git a/umtool/components/articles/ArticleBody.tsx b/umtool/components/articles/ArticleBody.tsx
@@ -0,0 +1,178 @@
+"use client";
+
+import { createContext, memo, useContext, type AnchorHTMLAttributes, type ReactNode } from "react";
+import { Markdown } from "yt-dlp-transcript-common/components/Markdown";
+import type { CitationView, ReportPageView } from "yt-dlp-transcript-common/lib/report/views";
+
+// The article's text, as blocks the notes anchor to (`data-block`: title,
+// subtitle, summary, method, each section by id). MEMOISED and never
+// re-rendered once mounted: the reader wraps notes' quotes in <mark>s by
+// editing this DOM directly (components/articles/anchorDom.ts), which is safe
+// only because React has no reason to touch it again. Everything that changes
+// -- which citation is open, which note is selected -- goes through the
+// context below, whose value never changes.
+//
+// A citation is a button (its label, which is part of the text) and its
+// number (which is not: `data-anchor-skip`). Clicking it opens the evidence
+// panel; nothing here links out of umtool.
+
+export type ArticleActions = {
+ openCite: (id: string) => void;
+ noteSection: (section: string, title: string) => void;
+};
+
+export const ArticleActionsContext = createContext<ArticleActions | null>(null);
+
+const CitationsContext = createContext<Readonly<Record<string, CitationView>>>({});
+
+const CITE_SCHEME = "cite:";
+
+function CiteLink({ href, children }: AnchorHTMLAttributes<HTMLAnchorElement>) {
+ const actions = useContext(ArticleActionsContext);
+ const citations = useContext(CitationsContext);
+ const id = href?.startsWith(CITE_SCHEME) ? href.slice(CITE_SCHEME.length).trim() : null;
+ if (id === null) {
+ return (
+ <a href={href} target="_blank" rel="noopener noreferrer" className="text-[var(--color-sel)] underline decoration-[var(--color-sel)]/40">
+ {children}
+ </a>
+ );
+ }
+ const c = citations[id];
+ return (
+ <>
+ <button
+ type="button"
+ data-cite={id}
+ onClick={() => actions?.openCite(id)}
+ title={c ? `${c.quote}${c.speaker ? ` — ${c.speaker}` : ""}` : `citation ${id}`}
+ className="cursor-pointer rounded-sm text-left text-[var(--color-text)] underline decoration-[var(--color-sel)] decoration-dotted underline-offset-2 hover:bg-[var(--color-panel-2)] data-[has-note=true]:bg-[color-mix(in_srgb,var(--color-dirty)_22%,transparent)]"
+ >
+ {children}
+ </button>
+ <sup data-anchor-skip className="ml-0.5 font-mono text-[10px] text-[var(--color-sel)]">
+ {c?.number ?? "?"}
+ </sup>
+ </>
+ );
+}
+
+// markdown-to-jsx's elements carry common's class names, which umtool's
+// stylesheet does not have; these descendant rules are umtool's.
+const MD =
+ "text-[14px] leading-relaxed text-[var(--color-text)] [&_p]:my-2.5 [&_ul]:my-2 [&_ul]:list-disc [&_ul]:pl-5 [&_ol]:my-2 [&_ol]:list-decimal [&_ol]:pl-5 [&_li]:my-1 [&_blockquote]:my-2 [&_blockquote]:border-l-2 [&_blockquote]:border-[var(--color-line)] [&_blockquote]:pl-3 [&_blockquote]:text-[var(--color-dim)] [&_strong]:font-semibold [&_em]:italic [&_h3]:mt-4 [&_h3]:font-semibold [&_h4]:mt-3 [&_h4]:font-semibold [&_code]:font-mono [&_code]:text-[12px] [&_hr]:my-4 [&_hr]:border-[var(--color-line)]";
+
+function Md({ text }: { text: string }) {
+ return (
+ <Markdown className={MD} linkComponent={CiteLink}>
+ {text}
+ </Markdown>
+ );
+}
+
+function NoteButton({ section, title }: { section: string; title: string }) {
+ const actions = useContext(ArticleActionsContext);
+ return (
+ <button
+ type="button"
+ data-anchor-skip
+ aria-label={`note on ${title}`}
+ onClick={() => actions?.noteSection(section, title)}
+ className="ml-2 align-middle text-[11px] font-normal text-[var(--color-dim)] opacity-60 hover:text-[var(--color-sel)] hover:opacity-100"
+ >
+ + note
+ </button>
+ );
+}
+
+function ArticleBodyInner({ view, video }: { view: ReportPageView; video?: ReactNode }) {
+ return (
+ <CitationsContext.Provider value={view.citations}>
+ <div data-article-body className="space-y-5">
+ <header className="space-y-1">
+ {view.series && <div className="micro">{view.series}</div>}
+ <h1 data-block="title" className="text-[22px] font-semibold leading-tight text-[var(--color-text)]">
+ {view.title}
+ </h1>
+ {view.subtitle && (
+ <p data-block="subtitle" className="text-[15px] text-[var(--color-dim)]">
+ {view.subtitle}
+ </p>
+ )}
+ </header>
+ {/* The report's own video, under its title as the published page has
+ it. Outside every data-block, so it is never part of a quote. */}
+ {video}
+ {view.summary && (
+ <section data-block="summary" className="rounded border border-[var(--color-line)] bg-[var(--color-panel)] px-4 py-2">
+ <div className="micro pt-1" data-anchor-skip>
+ summary
+ <NoteButton section="summary" title="summary" />
+ </div>
+ <Md text={view.summary} />
+ </section>
+ )}
+ {view.sections.map((s) => (
+ <section key={s.id} id={`s-${s.id}`} data-block={s.id} className="scroll-mt-4">
+ <h2 className="mt-2 text-[17px] font-semibold text-[var(--color-text)]">
+ {s.title}
+ <NoteButton section={s.id} title={s.title} />
+ </h2>
+ {s.body && <Md text={s.body} />}
+ {s.claims.map((cl) => (
+ <div key={cl.id} data-claim={cl.id} className="my-3 rounded border border-[var(--color-line)] bg-[var(--color-panel)] px-3 py-2">
+ {cl.title && <div className="text-[13px] font-semibold text-[var(--color-text)]">{cl.title}</div>}
+ <div className="flex items-start gap-2">
+ <div className="min-w-0 flex-1">
+ <Md text={cl.text} />
+ </div>
+ {cl.verdict && (
+ <span data-anchor-skip className="mt-2 shrink-0 rounded border border-[var(--color-line)] px-1.5 py-0.5 font-mono text-[10px] uppercase text-[var(--color-dim)]">
+ {view.verdicts?.[cl.verdict]?.label ?? cl.verdict}
+ </span>
+ )}
+ </div>
+ {cl.findings && <Md text={cl.findings} />}
+ {cl.citations.length > 0 && (
+ <div data-anchor-skip className="mt-1 flex flex-wrap gap-1">
+ {cl.citations.map((id) => (
+ <CiteChip key={id} id={id} />
+ ))}
+ </div>
+ )}
+ </div>
+ ))}
+ </section>
+ ))}
+ {view.method && (
+ <section data-block="method" className="border-t border-[var(--color-line)] pt-3">
+ <div className="micro" data-anchor-skip>
+ method
+ <NoteButton section="method" title="method" />
+ </div>
+ <Md text={view.method} />
+ </section>
+ )}
+ </div>
+ </CitationsContext.Provider>
+ );
+}
+
+function CiteChip({ id }: { id: string }) {
+ const actions = useContext(ArticleActionsContext);
+ const c = useContext(CitationsContext)[id];
+ return (
+ <button
+ type="button"
+ data-cite={id}
+ onClick={() => actions?.openCite(id)}
+ title={c?.quote ?? id}
+ className="rounded border border-[var(--color-line)] px-1.5 py-0.5 font-mono text-[10px] text-[var(--color-sel)] hover:border-[var(--color-sel)] data-[has-note=true]:border-[var(--color-dirty)]"
+ >
+ {c?.number ?? "?"} {c?.label ?? c?.speaker ?? id}
+ </button>
+ );
+}
+
+const ArticleBody = memo(ArticleBodyInner, (a, b) => a.view === b.view && a.video === b.video);
+export default ArticleBody;
diff --git a/umtool/components/articles/ArticleReader.tsx b/umtool/components/articles/ArticleReader.tsx
@@ -0,0 +1,413 @@
+"use client";
+
+import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState, type ReactNode } from "react";
+import type { ReportPageView } from "yt-dlp-transcript-common/lib/report/views";
+import CopyButton from "@/components/CopyButton";
+import { locateQuote, quoteAnchor } from "@/lib/annotations/anchor.mjs";
+import type { Anchor, Note, NotesRead } from "@/lib/annotations/types";
+import { useNotes } from "@/lib/annotations/useNotes";
+import { ShareNotes } from "@/components/notes/NotesProvider";
+import ArticleBody, { ArticleActionsContext, type ArticleActions } from "./ArticleBody";
+import ArticleVideo from "./ArticleVideo";
+import type { VideoProjectLink } from "./AddToVideo";
+import { blockText, selectionIn, unwrapMarks, wrapRange } from "./anchorDom";
+import EvidencePanel, { useEvidence } from "./EvidencePanel";
+import { Composer, NoteCard, anchorLabel } from "./NoteCards";
+
+// The article, its notes, and its evidence, in one screen: the text in a
+// ~70ch column, the notes in a rail beside it (a drawer below 1100px).
+//
+// select text → "Note" (or `n`) a note on that quote
+// "+ note" on a heading a note on the section
+// a citation its evidence in the rail; a note on it there
+// "+ whole article" a note on all of it
+//
+// Notes on a quote are found again in the text as it is NOW (lib/annotations/
+// anchor.mjs: the quote, then its context) and marked; one whose quote is gone
+// is still listed, flagged "orphaned", under its section.
+//
+// Keys (not while typing): j/k move between notes, r resolves, e edits, n
+// notes the selection (or the whole article), Esc closes whatever is open.
+
+type Filter = "open" | "resolved" | "all";
+
+const MARK_CLASS =
+ "rounded-sm bg-[color-mix(in_srgb,var(--color-sel)_22%,transparent)] text-inherit data-[active=true]:bg-[color-mix(in_srgb,var(--color-sel)_48%,transparent)] data-[status=resolved]:bg-transparent data-[status=resolved]:underline data-[status=resolved]:decoration-[var(--color-good)] data-[status=wontfix]:bg-transparent";
+
+type ComposerState = { anchor: Anchor; label: string } | null;
+
+export default function ArticleReader({
+ site,
+ report,
+ view,
+ initialNotes,
+ initialFilter = "open",
+ initialNote = null,
+ meta,
+ video,
+ videoProjects = [],
+ viewError,
+}: {
+ site: string;
+ report: string;
+ view: ReportPageView;
+ initialNotes: NotesRead;
+ initialFilter?: Filter;
+ initialNote?: string | null;
+ meta: ReactNode;
+ /** The report's own video (report.json `video`), played with timed notes. */
+ video?: { file: string; src: string; poster?: string; caption?: string } | null;
+ viewError?: string | null;
+ /** The article's LINKED video projects: where "Add to video" puts a cited span. */
+ videoProjects?: VideoProjectLink[];
+}) {
+ const notesApi = useNotes({ article: `${site}/${report}` }, { initial: initialNotes });
+ const { notes, write, busy, error } = notesApi;
+ // Stable across note writes: ArticleBody is memoised on it (its marks are
+ // laid over the rendered DOM). The player reads live notes through
+ // <ShareNotes>, not through this element.
+ const videoSlot = useMemo(
+ () =>
+ video ? (
+ <ArticleVideo site={site} report={report} file={video.file} src={video.src} poster={video.poster} caption={video.caption} />
+ ) : null,
+ [site, report, video?.file, video?.src, video?.poster, video?.caption],
+ );
+ const [filter, setFilter] = useState<Filter>(initialFilter);
+ const [selected, setSelected] = useState<string | null>(initialNote);
+ const [editing, setEditing] = useState<string | null>(null);
+ const [composer, setComposer] = useState<ComposerState>(null);
+ const [pending, setPending] = useState<{ anchor: Anchor; label: string; x: number; y: number } | null>(null);
+ const [cite, setCite] = useState<string | null>(null);
+ const [orphans, setOrphans] = useState<Set<string>>(new Set());
+ const [hover, setHover] = useState<string | null>(null);
+ const [drawer, setDrawer] = useState(false);
+ const bodyRef = useRef<HTMLDivElement>(null);
+ const evidence = useEvidence(site, report, cite);
+
+ const titles = useMemo(() => {
+ const t: Record<string, string> = { title: "title", subtitle: "subtitle", summary: "summary", method: "method" };
+ for (const s of view.sections) t[s.id] = s.title;
+ return t;
+ }, [view]);
+ const numbers = useMemo(() => Object.fromEntries(Object.entries(view.citations).map(([id, c]) => [id, c.number])), [view]);
+ const blockOrder = useMemo(() => ["title", "subtitle", "summary", ...view.sections.map((s) => s.id), "method"], [view]);
+
+ const visible = useMemo(() => {
+ const keep = (n: Note) => filter === "all" || (filter === "open" ? n.status === "open" : n.status !== "open");
+ const rank = (n: Note) => {
+ const a = n.anchor;
+ if (a.kind === "whole") return -1;
+ if (a.kind === "text" || a.kind === "section") return blockOrder.indexOf(a.section) + 0.5;
+ if (a.kind === "cite") return 1000 + (numbers[a.cite] ?? 0);
+ return 2000;
+ };
+ return notes.filter(keep).sort((a, b) => rank(a) - rank(b) || a.at.localeCompare(b.at));
+ }, [notes, filter, blockOrder, numbers]);
+
+ // ---- marks ---------------------------------------------------------------
+ useLayoutEffect(() => {
+ const root = bodyRef.current;
+ if (!root) return;
+ unwrapMarks(root);
+ const lost = new Set<string>();
+ for (const n of visible) {
+ const a = n.anchor;
+ if (a.kind !== "text") continue;
+ const block = root.querySelector(`[data-block="${CSS.escape(a.section)}"]`);
+ if (!block) {
+ lost.add(n.id);
+ continue;
+ }
+ const at = locateQuote(blockText(block).text, a);
+ if (!at.found) {
+ lost.add(n.id);
+ continue;
+ }
+ wrapRange(block, at.start, at.end, { "data-note": n.id, "data-status": n.status }, MARK_CLASS);
+ }
+ // Citation notes: mark the citation itself.
+ for (const el of Array.from(root.querySelectorAll("[data-cite][data-has-note]"))) el.removeAttribute("data-has-note");
+ for (const n of visible) {
+ if (n.anchor.kind === "cite") {
+ for (const el of Array.from(root.querySelectorAll(`[data-cite="${CSS.escape(n.anchor.cite)}"]`))) el.setAttribute("data-has-note", "true");
+ }
+ }
+ setOrphans((prev) => (prev.size === lost.size && [...lost].every((id) => prev.has(id)) ? prev : lost));
+ }, [visible]);
+
+ // The selected / hovered note's marks light up.
+ useEffect(() => {
+ const root = bodyRef.current;
+ if (!root) return;
+ for (const m of Array.from(root.querySelectorAll("mark[data-note]"))) {
+ const id = m.getAttribute("data-note");
+ m.setAttribute("data-active", String(id === selected || id === hover));
+ }
+ }, [selected, hover, visible]);
+
+ const focusNote = useCallback((id: string, scroll = true) => {
+ setSelected(id);
+ if (!scroll) return;
+ requestAnimationFrame(() => {
+ const mark = bodyRef.current?.querySelector(`mark[data-note="${CSS.escape(id)}"]`);
+ mark?.scrollIntoView({ block: "center", behavior: "smooth" });
+ document.querySelector(`[data-note-card="${CSS.escape(id)}"]`)?.scrollIntoView({ block: "nearest" });
+ });
+ }, []);
+
+ // A deep link (`?note=`) lands on its note, whatever the filter says.
+ useEffect(() => {
+ if (!initialNote) return;
+ const n = notes.find((x) => x.id === initialNote);
+ if (n && filter !== "all" && (filter === "open") !== (n.status === "open")) setFilter("all");
+ focusNote(initialNote);
+ // once, on mount
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, []);
+
+ // ---- selection → "Note" ---------------------------------------------------
+ const readSelection = useCallback(() => {
+ const root = bodyRef.current;
+ if (!root) return;
+ const sel = selectionIn(root);
+ if (!sel) {
+ setPending(null);
+ return;
+ }
+ const section = sel.block.getAttribute("data-block") ?? "";
+ const q = quoteAnchor(blockText(sel.block).text, sel.start, sel.end);
+ if (!q.quote) {
+ setPending(null);
+ return;
+ }
+ const anchor: Anchor = { kind: "text", section, ...q };
+ setPending({ anchor, label: anchorLabel(anchor, titles, numbers), x: sel.rect.right, y: sel.rect.top });
+ }, [titles, numbers]);
+
+ const openComposer = useCallback((anchor: Anchor) => {
+ setComposer({ anchor, label: anchorLabel(anchor, titles, numbers) });
+ setPending(null);
+ setDrawer(true);
+ window.getSelection()?.removeAllRanges();
+ }, [titles, numbers]);
+
+ // The body's buttons talk to the reader through a context whose value never
+ // changes (so the memoised body never re-renders): it reads the latest
+ // callbacks from a ref.
+ const latest = useRef({ setCite, openComposer, setDrawer });
+ latest.current = { setCite, openComposer, setDrawer };
+ const actions = useMemo<ArticleActions>(
+ () => ({
+ openCite: (id) => {
+ latest.current.setCite(id);
+ latest.current.setDrawer(true);
+ },
+ noteSection: (section) => latest.current.openComposer({ kind: "section", section }),
+ }),
+ [],
+ );
+
+ const add = async (anchor: Anchor, text: string) => {
+ const note = await write({ op: "add", text, anchor });
+ if (note) {
+ setComposer(null);
+ if (filter === "resolved") setFilter("open");
+ focusNote(note.id, false);
+ }
+ };
+
+ // ---- keys -----------------------------------------------------------------
+ useEffect(() => {
+ const onKey = (e: KeyboardEvent) => {
+ const t = e.target as HTMLElement | null;
+ if (t && (t.closest("input, textarea, select, [contenteditable=true]") || e.metaKey || e.ctrlKey || e.altKey)) return;
+ const idx = visible.findIndex((n) => n.id === selected);
+ if (e.key === "j" || e.key === "k") {
+ if (!visible.length) return;
+ e.preventDefault();
+ const next = e.key === "j" ? (idx < 0 ? 0 : Math.min(visible.length - 1, idx + 1)) : idx < 0 ? 0 : Math.max(0, idx - 1);
+ focusNote(visible[next].id);
+ } else if (e.key === "r" && idx >= 0) {
+ e.preventDefault();
+ void write({ op: "status", id: visible[idx].id, status: "resolved" });
+ } else if (e.key === "e" && idx >= 0 && visible[idx].author === "operator") {
+ e.preventDefault();
+ setEditing(visible[idx].id);
+ } else if (e.key === "n") {
+ e.preventDefault();
+ if (pending) openComposer(pending.anchor);
+ else if (cite) openComposer({ kind: "cite", cite });
+ else openComposer({ kind: "whole" });
+ } else if (e.key === "Escape") {
+ setComposer(null);
+ setEditing(null);
+ setPending(null);
+ setCite(null);
+ setDrawer(false);
+ }
+ };
+ window.addEventListener("keydown", onKey);
+ return () => window.removeEventListener("keydown", onKey);
+ }, [visible, selected, pending, cite, write, focusNote, openComposer]);
+
+ const counts = {
+ open: notes.filter((n) => n.status === "open").length,
+ resolved: notes.filter((n) => n.status !== "open").length,
+ all: notes.length,
+ };
+ const citeNotes = cite ? notes.filter((n) => n.anchor.kind === "cite" && n.anchor.cite === cite) : [];
+ const card = (n: Note) => (
+ <NoteCard
+ key={n.id}
+ note={n}
+ label={anchorLabel(n.anchor, titles, numbers)}
+ orphaned={orphans.has(n.id)}
+ selected={selected === n.id}
+ editing={editing === n.id}
+ busy={busy}
+ onSelect={() => focusNote(n.id)}
+ onHover={(on) => setHover(on ? n.id : null)}
+ onEdit={() => setEditing(n.id)}
+ onCancelEdit={() => setEditing(null)}
+ write={write}
+ />
+ );
+
+ return (
+ <div className="flex min-h-0 flex-1">
+ <main className="deck-main min-w-0 flex-1 p-4">
+ <div className="mx-auto max-w-[72ch] space-y-4">
+ {meta}
+ {viewError && <p className="text-[12px] text-[var(--color-bad)]">citations not shown: {viewError}</p>}
+ <ArticleActionsContext.Provider value={actions}>
+ <div
+ ref={bodyRef}
+ onMouseUp={() => setTimeout(readSelection, 0)}
+ onKeyUp={(e) => e.shiftKey && readSelection()}
+ onMouseOver={(e) => {
+ const m = (e.target as HTMLElement).closest?.("mark[data-note]");
+ setHover(m ? m.getAttribute("data-note") : null);
+ }}
+ onClick={(e) => {
+ const m = (e.target as HTMLElement).closest?.("mark[data-note]");
+ const id = m?.getAttribute("data-note");
+ if (id && window.getSelection()?.isCollapsed) {
+ setDrawer(true);
+ focusNote(id, false);
+ document.querySelector(`[data-note-card="${CSS.escape(id)}"]`)?.scrollIntoView({ block: "nearest" });
+ }
+ }}
+ >
+ <ShareNotes target={{ article: `${site}/${report}` }} notes={notesApi}>
+ <ArticleBody view={view} video={videoSlot} />
+ </ShareNotes>
+ </div>
+ </ArticleActionsContext.Provider>
+ </div>
+ </main>
+
+ {pending && (
+ <button
+ type="button"
+ onMouseDown={(e) => e.preventDefault()}
+ onClick={() => openComposer(pending.anchor)}
+ style={{ left: Math.min(pending.x + 6, window.innerWidth - 70), top: Math.min(window.innerHeight - 32, Math.max(8, pending.y - 30)) }}
+ className="fixed z-50 rounded bg-[var(--color-sel)] px-2 py-0.5 text-[12px] font-medium text-[var(--color-ink)] shadow"
+ >
+ Note
+ </button>
+ )}
+
+ <button
+ type="button"
+ onClick={() => setDrawer((d) => !d)}
+ className="fixed bottom-3 right-3 z-30 rounded border border-[var(--color-line)] bg-[var(--color-panel)] px-2 py-1 text-[12px] min-[1100px]:hidden"
+ >
+ notes {counts.open}
+ </button>
+
+ <aside
+ aria-label="notes"
+ data-drawer={drawer ? "open" : "closed"}
+ className={`fixed inset-y-0 right-0 z-40 flex w-[min(380px,92vw)] flex-col border-l border-[var(--color-line)] bg-[var(--color-ink)] transition-transform min-[1100px]:static min-[1100px]:z-auto min-[1100px]:w-[360px] min-[1100px]:shrink-0 min-[1100px]:translate-x-0 ${drawer ? "translate-x-0" : "translate-x-full"}`}
+ >
+ <div className="flex flex-wrap items-center gap-1 border-b border-[var(--color-line)] p-2">
+ {(["open", "resolved", "all"] as Filter[]).map((f) => (
+ <button
+ key={f}
+ type="button"
+ aria-pressed={filter === f}
+ onClick={() => setFilter(f)}
+ className={`rounded border px-1.5 py-0.5 font-mono text-[11px] ${filter === f ? "border-[var(--color-sel)] text-[var(--color-sel)]" : "border-[var(--color-line)] text-[var(--color-dim)]"}`}
+ >
+ {f} {counts[f]}
+ </button>
+ ))}
+ <button
+ type="button"
+ onClick={() => openComposer({ kind: "whole" })}
+ className="ml-auto text-[11px] text-[var(--color-dim)] hover:text-[var(--color-sel)]"
+ >
+ + whole article
+ </button>
+ <button type="button" aria-label="close notes" onClick={() => setDrawer(false)} className="text-[12px] text-[var(--color-dim)] min-[1100px]:hidden">
+ ✕
+ </button>
+ </div>
+ <div className="flex items-center gap-2 border-b border-[var(--color-line)] px-2 py-1">
+ <CopyButton
+ url={`/api/notes/context?${new URLSearchParams({ article: `${site}/${report}` })}`}
+ label="copy agent brief"
+ title={`umtool notes ${site}/${report}`}
+ className="shrink-0 whitespace-nowrap"
+ />
+ {notesApi.source?.draft && (
+ <span className="truncate font-mono text-[10px] text-[var(--color-dim)]" title={notesApi.source.how ?? ""}>
+ edit {notesApi.source.draft}
+ </span>
+ )}
+ </div>
+ <div className="min-h-0 flex-1 space-y-2 overflow-y-auto p-2">
+ {error && <p className="text-[12px] text-[var(--color-bad)]">{error}</p>}
+
+ {cite && (
+ <section data-evidence-rail className="space-y-2 rounded border border-[var(--color-line)] p-2">
+ <div className="flex items-center">
+ <span className="micro">citation [{numbers[cite] ?? cite}]</span>
+ <a href={`/sites/${site}/${report}/evidence?c=${encodeURIComponent(cite)}`} className="ml-auto text-[11px] text-[var(--color-dim)] hover:text-[var(--color-sel)]">
+ walk →
+ </a>
+ <button type="button" aria-label="close evidence" onClick={() => setCite(null)} className="ml-2 text-[12px] text-[var(--color-dim)] hover:text-[var(--color-text)]">
+ ✕
+ </button>
+ </div>
+ <EvidencePanel ev={evidence.ev} error={evidence.error} number={numbers[cite]} projects={videoProjects} />
+ {citeNotes.length > 0 && <ul className="space-y-2">{citeNotes.map(card)}</ul>}
+ {composer?.anchor.kind === "cite" && composer.anchor.cite === cite ? (
+ <Composer label={composer.label} busy={busy} onSave={(text) => add(composer.anchor, text)} onCancel={() => setComposer(null)} />
+ ) : (
+ <button type="button" onClick={() => openComposer({ kind: "cite", cite })} className="text-[11px] text-[var(--color-dim)] hover:text-[var(--color-sel)]">
+ + note on this citation
+ </button>
+ )}
+ </section>
+ )}
+
+ {composer && !(composer.anchor.kind === "cite" && composer.anchor.cite === cite) && (
+ <Composer label={composer.label} busy={busy} onSave={(text) => add(composer.anchor, text)} onCancel={() => setComposer(null)} />
+ )}
+
+ {visible.length === 0 ? (
+ <p className="text-[12px] text-[var(--color-dim)]">{notes.length === 0 ? "no notes" : `no ${filter} notes`}</p>
+ ) : (
+ <ul data-notes-list className="space-y-2">
+ {/* A note on the open citation is listed with its evidence above. */}
+ {visible.filter((n) => !(cite && n.anchor.kind === "cite" && n.anchor.cite === cite)).map(card)}
+ </ul>
+ )}
+ </div>
+ </aside>
+ </div>
+ );
+}
diff --git a/umtool/components/articles/ArticleTable.tsx b/umtool/components/articles/ArticleTable.tsx
@@ -0,0 +1,103 @@
+import Link from "next/link";
+import { badgeVariants } from "@/components/ui/badge";
+import { fmtAgo } from "@/lib/format";
+import type { ArticleRow } from "@/lib/articles/sites";
+
+// One row per article: what it is, where it stands, what is waiting on it,
+// and where it came from. Server-rendered, zero JS -- the /browse idiom.
+
+const posterUrl = (a: ArticleRow) =>
+ `/api/sites/media?site=${encodeURIComponent(a.site)}&report=${encodeURIComponent(a.id)}&file=poster.jpg`;
+
+const baseName = (p: string) => p.split("/").pop() ?? p;
+
+export function articleHref(a: { site: string; id: string }) {
+ return `/sites/${a.site}/${a.id}`;
+}
+
+export default function ArticleTable({ articles }: { articles: ArticleRow[] }) {
+ if (articles.length === 0) return <p className="text-[12px] text-[var(--color-dim)]">no articles</p>;
+ return (
+ <table className="w-full border-collapse text-[12px]" data-testid="article-table">
+ <thead>
+ <tr className="text-left text-[11px] text-[var(--color-dim)]">
+ <th className="w-[72px] py-1 font-normal" />
+ <th className="py-1 font-normal">article</th>
+ <th className="py-1 font-normal">status</th>
+ <th className="py-1 font-normal">updated</th>
+ <th className="py-1 text-right font-normal">cites</th>
+ <th className="py-1 text-right font-normal">notes</th>
+ <th className="py-1 pl-3 font-normal">video project</th>
+ <th className="py-1 font-normal">source</th>
+ </tr>
+ </thead>
+ <tbody>
+ {articles.map((a) => (
+ <tr
+ key={`${a.site}/${a.id}`}
+ data-article={`${a.site}/${a.id}`}
+ data-status={a.status}
+ className="border-t border-[var(--color-line)] align-top"
+ >
+ <td className="py-1.5 pr-2">
+ {a.hasPoster ? (
+ // eslint-disable-next-line @next/next/no-img-element
+ <img src={posterUrl(a)} alt="" loading="lazy" className="h-9 w-16 rounded object-cover" />
+ ) : (
+ <div className="h-9 w-16 rounded bg-[var(--color-panel-2)]" />
+ )}
+ </td>
+ <td className="py-1.5 pr-3">
+ <Link href={articleHref(a)} className="text-[var(--color-text)] hover:text-[var(--color-sel)]">
+ {a.title}
+ </Link>
+ <div className="font-mono text-[11px] text-[var(--color-dim)]">
+ {a.id}
+ {a.series ? ` · ${a.series}` : ""}
+ {a.problem ? <span className="text-[var(--color-bad)]"> · {a.problems} problem{a.problems === 1 ? "" : "s"}</span> : null}
+ </div>
+ </td>
+ <td className="py-1.5 pr-3">
+ <span className={badgeVariants({ variant: a.status === "published" ? "on" : "neutral", size: "sm" })}>
+ {a.status}
+ </span>
+ </td>
+ <td className="num py-1.5 pr-3 text-[var(--color-dim)]">
+ {a.updated ?? (a.mtimeMs ? fmtAgo(a.mtimeMs) : "—")}
+ </td>
+ <td className="num py-1.5 pr-3 text-right">{a.citations}</td>
+ <td className="num py-1.5 text-right">
+ {a.openNotes > 0 ? (
+ <Link
+ href={`${articleHref(a)}?status=open`}
+ data-open-notes={a.openNotes}
+ className={badgeVariants({ variant: "open", size: "sm" })}
+ >
+ {a.openNotes} open
+ </Link>
+ ) : (
+ <span className="text-[var(--color-dim)]">{a.notes || "—"}</span>
+ )}
+ </td>
+ <td className="py-1.5 pl-3 pr-3">
+ {a.projects.linked.map((p) => (
+ <Link key={p.id} href={`/browse/${p.id}`} data-project-link={p.id} className="block font-mono text-[11px] text-[var(--color-sel)] hover:underline">
+ {p.id}
+ </Link>
+ ))}
+ {a.projects.possible.map((p) => (
+ <Link key={p.id} href={`/browse/${p.id}`} title="shares the slug; not linked" className="block font-mono text-[11px] text-[var(--color-dim)] hover:underline">
+ {p.id} (possible)
+ </Link>
+ ))}
+ {a.projects.linked.length + a.projects.possible.length === 0 && <span className="text-[var(--color-dim)]">—</span>}
+ </td>
+ <td className="py-1.5 font-mono text-[11px] text-[var(--color-dim)]" title={a.source?.how ?? ""}>
+ {a.source?.draft ? baseName(a.source.draft) : a.source?.generator ? baseName(a.source.generator) : "—"}
+ </td>
+ </tr>
+ ))}
+ </tbody>
+ </table>
+ );
+}
diff --git a/umtool/components/articles/ArticleVideo.tsx b/umtool/components/articles/ArticleVideo.tsx
@@ -0,0 +1,34 @@
+"use client";
+
+import { TimedVideo } from "@/components/notes/TimedNotes";
+
+// THE ARTICLE'S OWN VIDEO -- the report's video (report.json `video.src`), as
+// the published page plays it, with timed notes under it: `n` or Mark at the
+// playhead writes a moment anchor `{ kind: "moment", file: <video.src>, t }`
+// to the ARTICLE's notes. The handle is the reader's own, shared through
+// <ShareNotes> (components/notes/NotesProvider.tsx), so a mark and a text note
+// never race each other's token. No schedule: a report video is a published
+// file, so a mark keeps its time only. Keep this the only place the article
+// page renders its video.
+export default function ArticleVideo({
+ site,
+ report,
+ file,
+ src,
+ poster,
+ caption,
+}: {
+ site: string;
+ report: string;
+ file: string;
+ src: string;
+ poster?: string;
+ caption?: string;
+}) {
+ return (
+ <figure data-article-video className="space-y-1">
+ <TimedVideo target={{ article: `${site}/${report}` }} file={file} src={src} poster={poster} testId="article-video" showErrors={false} />
+ {caption && <figcaption className="text-[12px] text-[var(--color-dim)]">{caption}</figcaption>}
+ </figure>
+ );
+}
diff --git a/umtool/components/articles/EvidencePanel.tsx b/umtool/components/articles/EvidencePanel.tsx
@@ -0,0 +1,167 @@
+"use client";
+
+import { forwardRef, useEffect, useImperativeHandle, useRef, useState } from "react";
+import CopyButton from "@/components/CopyButton";
+import type { Evidence } from "@/lib/articles/evidence";
+import AddToVideo, { type VideoProjectLink } from "./AddToVideo";
+
+// One citation's evidence: what it quotes, who and when, the transcript around
+// it with the cited cues marked, and the media that plays it -- the prepared
+// clip, a fetched window or a saved file, seeked to the cited second -- or,
+// when there is none on disk, the MCP line that fetches it through the editor.
+
+export type EvidenceHandle = { togglePlay: () => void };
+
+const fmt = (s: number) => {
+ const m = Math.floor(s / 60);
+ const r = Math.floor(s % 60);
+ return `${m}:${String(r).padStart(2, "0")}`;
+};
+
+export function useEvidence(site: string, report: string, cite: string | null) {
+ const [ev, setEv] = useState<Evidence | null>(null);
+ const [error, setError] = useState<string | null>(null);
+ useEffect(() => {
+ if (!cite) return;
+ let live = true;
+ setEv(null);
+ setError(null);
+ fetch(`/api/sites/evidence?${new URLSearchParams({ site, report, cite })}`, { cache: "no-store" })
+ .then(async (r) => {
+ const j = await r.json();
+ if (!r.ok) throw new Error(j.error ?? r.statusText);
+ if (live) setEv(j);
+ })
+ .catch((err) => live && setError(err instanceof Error ? err.message : String(err)));
+ return () => {
+ live = false;
+ };
+ }, [site, report, cite]);
+ return { ev, error };
+}
+
+const EvidencePanel = forwardRef<
+ EvidenceHandle,
+ { ev: Evidence | null; error: string | null; number?: number; autoPlay?: boolean; projects?: VideoProjectLink[] }
+>(
+ function EvidencePanel({ ev, error, number, autoPlay = false, projects = [] }, ref) {
+ const media = useRef<HTMLMediaElement | null>(null);
+ useImperativeHandle(ref, () => ({
+ togglePlay: () => {
+ const m = media.current;
+ if (!m) return;
+ if (m.paused) void m.play().catch(() => {});
+ else m.pause();
+ },
+ }));
+
+ if (error) return <p className="text-[12px] text-[var(--color-bad)]">{error}</p>;
+ if (!ev) return <p className="text-[12px] text-[var(--color-dim)]">loading…</p>;
+
+ const play = ev.play;
+ // File time of a record second: the cited span starts `offset` into the file.
+ const fileT = (t: number) => (play && ev.start !== undefined ? Math.max(0, t - ev.start + play.offset) : 0);
+ const startAt = play ? (play.kind === "prepared" ? 0 : Math.max(0, play.offset - 3)) : 0;
+ const seek = (t: number) => {
+ if (media.current) {
+ media.current.currentTime = fileT(t);
+ void media.current.play().catch(() => {});
+ }
+ };
+
+ return (
+ <div data-evidence={ev.cite} className="space-y-2 text-[12px]">
+ <blockquote className="border-l-2 border-[var(--color-sel)] pl-2 text-[13px] text-[var(--color-text)]">
+ {number !== undefined && <span className="mr-1 font-mono text-[11px] text-[var(--color-sel)]">[{number}]</span>}“{ev.quote}”
+ </blockquote>
+ <div className="text-[var(--color-dim)]">
+ {[ev.speaker ?? ev.record?.channelTitle, ev.date, ev.label].filter(Boolean).join(" · ")}
+ {ev.start !== undefined && ev.end !== undefined && (
+ <span className="num ml-1">
+ @ {fmt(ev.start)}–{fmt(ev.end)}
+ </span>
+ )}
+ </div>
+ {ev.record?.title && <div className="text-[var(--color-dim)]">{ev.record.title}</div>}
+ {ev.originalUrl && (
+ <a href={ev.originalUrl} target="_blank" rel="noopener noreferrer" className="text-[var(--color-sel)] hover:underline">
+ original ↗
+ </a>
+ )}
+
+ {play ? (
+ <div data-play={play.kind}>
+ {play.audio ? (
+ <audio
+ ref={(el) => {
+ media.current = el;
+ }}
+ controls
+ src={play.url}
+ autoPlay={autoPlay}
+ onLoadedMetadata={(e) => {
+ e.currentTarget.currentTime = startAt;
+ }}
+ className="w-full"
+ />
+ ) : (
+ <video
+ ref={(el) => {
+ media.current = el;
+ }}
+ controls
+ src={play.url}
+ autoPlay={autoPlay}
+ onLoadedMetadata={(e) => {
+ e.currentTarget.currentTime = startAt;
+ }}
+ className="aspect-video w-full rounded bg-black"
+ />
+ )}
+ <div className="micro mt-0.5">{play.label}</div>
+ </div>
+ ) : ev.fetchLine ? (
+ <div data-play="none" className="space-y-1 rounded border border-[var(--color-line)] bg-[var(--color-panel-2)] p-2">
+ <div className="text-[var(--color-dim)]">not on disk — fetch it through the editor:</div>
+ <code className="block break-all font-mono text-[11px] text-[var(--color-text)]">{ev.fetchLine}</code>
+ <CopyButton text={ev.fetchLine} label="copy fetch_clip" />
+ </div>
+ ) : null}
+
+ {ev.post && (
+ <div data-post className="space-y-1 rounded border border-[var(--color-line)] p-2">
+ {ev.post.author && <div className="text-[var(--color-dim)]">{ev.post.author}</div>}
+ {ev.post.text && <p className="whitespace-pre-wrap text-[var(--color-text)]">{ev.post.text}</p>}
+ {ev.post.shot && (
+ // eslint-disable-next-line @next/next/no-img-element
+ <img src={ev.post.shot} alt="capture of the post" className="w-full rounded border border-[var(--color-line)]" />
+ )}
+ </div>
+ )}
+
+ {ev.cues.length > 0 ? (
+ <ol data-cues className="space-y-0.5">
+ {ev.cues.map((c) => (
+ <li key={c.start} data-cited={c.cited ? "true" : undefined}>
+ <button
+ type="button"
+ onClick={() => seek(c.start)}
+ disabled={!play}
+ className={`flex w-full gap-2 rounded px-1 text-left ${c.cited ? "bg-[var(--color-panel-2)] text-[var(--color-text)]" : "text-[var(--color-dim)]"} enabled:hover:text-[var(--color-text)]`}
+ >
+ <span className="num shrink-0 font-mono text-[10px] leading-5">{fmt(c.start)}</span>
+ <span>{c.text}</span>
+ </button>
+ </li>
+ ))}
+ </ol>
+ ) : ev.cuesNote ? (
+ <div className="micro">{ev.cuesNote}</div>
+ ) : null}
+ <AddToVideo ev={ev} projects={projects} />
+ </div>
+ );
+ },
+);
+
+export default EvidencePanel;
diff --git a/umtool/components/articles/EvidenceWalk.tsx b/umtool/components/articles/EvidenceWalk.tsx
@@ -0,0 +1,129 @@
+"use client";
+
+import type { VideoProjectLink } from "./AddToVideo";
+import Link from "next/link";
+import { useCallback, useEffect, useRef, useState } from "react";
+import type { NotesRead } from "@/lib/annotations/types";
+import { useNotes } from "@/lib/annotations/useNotes";
+import EvidencePanel, { useEvidence, type EvidenceHandle } from "./EvidencePanel";
+import { Composer, NoteCard } from "./NoteCards";
+
+// Every citation of one article, one per screen: the evidence on the left, its
+// notes on the right. j/k (or ←/→) move, space plays, n notes it, Esc cancels.
+// The citation in view is in the URL (`?c=`), so a walk can be resumed.
+
+export type WalkItem = { id: string; number: number; kind: string; quote: string };
+
+export default function EvidenceWalk({
+ site,
+ report,
+ items,
+ initial,
+ initialNotes,
+ videoProjects = [],
+}: {
+ site: string;
+ report: string;
+ items: WalkItem[];
+ initial: number;
+ initialNotes: NotesRead;
+ videoProjects?: VideoProjectLink[];
+}) {
+ const [idx, setIdx] = useState(initial);
+ const [composing, setComposing] = useState(false);
+ const item = items[idx];
+ const { ev, error } = useEvidence(site, report, item?.id ?? null);
+ const { notes, write, busy, error: notesError } = useNotes({ article: `${site}/${report}` }, { initial: initialNotes });
+ const panel = useRef<EvidenceHandle>(null);
+
+ const go = useCallback(
+ (to: number) => {
+ const next = Math.max(0, Math.min(items.length - 1, to));
+ setIdx(next);
+ setComposing(false);
+ const url = new URL(window.location.href);
+ url.searchParams.set("c", items[next].id);
+ window.history.replaceState(null, "", url);
+ },
+ [items],
+ );
+
+ useEffect(() => {
+ const onKey = (e: KeyboardEvent) => {
+ const t = e.target as HTMLElement | null;
+ if (t?.closest("input, textarea, select, [contenteditable=true]") || e.metaKey || e.ctrlKey || e.altKey) return;
+ if (e.key === "j" || e.key === "ArrowRight") {
+ e.preventDefault();
+ go(idx + 1);
+ } else if (e.key === "k" || e.key === "ArrowLeft") {
+ e.preventDefault();
+ go(idx - 1);
+ } else if (e.key === " ") {
+ e.preventDefault();
+ panel.current?.togglePlay();
+ } else if (e.key === "n") {
+ e.preventDefault();
+ setComposing(true);
+ } else if (e.key === "Escape") {
+ setComposing(false);
+ }
+ };
+ window.addEventListener("keydown", onKey);
+ return () => window.removeEventListener("keydown", onKey);
+ }, [idx, go]);
+
+ if (!item) return <p className="p-4 text-[12px] text-[var(--color-dim)]">no citations</p>;
+ const mine = notes.filter((n) => n.anchor.kind === "cite" && n.anchor.cite === item.id);
+
+ return (
+ <div data-walk={item.id} className="grid min-h-0 flex-1 grid-cols-1 gap-4 overflow-auto p-4 min-[1100px]:grid-cols-[minmax(0,1.3fr)_minmax(320px,1fr)]">
+ <div className="min-w-0 space-y-2">
+ <div className="flex items-center gap-2 text-[12px]">
+ <button type="button" onClick={() => go(idx - 1)} disabled={idx === 0} className="rounded border border-[var(--color-line)] px-2 disabled:opacity-30">
+ ← k
+ </button>
+ <span className="num" data-walk-position>
+ {idx + 1} / {items.length}
+ </span>
+ <button type="button" onClick={() => go(idx + 1)} disabled={idx === items.length - 1} className="rounded border border-[var(--color-line)] px-2 disabled:opacity-30">
+ j →
+ </button>
+ <span className="micro ml-2">{item.kind}</span>
+ <span className="micro ml-auto">space plays · n notes</span>
+ </div>
+ <EvidencePanel key={item.id} ref={panel} ev={ev} error={error} number={item.number} projects={videoProjects} />
+ </div>
+ <div className="min-w-0 space-y-2">
+ <div className="micro">notes on [{item.number}]</div>
+ {notesError && <p className="text-[12px] text-[var(--color-bad)]">{notesError}</p>}
+ {mine.length > 0 && (
+ <ul className="space-y-2">
+ {mine.map((n) => (
+ <NoteCard key={n.id} note={n} label={`citation [${item.number}]`} busy={busy} write={write} />
+ ))}
+ </ul>
+ )}
+ {composing ? (
+ <Composer
+ label={`citation [${item.number}]`}
+ busy={busy}
+ onSave={async (text) => {
+ const note = await write({ op: "add", text, anchor: { kind: "cite", cite: item.id } });
+ if (note) setComposing(false);
+ }}
+ onCancel={() => setComposing(false)}
+ />
+ ) : (
+ <button type="button" onClick={() => setComposing(true)} className="text-[12px] text-[var(--color-dim)] hover:text-[var(--color-sel)]">
+ + note on this citation
+ </button>
+ )}
+ <div className="pt-2">
+ <Link href={`/sites/${site}/${report}`} className="text-[12px] text-[var(--color-dim)] hover:text-[var(--color-text)]">
+ ← article
+ </Link>
+ </div>
+ </div>
+ </div>
+ );
+}
diff --git a/umtool/components/articles/NoteCards.tsx b/umtool/components/articles/NoteCards.tsx
@@ -0,0 +1,240 @@
+"use client";
+
+import { useEffect, useRef, useState } from "react";
+import { NOTE_TEXT_LIMIT } from "@/lib/annotations/types";
+import type { Anchor, Note, NoteOp } from "@/lib/annotations/types";
+import { fmtAgo } from "@/lib/format";
+
+// A note in the rail, and the box that writes one. Shared by the article
+// reader and the evidence walk.
+
+export function anchorLabel(a: Anchor, titles: Record<string, string>, cites: Record<string, number | undefined>): string {
+ const sec = (id: string) => titles[id] ?? id;
+ switch (a.kind) {
+ case "text":
+ return `${sec(a.section)} · “${a.quote.length > 60 ? `${a.quote.slice(0, 59)}…` : a.quote}”`;
+ case "section":
+ return `section · ${sec(a.section)}`;
+ case "cite":
+ return `citation [${cites[a.cite] ?? a.cite}]`;
+ case "whole":
+ return "whole article";
+ case "moment":
+ return `${a.file} @ ${a.t.toFixed(1)}s`;
+ default:
+ return a.kind;
+ }
+}
+
+export function Composer({
+ label,
+ initial = "",
+ busy,
+ onSave,
+ onCancel,
+ saveLabel = "save note",
+}: {
+ label: string;
+ initial?: string;
+ busy?: boolean;
+ onSave: (text: string) => void | Promise<void>;
+ onCancel: () => void;
+ saveLabel?: string;
+}) {
+ const [text, setText] = useState(initial);
+ const ref = useRef<HTMLTextAreaElement>(null);
+ useEffect(() => ref.current?.focus(), []);
+ const save = () => {
+ if (text.trim()) void onSave(text);
+ };
+ return (
+ <div data-composer className="space-y-1 rounded border border-[var(--color-sel)] bg-[var(--color-panel)] p-2">
+ <div className="micro truncate" title={label}>
+ {label}
+ </div>
+ <textarea
+ ref={ref}
+ aria-label="note text"
+ value={text}
+ maxLength={NOTE_TEXT_LIMIT}
+ onChange={(e) => setText(e.target.value)}
+ onKeyDown={(e) => {
+ if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) {
+ e.preventDefault();
+ save();
+ } else if (e.key === "Escape") {
+ e.preventDefault();
+ onCancel();
+ }
+ }}
+ rows={4}
+ className="w-full resize-y rounded border border-[var(--color-line)] bg-[var(--color-ink)] p-1.5 text-[12px] text-[var(--color-text)] outline-none focus:border-[var(--color-sel)]"
+ />
+ <div className="flex items-center gap-2">
+ <button
+ type="button"
+ onClick={save}
+ disabled={busy || !text.trim()}
+ className="rounded bg-[var(--color-sel)] px-2 py-0.5 text-[12px] text-[var(--color-ink)] disabled:opacity-40"
+ >
+ {saveLabel}
+ </button>
+ <button type="button" onClick={onCancel} className="text-[12px] text-[var(--color-dim)] hover:text-[var(--color-text)]">
+ cancel
+ </button>
+ <span className="micro ml-auto">ctrl+enter</span>
+ </div>
+ </div>
+ );
+}
+
+const STATUS_TONE: Record<Note["status"], string> = {
+ open: "text-[var(--color-dirty)] border-[var(--color-dirty)]",
+ resolved: "text-[var(--color-good)] border-[var(--color-good)]",
+ wontfix: "text-[var(--color-dim)] border-[var(--color-line)]",
+};
+
+export function NoteCard({
+ note,
+ label,
+ orphaned,
+ selected,
+ editing,
+ busy,
+ onSelect,
+ onHover,
+ onEdit,
+ onCancelEdit,
+ write,
+}: {
+ note: Note;
+ label: string;
+ orphaned?: boolean;
+ selected?: boolean;
+ editing?: boolean;
+ busy?: boolean;
+ onSelect?: () => void;
+ onHover?: (on: boolean) => void;
+ onEdit?: () => void;
+ onCancelEdit?: () => void;
+ write: (op: NoteOp) => Promise<unknown>;
+}) {
+ const [replying, setReplying] = useState(false);
+ const mine = note.author === "operator";
+ return (
+ <li
+ data-note-card={note.id}
+ data-status={note.status}
+ data-orphaned={orphaned ? "true" : undefined}
+ aria-current={selected ? "true" : undefined}
+ onMouseEnter={() => onHover?.(true)}
+ onMouseLeave={() => onHover?.(false)}
+ onClick={onSelect}
+ className={`cursor-default space-y-1 rounded border bg-[var(--color-panel)] p-2 text-[12px] ${selected ? "border-[var(--color-sel)]" : "border-[var(--color-line)]"}`}
+ >
+ <div className="flex items-center gap-1.5">
+ <span className={`rounded border px-1 font-mono text-[10px] uppercase ${STATUS_TONE[note.status]}`}>{note.status}</span>
+ {note.author === "agent" && <span className="rounded border border-[var(--color-meter)] px-1 font-mono text-[10px] text-[var(--color-meter)]">agent</span>}
+ {orphaned && (
+ <span title="the quoted text is no longer in the article" className="rounded border border-[var(--color-bad)] px-1 font-mono text-[10px] text-[var(--color-bad)]">
+ orphaned
+ </span>
+ )}
+ <span className="num ml-auto text-[10px] text-[var(--color-dim)]" title={note.updatedAt}>
+ {fmtAgo(Date.parse(note.updatedAt))}
+ </span>
+ </div>
+ <div className="truncate text-[11px] text-[var(--color-dim)]" title={label}>
+ {label}
+ </div>
+ {editing ? (
+ <Composer
+ label="edit note"
+ initial={note.text}
+ busy={busy}
+ saveLabel="save"
+ onSave={async (text) => {
+ await write({ op: "edit", id: note.id, text });
+ onCancelEdit?.();
+ }}
+ onCancel={() => onCancelEdit?.()}
+ />
+ ) : (
+ <p data-note-text className="whitespace-pre-wrap text-[var(--color-text)]">
+ {note.text}
+ </p>
+ )}
+ {note.replies.length > 0 && (
+ <ul className="space-y-1 border-l border-[var(--color-line)] pl-2">
+ {note.replies.map((r, i) => (
+ <li key={`${r.at}-${i}`} data-reply={r.author} className="text-[12px]">
+ <span className={`mr-1 font-mono text-[10px] ${r.author === "agent" ? "text-[var(--color-meter)]" : "text-[var(--color-dim)]"}`}>{r.author}</span>
+ <span className="whitespace-pre-wrap text-[var(--color-text)]">{r.text}</span>
+ </li>
+ ))}
+ </ul>
+ )}
+ {replying ? (
+ <Composer
+ label="reply"
+ busy={busy}
+ saveLabel="reply"
+ onSave={async (text) => {
+ await write({ op: "reply", id: note.id, text });
+ setReplying(false);
+ }}
+ onCancel={() => setReplying(false)}
+ />
+ ) : (
+ <div className="flex flex-wrap gap-2 pt-0.5 text-[11px]">
+ {note.status === "open" ? (
+ <>
+ <Act onClick={() => write({ op: "status", id: note.id, status: "resolved" })} disabled={busy}>
+ resolve
+ </Act>
+ <Act onClick={() => write({ op: "status", id: note.id, status: "wontfix" })} disabled={busy}>
+ won’t fix
+ </Act>
+ </>
+ ) : (
+ <Act onClick={() => write({ op: "status", id: note.id, status: "open" })} disabled={busy}>
+ reopen
+ </Act>
+ )}
+ <Act onClick={() => setReplying(true)} disabled={busy}>
+ reply
+ </Act>
+ {mine && onEdit && (
+ <Act onClick={onEdit} disabled={busy}>
+ edit
+ </Act>
+ )}
+ <Act
+ onClick={() => {
+ if (window.confirm("Delete this note?")) void write({ op: "delete", id: note.id });
+ }}
+ disabled={busy}
+ >
+ delete
+ </Act>
+ </div>
+ )}
+ </li>
+ );
+}
+
+function Act({ onClick, disabled, children }: { onClick: () => void; disabled?: boolean; children: React.ReactNode }) {
+ return (
+ <button
+ type="button"
+ onClick={(e) => {
+ e.stopPropagation();
+ onClick();
+ }}
+ disabled={disabled}
+ className="text-[var(--color-dim)] hover:text-[var(--color-sel)] disabled:opacity-40"
+ >
+ {children}
+ </button>
+ );
+}
diff --git a/umtool/components/articles/SiteChips.tsx b/umtool/components/articles/SiteChips.tsx
@@ -0,0 +1,14 @@
+import { badgeVariants } from "@/components/ui/badge";
+
+// A site's audience, listing and search, as three short chips.
+export default function SiteChips({ site }: { site: { private: boolean; listed: boolean; search: boolean } }) {
+ return (
+ <span className="inline-flex gap-1">
+ <span data-audience={site.private ? "private" : "public"} className={badgeVariants({ variant: site.private ? "meter" : "neutral", size: "sm" })}>
+ {site.private ? "private" : "public"}
+ </span>
+ <span className={badgeVariants({ variant: "info", size: "sm" })}>{site.listed ? "listed" : "unlisted"}</span>
+ <span className={badgeVariants({ variant: "info", size: "sm" })}>{site.search ? "search" : "cited only"}</span>
+ </span>
+ );
+}
diff --git a/umtool/components/articles/WorkspacePanel.tsx b/umtool/components/articles/WorkspacePanel.tsx
@@ -0,0 +1,110 @@
+import Link from "next/link";
+import { Markdown } from "@/lib/markdown";
+import { fmtAgo, fmtBytes } from "@/lib/format";
+import type { OpenedFile, WorkspaceListing } from "@/lib/articles/files";
+
+// The files an article was written from, and the one that is open. Zero JS:
+// each file is a link that sets `?ws=&rel=` on the page it sits on; markdown
+// renders, a draft's JSON is pretty-printed with each top-level key folded,
+// and HTML opens in a sandboxed iframe (no scripts, no same origin).
+
+export default function WorkspacePanel({
+ workspaces,
+ opened,
+ hrefFor,
+ highlight,
+}: {
+ workspaces: WorkspaceListing[];
+ opened: OpenedFile | null;
+ hrefFor: (ws: string, rel: string) => string;
+ /** A file to mark (the article's own draft), as `<ws>/<rel>`. */
+ highlight?: string | null;
+}) {
+ if (workspaces.length === 0) return <p className="text-[12px] text-[var(--color-dim)]">no workspace found</p>;
+ return (
+ <div className="grid gap-4 min-[1100px]:grid-cols-[minmax(240px,320px)_1fr]" data-testid="workspace-panel">
+ <div className="space-y-3">
+ {workspaces.map((w) => (
+ <div key={w.name} data-workspace={w.name}>
+ <div className="mb-1 font-mono text-[11px] text-[var(--color-dim)]">{w.dir}</div>
+ <ul className="space-y-0.5">
+ {w.files.map((f) => {
+ const on = opened?.ws === w.name && opened.rel === f.rel;
+ const mine = highlight === `${w.name}/${f.rel}`;
+ return (
+ <li key={f.rel} className="flex items-baseline gap-2 text-[12px]">
+ <Link
+ href={hrefFor(w.name, f.rel)}
+ aria-current={on ? "true" : undefined}
+ data-file={f.rel}
+ className={`truncate font-mono ${on ? "text-[var(--color-sel)]" : mine ? "text-[var(--color-text)]" : "text-[var(--color-dim)] hover:text-[var(--color-text)]"}`}
+ >
+ {f.rel}
+ </Link>
+ {mine && <span className="micro">this article</span>}
+ <span className="num ml-auto shrink-0 text-[10px] text-[var(--color-dim)]">
+ {fmtBytes(f.bytes)} · {fmtAgo(f.mtimeMs)}
+ </span>
+ </li>
+ );
+ })}
+ </ul>
+ </div>
+ ))}
+ </div>
+ <div className="min-w-0">{opened ? <Opened file={opened} /> : <p className="text-[12px] text-[var(--color-dim)]">pick a file</p>}</div>
+ </div>
+ );
+}
+
+function Opened({ file }: { file: OpenedFile }) {
+ const head = <div className="mb-2 font-mono text-[11px] text-[var(--color-dim)]">{file.rel}</div>;
+ if (file.kind === "error") {
+ return (
+ <div>
+ {head}
+ <p className="text-[12px] text-[var(--color-bad)]">{file.message}</p>
+ </div>
+ );
+ }
+ if (file.kind === "html") {
+ return (
+ <div>
+ {head}
+ <iframe title={file.rel} src={file.url} sandbox="" className="h-[70vh] w-full rounded border border-[var(--color-line)] bg-white" />
+ </div>
+ );
+ }
+ if (file.kind === "json") {
+ const v = file.value;
+ const entries = v && typeof v === "object" && !Array.isArray(v) ? Object.entries(v as Record<string, unknown>) : null;
+ return (
+ <div data-opened="json">
+ {head}
+ {entries ? (
+ <div className="space-y-1">
+ {entries.map(([k, val]) => (
+ <details key={k} open={typeof val !== "object" || val === null} className="rounded border border-[var(--color-line)] bg-[var(--color-panel)]">
+ <summary className="cursor-pointer px-2 py-1 font-mono text-[12px] text-[var(--color-text)]">
+ {k}
+ {Array.isArray(val) ? <span className="micro ml-2">{val.length} items</span> : null}
+ </summary>
+ <pre className="max-h-[50vh] overflow-auto whitespace-pre-wrap px-2 pb-2 font-mono text-[11px] text-[var(--color-dim)]">
+ {JSON.stringify(val, null, 2)}
+ </pre>
+ </details>
+ ))}
+ </div>
+ ) : (
+ <pre className="overflow-auto whitespace-pre-wrap font-mono text-[11px]">{JSON.stringify(v, null, 2)}</pre>
+ )}
+ </div>
+ );
+ }
+ return (
+ <div data-opened="md">
+ {head}
+ <Markdown text={file.text} />
+ </div>
+ );
+}
diff --git a/umtool/components/articles/anchorDom.ts b/umtool/components/articles/anchorDom.ts
@@ -0,0 +1,98 @@
+// The article's TEXT, as the notes see it, and the <mark>s that show them.
+//
+// A block (`[data-block]`: title, subtitle, summary, method, or a section) has
+// one plain text: its text nodes in document order, MINUS anything inside
+// `[data-anchor-skip]` -- a citation's superscript number, a "+ note" button --
+// so a quote never carries a "3" from a cite marker and re-anchors the same
+// way `umtool notes` reads the section from report.json. lib/annotations/
+// anchor.mjs locates a quote in that text; these turn offsets into DOM and back.
+//
+// The marks are DOM the app adds AFTER React has rendered the (memoised,
+// never re-rendered) article body, and removes before adding them again.
+
+export const SKIP = "[data-anchor-skip]";
+
+export type TextModel = { text: string; nodes: { node: Text; start: number }[] };
+
+export function blockText(root: Element): TextModel {
+ const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT, {
+ acceptNode: (n) => {
+ const skip = n.parentElement?.closest(SKIP);
+ return skip && root.contains(skip) ? NodeFilter.FILTER_REJECT : NodeFilter.FILTER_ACCEPT;
+ },
+ });
+ const nodes: TextModel["nodes"] = [];
+ let text = "";
+ for (let n = walker.nextNode(); n; n = walker.nextNode()) {
+ const t = n as Text;
+ nodes.push({ node: t, start: text.length });
+ text += t.data;
+ }
+ return { text, nodes };
+}
+
+/** The text offset of a DOM point inside `root`. */
+export function offsetOf(root: Element, container: Node, offset: number): number {
+ const { nodes, text } = blockText(root);
+ const point = document.createRange();
+ point.setStart(container, offset);
+ for (const { node, start } of nodes) {
+ if (node === container) return start + Math.min(offset, node.data.length);
+ const r = document.createRange();
+ r.selectNodeContents(node);
+ if (r.compareBoundaryPoints(Range.START_TO_START, point) >= 0) return start;
+ }
+ return text.length;
+}
+
+export function unwrapMarks(root: Element) {
+ const parents = new Set<Node>();
+ for (const m of Array.from(root.querySelectorAll("mark[data-note]"))) {
+ const p = m.parentNode;
+ if (!p) continue;
+ while (m.firstChild) p.insertBefore(m.firstChild, m);
+ p.removeChild(m);
+ parents.add(p);
+ }
+ for (const p of parents) p.normalize();
+}
+
+/** Wrap [start, end) of a block's text in marks carrying `attrs`. Returns the marks. */
+export function wrapRange(root: Element, start: number, end: number, attrs: Record<string, string>, className: string): HTMLElement[] {
+ const { nodes } = blockText(root);
+ const out: HTMLElement[] = [];
+ for (const { node, start: ns } of nodes) {
+ const ne = ns + node.data.length;
+ if (ne <= start || ns >= end) continue;
+ const a = Math.max(start, ns) - ns;
+ const b = Math.min(end, ne) - ns;
+ if (b <= a) continue;
+ let t: Text = node;
+ if (a > 0) t = t.splitText(a);
+ if (b - a < t.data.length) t.splitText(b - a);
+ // Whitespace between two block elements is not worth a mark.
+ if (!t.data.trim()) continue;
+ const mark = document.createElement("mark");
+ for (const [k, v] of Object.entries(attrs)) mark.setAttribute(k, v);
+ mark.className = className;
+ t.parentNode!.insertBefore(mark, t);
+ mark.appendChild(t);
+ out.push(mark);
+ }
+ return out;
+}
+
+/** The block a selection lies in, and its offsets, or null (collapsed, or across blocks). */
+export function selectionIn(container: Element): { block: Element; start: number; end: number; rect: DOMRect } | null {
+ const sel = window.getSelection();
+ if (!sel || sel.rangeCount === 0 || sel.isCollapsed) return null;
+ const range = sel.getRangeAt(0);
+ const el = (n: Node) => (n.nodeType === Node.ELEMENT_NODE ? (n as Element) : n.parentElement);
+ const a = el(range.startContainer)?.closest("[data-block]");
+ const b = el(range.endContainer)?.closest("[data-block]");
+ if (!a || a !== b || !container.contains(a)) return null;
+ const start = offsetOf(a, range.startContainer, range.startOffset);
+ const end = offsetOf(a, range.endContainer, range.endOffset);
+ if (end <= start) return null;
+ return { block: a, start, end, rect: range.getBoundingClientRect() };
+}
diff --git a/umtool/components/notes/AnchoredNotes.tsx b/umtool/components/notes/AnchoredNotes.tsx
@@ -0,0 +1,74 @@
+"use client";
+
+import { useState } from "react";
+import { badgeVariants } from "@/components/ui/badge";
+import type { Anchor, Note } from "@/lib/annotations/types";
+import type { UseNotes } from "@/lib/annotations/useNotes";
+import { NoteComposer, NoteThread } from "./NoteThread";
+
+// The notes on one THING: a timeline row (`entry`), a take (`take`). A count
+// of the open ones, folded; unfolded, every note on it and a box for another.
+
+type Pin = { kind: "entry"; entry: string } | { kind: "take"; take: string };
+
+export const notesOn = (notes: Note[], pin: Pin) =>
+ notes.filter((n) => {
+ const a = n.anchor as Anchor;
+ if (pin.kind === "entry") {
+ return (a.kind === "entry" && a.entry === pin.entry) || (a.kind === "edit" && a.entry === pin.entry);
+ }
+ return (a.kind === "take" && a.take === pin.take) || (a.kind === "moment" && a.take === pin.take);
+ });
+
+export default function AnchoredNotes({
+ notes,
+ pin,
+ startOpen = false,
+ label = "notes",
+}: {
+ notes: UseNotes;
+ pin: Pin;
+ startOpen?: boolean;
+ label?: string;
+}) {
+ const [open, setOpen] = useState(startOpen);
+ const mine = notesOn(notes.notes, pin).filter((n) => n.anchor.kind !== "moment");
+ const openCount = mine.filter((n) => n.status === "open").length;
+ const key = pin.kind === "entry" ? pin.entry : pin.take;
+ return (
+ <div data-anchored-notes={key} data-open-notes={openCount} className="text-[11px]">
+ <button
+ type="button"
+ data-action="toggle-notes"
+ onClick={() => setOpen((o) => !o)}
+ className={openCount ? badgeVariants({ variant: "open", size: "sm" }) : "text-[var(--color-sel)] hover:underline"}
+ >
+ {openCount ? `${openCount} open ${openCount === 1 ? "note" : "notes"}` : mine.length ? `${label} (${mine.length})` : `+ ${label.replace(/s$/, "")}`}
+ </button>
+ {open && (
+ <div className="mt-1 space-y-1 rounded bg-[var(--color-panel-2)] p-2">
+ {mine.map((n) => (
+ <NoteThread
+ key={n.id}
+ note={n}
+ write={notes.write}
+ busy={notes.busy}
+ head={
+ n.anchor.kind === "edit" ? (
+ <span className={badgeVariants({ variant: "info", size: "sm" })}>edit · {n.anchor.field}</span>
+ ) : null
+ }
+ />
+ ))}
+ <NoteComposer
+ busy={notes.busy}
+ placeholder={pin.kind === "entry" ? `a note on ${key}` : `a note on this take`}
+ testId={`note-input-${key}`}
+ onAdd={(text) => void notes.write({ op: "add", text, anchor: pin })}
+ />
+ {notes.error && <div className="text-[var(--color-bad)]">{notes.error}</div>}
+ </div>
+ )}
+ </div>
+ );
+}
diff --git a/umtool/components/notes/GeneratedBanner.tsx b/umtool/components/notes/GeneratedBanner.tsx
@@ -0,0 +1,15 @@
+// One line, wherever a generated manifest can be edited: the project page,
+// the clip bench, the On-screen section. Edits are still allowed; each one
+// leaves an `edit` note for the agent that runs the generator
+// (lib/report/guard.ts).
+export default function GeneratedBanner({ generatedBy }: { generatedBy: string | null | undefined }) {
+ if (!generatedBy) return null;
+ return (
+ <p
+ data-testid="generated-banner"
+ className="rounded border border-[var(--color-dirty)] px-2 py-1 text-[11px] text-[var(--color-dirty)]"
+ >
+ Generated by <code className="font-mono">{generatedBy}</code>; a rebuild of manifests overwrites edits made here.
+ </p>
+ );
+}
diff --git a/umtool/components/notes/NoteThread.tsx b/umtool/components/notes/NoteThread.tsx
@@ -0,0 +1,136 @@
+"use client";
+
+import { useState } from "react";
+import { badgeVariants } from "@/components/ui/badge";
+import type { Note, NoteOp } from "@/lib/annotations/types";
+
+// One note: its text, who wrote it, its status, the replies, and what can be
+// done to it. Shared by the timed notes, the row notes and the take notes.
+
+const STATUS_TONE = { open: "open", resolved: "info", wontfix: "info" } as const;
+
+export function NoteThread({
+ note,
+ write,
+ busy,
+ head,
+}: {
+ note: Note;
+ write: (op: NoteOp) => Promise<unknown>;
+ busy: boolean;
+ /** What the note is on, drawn before its text (a timestamp, an entry id). */
+ head?: React.ReactNode;
+}) {
+ const [reply, setReply] = useState<string | null>(null);
+ const open = note.status === "open";
+ return (
+ <div
+ data-note-id={note.id}
+ data-note-status={note.status}
+ className={`space-y-0.5 text-[12px] ${open ? "" : "opacity-60"}`}
+ >
+ <div className="flex flex-wrap items-baseline gap-1.5">
+ {head}
+ {note.author === "agent" && <span className={badgeVariants({ variant: "meter", size: "sm" })}>agent</span>}
+ {!open && <span className={badgeVariants({ variant: STATUS_TONE[note.status], size: "sm" })}>{note.status}</span>}
+ <span className="flex-1 whitespace-pre-wrap text-[var(--color-text)]">{note.text}</span>
+ <span className="flex gap-1.5 text-[10px]">
+ {open ? (
+ <>
+ <button type="button" data-note-action="resolve" disabled={busy} onClick={() => void write({ op: "status", id: note.id, status: "resolved" })} className="text-[var(--color-good)] hover:underline disabled:opacity-40">
+ resolve
+ </button>
+ <button type="button" data-note-action="wontfix" disabled={busy} onClick={() => void write({ op: "status", id: note.id, status: "wontfix" })} className="text-[var(--color-dim)] hover:underline disabled:opacity-40">
+ won’t fix
+ </button>
+ </>
+ ) : (
+ <button type="button" data-note-action="reopen" disabled={busy} onClick={() => void write({ op: "status", id: note.id, status: "open" })} className="text-[var(--color-sel)] hover:underline disabled:opacity-40">
+ reopen
+ </button>
+ )}
+ <button type="button" data-note-action="reply" disabled={busy} onClick={() => setReply((r) => (r === null ? "" : null))} className="text-[var(--color-sel)] hover:underline disabled:opacity-40">
+ reply
+ </button>
+ {note.author === "operator" && (
+ <button type="button" data-note-action="delete" disabled={busy} onClick={() => void write({ op: "delete", id: note.id })} className="text-[var(--color-dim)] hover:text-[var(--color-bad)] disabled:opacity-40">
+ delete
+ </button>
+ )}
+ </span>
+ </div>
+ {note.replies.map((r, i) => (
+ <div key={i} data-note-reply={i} className="ml-4 flex flex-wrap items-baseline gap-1.5 text-[11px]">
+ {r.author === "agent" && <span className={badgeVariants({ variant: "meter", size: "sm" })}>agent</span>}
+ <span className="whitespace-pre-wrap text-[var(--color-dim)]">{r.text}</span>
+ </div>
+ ))}
+ {reply !== null && (
+ <input
+ autoFocus
+ value={reply}
+ data-note-reply-input=""
+ onChange={(e) => setReply(e.target.value)}
+ onKeyDown={(e) => {
+ if (e.key === "Enter" && reply.trim()) {
+ void write({ op: "reply", id: note.id, text: reply });
+ setReply(null);
+ }
+ if (e.key === "Escape") setReply(null);
+ }}
+ placeholder="reply — Enter to send"
+ className="ml-4 w-[calc(100%-1rem)] rounded border border-[var(--color-line)] bg-[var(--color-ink)] px-1.5 py-0.5 text-[12px] text-[var(--color-text)] outline-none focus:border-[var(--color-sel)]"
+ />
+ )}
+ </div>
+ );
+}
+
+/** A one-line composer: Enter adds, Escape cancels. */
+export function NoteComposer({
+ onAdd,
+ onCancel,
+ busy,
+ placeholder,
+ head,
+ testId,
+}: {
+ onAdd: (text: string) => void;
+ onCancel?: () => void;
+ busy: boolean;
+ placeholder: string;
+ head?: React.ReactNode;
+ testId?: string;
+}) {
+ const [text, setText] = useState("");
+ const add = () => {
+ if (!text.trim()) return;
+ onAdd(text);
+ setText("");
+ };
+ return (
+ <div className="flex flex-wrap items-center gap-2">
+ {head}
+ <input
+ autoFocus
+ value={text}
+ data-testid={testId}
+ onChange={(e) => setText(e.target.value)}
+ onKeyDown={(e) => {
+ if (e.key === "Enter") add();
+ if (e.key === "Escape") onCancel?.();
+ }}
+ placeholder={placeholder}
+ className="min-w-[14rem] flex-1 rounded border border-[var(--color-line)] bg-[var(--color-ink)] px-1.5 py-0.5 text-[12px] text-[var(--color-text)] outline-none focus:border-[var(--color-sel)]"
+ />
+ <button
+ type="button"
+ disabled={busy || !text.trim()}
+ onClick={add}
+ className="rounded border border-[var(--color-good)] px-2 py-0.5 text-[11px] text-[var(--color-good)] disabled:opacity-40"
+ >
+ add
+ </button>
+ </div>
+ );
+}
diff --git a/umtool/components/notes/NotesProvider.tsx b/umtool/components/notes/NotesProvider.tsx
@@ -0,0 +1,65 @@
+"use client";
+
+import { createContext, useContext, useEffect } from "react";
+import { notesQuery, useNotes, type NotesTarget, type UseNotes } from "@/lib/annotations/useNotes";
+
+// ONE handle on a notes.json per page.
+//
+// Every write carries the token its handle last read, so two handles on the
+// same file in one page -- the timeline's row notes and the final video's
+// timed notes, say -- would 409 each other on every other write. A page wraps
+// itself in <NotesProvider target>, and every notes component under it shares
+// that handle; a component outside any provider (or under one for another
+// file) opens its own.
+
+const Ctx = createContext<{ key: string; notes: UseNotes } | null>(null);
+
+/** Something on the page wrote to a project's notes server-side (an edit note): re-read them. */
+export const NOTES_CHANGED = "umtool:notes-changed";
+export function announceNotesChanged(project: string) {
+ window.dispatchEvent(new CustomEvent(NOTES_CHANGED, { detail: { project } }));
+}
+
+/** Re-read `notes` when the page says its project's notes changed under it. */
+export function useNotesRefresh(target: NotesTarget | null, notes: UseNotes) {
+ const project = target && "project" in target ? target.project : null;
+ const { reload } = notes;
+ useEffect(() => {
+ if (!project) return;
+ const on = (e: Event) => {
+ if ((e as CustomEvent).detail?.project === project) void reload();
+ };
+ window.addEventListener(NOTES_CHANGED, on);
+ return () => window.removeEventListener(NOTES_CHANGED, on);
+ }, [project, reload]);
+}
+
+export function NotesProvider({ target, children }: { target: NotesTarget; children: React.ReactNode }) {
+ const notes = useNotes(target);
+ useNotesRefresh(target, notes);
+ return <Ctx.Provider value={{ key: notesQuery(target), notes }}>{children}</Ctx.Provider>;
+}
+
+/**
+ * Share a handle the page already holds (the article reader's) with the notes
+ * components under it, so they write with ITS token rather than opening a
+ * second one on the same file.
+ */
+export function ShareNotes({ target, notes, children }: { target: NotesTarget; notes: UseNotes; children: React.ReactNode }) {
+ return <Ctx.Provider value={{ key: notesQuery(target), notes }}>{children}</Ctx.Provider>;
+}
+
+/** The page's shared handle for `target`, or a handle of this component's own. */
+export function useSharedNotes(target: NotesTarget | null): UseNotes {
+ const ctx = useContext(Ctx);
+ const shared = !!(ctx && target && ctx.key === notesQuery(target));
+ const own = useNotes(shared ? null : target);
+ useNotesRefresh(shared ? null : target, own);
+ return shared ? ctx!.notes : own;
+}
+
+/** True when a write's response says the server wrote notes too (an edit on a generated manifest). */
+export const wroteNotes = (j: unknown): boolean => {
+ const e = (j as { editNotes?: { added?: number; updated?: number; deleted?: number } | null })?.editNotes;
+ return !!e && (e.added ?? 0) + (e.updated ?? 0) + (e.deleted ?? 0) > 0;
+};
diff --git a/umtool/components/notes/TimedNotes.tsx b/umtool/components/notes/TimedNotes.tsx
@@ -0,0 +1,239 @@
+"use client";
+
+import { useCallback, useEffect, useState } from "react";
+import { badgeVariants } from "@/components/ui/badge";
+import type { Anchor, MomentResolved, Note } from "@/lib/annotations/types";
+import type { NotesTarget, UseNotes } from "@/lib/annotations/useNotes";
+import { useSharedNotes } from "./NotesProvider";
+import { NoteComposer, NoteThread } from "./NoteThread";
+
+// Notes at a SECOND of a rendered video, in the notes.json of whatever the
+// video belongs to (a report-video project, an article). The song tool's
+// MomentMarks keeps its own store; this is the same idea bound to
+// lib/annotations, so an agent reads it back with everything else.
+//
+// `n` (with the video focused) or "Mark" opens a note at the playhead. When
+// the video is a report project's build, the second is RESOLVED at write time
+// (/api/report/moment: the entry on screen, its title and quote, the source
+// second and its archive link) and stored in the note -- a rebuild moves
+// entries, and the agent must see what was on screen when the key was pressed.
+// A file with no build schedule keeps `t` alone.
+
+const ts = (t: number) => {
+ const m = Math.floor(t / 60);
+ const s = t - m * 60;
+ return `${m}:${s.toFixed(1).padStart(4, "0")}`;
+};
+
+type MomentAnchor = Extract<Anchor, { kind: "moment" }>;
+const isMomentOn = (file: string) => (n: Note): n is Note & { anchor: MomentAnchor } =>
+ n.anchor.kind === "moment" && n.anchor.file === file;
+
+export default function TimedNotes({
+ notes,
+ file,
+ video,
+ take,
+ resolveProject,
+ showErrors = true,
+}: {
+ notes: UseNotes;
+ /** The file's path as the anchor stores it: project-relative (`takes/<id>/preview.mp4`), or the article's `video.mp4`. */
+ file: string;
+ video: HTMLVideoElement | null;
+ /** The take this file is a preview of, stored on the anchor. */
+ take?: string;
+ /** Resolve each mark against this project's build schedule. */
+ resolveProject?: string | null;
+ /** Show the handle's errors here. Off when the handle is shared with a page that shows them itself (the article reader). */
+ showErrors?: boolean;
+}) {
+ const [duration, setDuration] = useState<number | null>(null);
+ const [pending, setPending] = useState<number | null>(null);
+ const marks = notes.notes.filter(isMomentOn(file)).sort((a, b) => a.anchor.t - b.anchor.t);
+
+ const mark = useCallback(() => {
+ if (!video) return;
+ setPending(Number(video.currentTime.toFixed(2)));
+ }, [video]);
+
+ useEffect(() => {
+ if (!video) return;
+ const dur = () => setDuration(Number.isFinite(video.duration) ? video.duration : null);
+ const key = (e: KeyboardEvent) => {
+ if ((e.key === "n" || e.key === "N") && !e.ctrlKey && !e.metaKey && !e.altKey) {
+ e.preventDefault();
+ e.stopPropagation();
+ mark();
+ }
+ };
+ dur();
+ video.addEventListener("loadedmetadata", dur);
+ video.addEventListener("durationchange", dur);
+ video.addEventListener("keydown", key);
+ return () => {
+ video.removeEventListener("loadedmetadata", dur);
+ video.removeEventListener("durationchange", dur);
+ video.removeEventListener("keydown", key);
+ };
+ }, [video, mark]);
+
+ const seek = (t: number) => {
+ if (!video) return;
+ video.currentTime = t;
+ video.focus();
+ };
+
+ const add = async (t: number, text: string) => {
+ let resolved: (MomentResolved & { entry?: string | null }) | null = null;
+ if (resolveProject) {
+ const q = new URLSearchParams({ project: resolveProject, file, t: String(t) });
+ if (duration) q.set("duration", String(duration));
+ resolved = await fetch(`/api/report/moment?${q}`, { cache: "no-store" })
+ .then((r) => (r.ok ? r.json() : null))
+ .catch(() => null);
+ }
+ const anchor: MomentAnchor = { kind: "moment", file, t };
+ if (take) anchor.take = take;
+ if (resolved?.entry) {
+ anchor.entry = resolved.entry;
+ const { entry: _e, ...rest } = resolved as Record<string, unknown>;
+ delete rest.schedule;
+ anchor.resolved = rest as MomentResolved;
+ }
+ await notes.write({ op: "add", text, anchor });
+ };
+
+ return (
+ <div data-timed-notes={file} data-marks={marks.length} className="space-y-1">
+ {/* the tick strip: where the marks are, under the player */}
+ {duration ? (
+ <div className="relative h-2 rounded bg-[var(--color-panel-2)]" data-tick-strip="">
+ {marks.map((n) => (
+ <button
+ key={n.id}
+ type="button"
+ title={`${ts(n.anchor.t)} — ${n.text}`}
+ onClick={() => seek(n.anchor.t)}
+ data-tick={n.anchor.t}
+ className={`absolute top-0 h-2 w-1 -translate-x-1/2 rounded ${n.status === "open" ? "bg-[var(--color-dirty)]" : "bg-[var(--color-dim)]"}`}
+ style={{ left: `${Math.min(100, (n.anchor.t / duration) * 100)}%` }}
+ />
+ ))}
+ </div>
+ ) : null}
+ <div className="flex flex-wrap items-center gap-2 text-[11px]">
+ <button
+ type="button"
+ data-action="mark"
+ disabled={!video}
+ onClick={mark}
+ className="rounded border border-[var(--color-sel)] px-2 py-0.5 text-[11px] text-[var(--color-sel)] disabled:opacity-40"
+ >
+ Mark <kbd>n</kbd>
+ </button>
+ {marks.length > 0 && (
+ <span className="text-[var(--color-dim)]">
+ {marks.filter((m) => m.status === "open").length} open of {marks.length}
+ </span>
+ )}
+ {showErrors && notes.error && <span className="text-[var(--color-bad)]">{notes.error}</span>}
+ </div>
+ {pending !== null && (
+ <div className="rounded bg-[var(--color-panel-2)] p-2">
+ <NoteComposer
+ busy={notes.busy}
+ placeholder="what is wrong here"
+ testId="timed-note-input"
+ head={<span className="num text-[11px] text-[var(--color-meter)]">{ts(pending)}</span>}
+ onCancel={() => setPending(null)}
+ onAdd={(text) => {
+ const t = pending;
+ setPending(null);
+ void add(t, text);
+ }}
+ />
+ </div>
+ )}
+ {marks.length > 0 && (
+ <ul className="space-y-1">
+ {marks.map((n) => {
+ const r = n.anchor.resolved;
+ return (
+ <li key={n.id} data-mark-at={n.anchor.t} data-mark-entry={n.anchor.entry ?? ""}>
+ <NoteThread
+ note={n}
+ write={notes.write}
+ busy={notes.busy}
+ head={
+ <>
+ <button
+ type="button"
+ onClick={() => seek(n.anchor.t)}
+ className="num text-[11px] text-[var(--color-sel)] underline"
+ title="seek here"
+ >
+ {ts(n.anchor.t)}
+ </button>
+ {n.anchor.entry && (
+ <span className="font-mono text-[11px] text-[var(--color-dim)]" title={r?.quote ?? undefined}>
+ {n.anchor.entry}
+ {r?.title ? ` · ${r.title}` : ""}
+ </span>
+ )}
+ {r?.approx && <span className={badgeVariants({ variant: "info", size: "sm" })}>approx</span>}
+ {r?.url && (
+ <a href={r.url} target="_blank" rel="noreferrer" className="text-[11px] text-[var(--color-sel)] hover:underline">
+ source
+ </a>
+ )}
+ </>
+ }
+ />
+ </li>
+ );
+ })}
+ </ul>
+ )}
+ </div>
+ );
+}
+
+/**
+ * A video with its timed notes under it: what the project's final cut, a
+ * take's preview or an article's report video mounts. `notes` defaults to the
+ * page's shared handle for `target` (NotesProvider), else one of its own.
+ */
+export function TimedVideo({
+ target,
+ file,
+ src,
+ take,
+ resolveProject,
+ notes: given,
+ poster,
+ testId,
+ showErrors,
+ className = "aspect-video w-full rounded border border-[var(--color-line)] bg-black",
+}: {
+ target: NotesTarget;
+ file: string;
+ src: string;
+ take?: string;
+ resolveProject?: string | null;
+ notes?: UseNotes;
+ poster?: string;
+ testId?: string;
+ showErrors?: boolean;
+ className?: string;
+}) {
+ const own = useSharedNotes(given ? null : target);
+ const notes = given ?? own;
+ const [el, setEl] = useState<HTMLVideoElement | null>(null);
+ return (
+ <div className="space-y-1">
+ <video ref={setEl} data-testid={testId} src={src} poster={poster} controls preload="metadata" playsInline className={className} />
+ <TimedNotes notes={notes} file={file} video={el} take={take} resolveProject={resolveProject} showErrors={showErrors} />
+ </div>
+ );
+}
diff --git a/umtool/components/projects/ClipBench.tsx b/umtool/components/projects/ClipBench.tsx
@@ -1,5 +1,8 @@
"use client";
+import AnchoredNotes from "@/components/notes/AnchoredNotes";
+import GeneratedBanner from "@/components/notes/GeneratedBanner";
+import { useSharedNotes, wroteNotes } from "@/components/notes/NotesProvider";
import { useCallback, useEffect, useRef, useState } from "react";
import Link from "next/link";
import { useRouter } from "next/navigation";
@@ -256,6 +259,8 @@ export type ClipBenchData = {
siblings: Sibling[];
/** Is the cut built with the on-screen panel (`render.chrome`)? */
deckOn: boolean;
+ /** The manifest's `generatedBy`: edits here are noted for the agent that runs it. */
+ generatedBy?: string | null;
};
const hms = (t: number) => {
@@ -377,6 +382,9 @@ type Session = {
};
export default function ClipBench({ data }: { data: ClipBenchData }) {
+ // The project's notes: this clip's row notes, and the edit notes a save on a
+ // generated manifest leaves (the save says so, and they are re-read).
+ const notes = useSharedNotes({ project: data.project });
const [clip, setClip] = useState<Clip>(data.clip);
const [windows, setWindows] = useState<Win[]>(data.windows);
const [cues, setCues] = useState<Cue[]>(data.cues);
@@ -1084,6 +1092,7 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
});
const j = (await r.json()) as Record<string, unknown>;
setBusy(null);
+ if (wroteNotes(j)) void notes.reload();
if (!r.ok) {
setNote(
j.stale
@@ -1134,7 +1143,7 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
void refresh();
return true;
},
- [data.project, clip],
+ [data.project, clip, notes.reload],
);
/**
@@ -1484,7 +1493,7 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
// Returned, not just stored: "did anything actually arrive" is a question
// the caller has to answer before it claims a fetch worked.
return j;
- }, [data.project, clip.id]);
+ }, [data.project, clip.id, notes.reload]);
// ---- fetching more --------------------------------------------------------
//
@@ -1684,6 +1693,7 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
}
setClip((prev) => fromEntry(prev, j.entry ?? {}));
token.current = String(j.token ?? "");
+ if (wroteNotes(j)) void notes.reload();
setCutScore(j.score ?? null);
setNote(`cut to the quote (match ${(j.score ?? 0).toFixed(2)})`);
}, [data.project, clip.id]);
@@ -1849,6 +1859,7 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
data-verdict={verdict}
className="flex flex-col gap-2 lg:h-full lg:min-h-0 lg:overflow-hidden"
>
+ <GeneratedBanner generatedBy={data.generatedBy} />
{/* ---- where you are, where you can go, and what will be burned in ---- */}
<div className="flex flex-wrap items-center gap-x-3 gap-y-1 text-[11px]">
<span className="num font-mono text-[var(--color-text)]">
@@ -2942,6 +2953,9 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
})}
</div>
+ {/* ---- notes on this clip, for whoever acts on them ---- */}
+ <AnchoredNotes notes={notes} pin={{ kind: "entry", entry: clip.id }} startOpen={false} />
+
{/* ---- on-screen: what the panel under the footage says ----
Beside the header's fields because it is the same sitting: the
words you hear are the words a title should summarise. Both
diff --git a/umtool/components/projects/ClipBenchPage.tsx b/umtool/components/projects/ClipBenchPage.tsx
@@ -105,6 +105,7 @@ export default async function ClipBenchPage({
// Whether the cut is built with the on-screen panel. Decides whether the
// bench composes a preview of it; the fields are there either way.
deckOn: deckOn(manifest.render ?? {}),
+ generatedBy: typeof manifest.generatedBy === "string" ? manifest.generatedBy : null,
view,
windows: windows.map((w: { name: string; from: number; to: number }) => ({
name: w.name,
diff --git a/umtool/components/projects/OnscreenSection.tsx b/umtool/components/projects/OnscreenSection.tsx
@@ -1,5 +1,10 @@
"use client";
+import GeneratedBanner from "@/components/notes/GeneratedBanner";
+import { TimedVideo } from "@/components/notes/TimedNotes";
+import StructureEditors from "./StructureEditors";
+import { MANIFEST_CHANGED } from "./timelineApi";
+import { announceNotesChanged, wroteNotes } from "@/components/notes/NotesProvider";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { badgeVariants } from "@/components/ui/badge";
import { buttonVariants } from "@/components/ui/button";
@@ -772,8 +777,11 @@ export default function OnscreenSection({
project,
entries,
built,
+ generatedBy = null,
}: {
project: string;
+ /** The manifest's `generatedBy`: its edits are noted for the agent, and the section says so. */
+ generatedBy?: string | null;
/**
* Is the default cut's deliverable on disk? The server already knows, and
* asking the video route about a file that is not there is a 404 in the
@@ -821,6 +829,9 @@ export default function OnscreenSection({
const [job, setJob] = useState<JobView | null>(null);
const [jobError, setJobError] = useState<string | null>(null);
const [finalV, setFinalV] = useState<number | null | "none">(null);
+ // The built file's project-relative path (the video route's x-video): what a
+ // timed note on it is anchored to.
+ const [finalRel, setFinalRel] = useState<string | null>(null);
// The switch as pressed, while its write is in flight: the box follows the
// hand at once and falls back to the manifest's answer if the write fails.
const [switching, setSwitching] = useState<boolean | null>(null);
@@ -933,6 +944,7 @@ export default function OnscreenSection({
const loadFinal = useCallback(async () => {
const r = await fetch(`/api/report/video?${q}&kind=final`, { method: "HEAD", cache: "no-store" }).catch(() => null);
const m = r?.ok ? r.headers.get("x-video-mtime") : null;
+ setFinalRel(r?.ok ? r.headers.get("x-video") : null);
setFinalV(m ? Number(m) : "none");
}, [q]);
@@ -959,6 +971,28 @@ export default function OnscreenSection({
// `built` and `variant` are read once per load; loadFinal already follows the variant.
}, [loadChrome, loadRows, loadFinal, recompose]);
+ // A structural write elsewhere on the page (the timeline, the teaser and
+ // posts editors) changed the manifest: re-read the tokens and the rows,
+ // keeping every unsaved edit here, or the next save would 409.
+ const formDirtyRef = useRef(formDirty);
+ formDirtyRef.current = formDirty;
+ useEffect(() => {
+ const on = (e: Event) => {
+ if ((e as CustomEvent).detail?.project !== project) return;
+ void (async () => {
+ const r = await fetch(`/api/report/chrome?project=${encodeURIComponent(project)}`, { cache: "no-store" });
+ const j = (await r.json().catch(() => null)) as (ChromeDoc & { error?: string }) | null;
+ if (r.ok && j) {
+ token.current = j.token;
+ if (!formDirtyRef.current) setDoc(j);
+ }
+ await loadRows(true);
+ })();
+ };
+ window.addEventListener(MANIFEST_CHANGED, on);
+ return () => window.removeEventListener(MANIFEST_CHANGED, on);
+ }, [project, loadRows]);
+
// ---- writing ------------------------------------------------------------
const queued = useCallback(<T,>(fn: () => Promise<T>): Promise<T> => {
const run = saving.current.catch(() => null).then(fn);
@@ -978,6 +1012,7 @@ export default function OnscreenSection({
body: JSON.stringify({ project, chrome, token: token.current }),
});
const j = (await r.json()) as Record<string, unknown>;
+ if (wroteNotes(j)) announceNotesChanged(project);
setBusy(null);
if (!r.ok) {
if (j.stale) {
@@ -1072,6 +1107,7 @@ export default function OnscreenSection({
body: JSON.stringify({ project, onscreen: draftMap(), token: token.current }),
});
const j = (await r.json()) as Record<string, unknown>;
+ if (wroteNotes(j)) announceNotesChanged(project);
setBusy(null);
if (!r.ok) {
// The drafts stay exactly as typed. A refused batch is a typo or a
@@ -1114,6 +1150,7 @@ export default function OnscreenSection({
body: JSON.stringify({ project, posts: postsDraftMap(), token: token.current }),
});
const j = (await r.json()) as Record<string, unknown>;
+ if (wroteNotes(j)) announceNotesChanged(project);
setBusy(null);
if (!r.ok) {
if (j.stale) {
@@ -1626,6 +1663,7 @@ export default function OnscreenSection({
data-onscreen={on ? "on" : "off"}
className="space-y-3 rounded border border-[var(--color-line)] bg-[var(--color-panel)] p-3"
>
+ <GeneratedBanner generatedBy={generatedBy} />
{/* ---- the switch ---- */}
<div className="flex flex-wrap items-center gap-x-3 gap-y-1">
<h2 className="micro">on-screen</h2>
@@ -2050,12 +2088,12 @@ export default function OnscreenSection({
<div className="space-y-1">
<span className="micro">the built video</span>
{typeof finalV === "number" ? (
- <video
- data-testid="onscreen-final-video"
+ <TimedVideo
+ target={{ project }}
+ testId="onscreen-final-video"
+ file={finalRel ?? `out/${variant || "sourced"}/final.mp4`}
src={`/api/report/video?${q}&kind=final&v=${finalV}`}
- controls
- preload="metadata"
- className="aspect-video w-full rounded border border-[var(--color-line)] bg-black"
+ resolveProject={project}
/>
) : (
<p data-testid="onscreen-no-final" className="text-[11px] text-[var(--color-dim)]">
@@ -2093,6 +2131,14 @@ export default function OnscreenSection({
<div className="mt-1.5">{postsBlock}</div>
</details>
)}
+
+ {/* ---- the lists: teasers, posts, fact-check labels ---- */}
+ <details data-testid="structure-folded">
+ <summary className="cursor-pointer text-[11px] text-[var(--color-dim)]">teasers, posts and fact-check labels</summary>
+ <div className="mt-1.5">
+ <StructureEditors project={project} />
+ </div>
+ </details>
</section>
);
}
diff --git a/umtool/components/projects/ReportProject.tsx b/umtool/components/projects/ReportProject.tsx
@@ -24,6 +24,9 @@ import OnscreenSection from "./OnscreenSection";
import ReportBuildChain from "./ReportBuildChain";
import SnapshotButton from "./SnapshotButton";
import TagCitedButton from "./TagCitedButton";
+import { RowControls, RowNotes, TimelineList } from "./TimelineEditor";
+import GeneratedBanner from "@/components/notes/GeneratedBanner";
+import { NotesProvider } from "@/components/notes/NotesProvider";
import { decisionsForProject } from "@/lib/projects";
import { badgeVariants, type BadgeVariants } from "@/components/ui/badge";
import { buttonVariants } from "@/components/ui/button";
@@ -257,7 +260,9 @@ export default async function ReportProject({
)}
</div>
+ <NotesProvider target={{ project: project.id }}>
<main className="deck-main flex-1 space-y-4 p-4">
+ <GeneratedBanner generatedBy={typeof m.generatedBy === "string" ? m.generatedBy : null} />
{/* --- the author's own account of the cut, first ------------------ */}
{readme && (
<section data-readme="" className="rounded border border-[var(--color-line)] bg-[var(--color-panel)] px-3 py-2">
@@ -430,6 +435,7 @@ export default async function ReportProject({
because the panel is part of the video that gets delivered. */}
<OnscreenSection
project={project.id}
+ generatedBy={typeof m.generatedBy === "string" ? m.generatedBy : null}
entries={entries.map((e) => ({ id: e.id, kind: e.kind, segment: !!e.segment }))}
built={!!build.built}
/>
@@ -459,8 +465,8 @@ export default async function ReportProject({
/>
</div>
)}
- <ul className="space-y-1">
- {entries.map((e) => {
+ <TimelineList project={project.id} count={entries.length}>
+ {entries.map((e, at) => {
// Anything that is not a CLIP renders generically. The timeline's
// vocabulary is open -- one real manifest carries `scroll` and
// `chart` beside its cards -- and a page that only knows two words
@@ -468,17 +474,23 @@ export default async function ReportProject({
if (e.kind !== "clip") {
return (
<li
- key={e.id}
+ key={`${e.id}@${at}`}
data-entry={e.id}
data-kind={e.kind}
- className="flex flex-wrap items-baseline gap-2 rounded border border-dashed border-[var(--color-line)] px-3 py-1.5 text-[12px]"
+ data-at={at}
+ tabIndex={0}
+ className="flex flex-wrap items-baseline gap-2 rounded border border-dashed border-[var(--color-line)] px-3 py-1.5 text-[12px] outline-none focus:border-[var(--color-sel)]"
>
+ <RowControls id={e.id} at={at} />
<span className="font-mono text-[var(--color-dim)]">{e.id}</span>
<Pill>{e.style ? `${e.kind} · ${e.style}` : e.kind}</Pill>
<span className="text-[var(--color-text)]">
{e.heading ?? e.title ?? e.label ?? ""}
</span>
{e.seconds != null && <span className="num micro ml-auto">{e.seconds}s</span>}
+ <div className="basis-full">
+ <RowNotes id={e.id} project={project.id} />
+ </div>
</li>
);
}
@@ -487,15 +499,18 @@ export default async function ReportProject({
const midSentence = e.endsSentence === false && !e.lockEnd && !e.lock;
return (
<li
- key={e.id}
+ key={`${e.id}@${at}`}
data-entry={e.id}
data-kind="clip"
+ data-at={at}
+ tabIndex={0}
data-cached={e.cached ? "1" : "0"}
data-fetched={e.fetched ? "1" : "0"}
data-mid-sentence={midSentence ? "1" : "0"}
- className="rounded border border-[var(--color-line)] bg-[var(--color-panel)] px-3 py-1.5"
+ className="rounded border border-[var(--color-line)] bg-[var(--color-panel)] px-3 py-1.5 outline-none focus:border-[var(--color-sel)]"
>
<div className="flex flex-wrap items-baseline gap-2 text-[12px]">
+ <RowControls id={e.id} at={at} />
<span className="font-mono text-[var(--color-sel)]">{e.id}</span>
<span className="num font-mono text-[11px] text-[var(--color-dim)]">
{e.video} {hms(e.start)}–{hms(e.end)} ({(e.end - e.start).toFixed(1)}s)
@@ -568,10 +583,11 @@ export default async function ReportProject({
set <code className="font-mono">lock</code> if this window is deliberate
</p>
)}
+ <RowNotes id={e.id} project={project.id} />
</li>
);
})}
- </ul>
+ </TimelineList>
<div className="mt-2">
<Link
href={`/browse/${project.id}${showAll ? "" : "?all=1"}`}
@@ -771,6 +787,7 @@ export default async function ReportProject({
</section>
)}
</main>
+ </NotesProvider>
</div>
);
}
diff --git a/umtool/components/projects/StructureEditors.tsx b/umtool/components/projects/StructureEditors.tsx
@@ -0,0 +1,286 @@
+"use client";
+
+import { useRouter } from "next/navigation";
+import { useCallback, useEffect, useRef, useState } from "react";
+import { announceNotesChanged, wroteNotes } from "@/components/notes/NotesProvider";
+import { buttonVariants } from "@/components/ui/button";
+import { MANIFEST_CHANGED, announceManifestChanged, refusalOf, timelineOp } from "./timelineApi";
+
+// The parts of the cut that are lists, edited in place: each teaser's lines
+// and timing, the posts, and the fact-check's labels and colours. Every save
+// is one structural write (POST /api/report/timeline): checked by the build's
+// own validators, snapshotted first, and -- on a generated manifest -- noted
+// for the agent.
+
+type Teaser = { at: number; id: string; lines: unknown[]; beat: number | null; dip: { fade: number; black: number } | null; tail: string | null; tailWait: number | null };
+type Post = Record<string, unknown> & { id: string };
+type Verdict = { label: string; color: string };
+type Doc = {
+ teasers: Teaser[];
+ posts: Post[];
+ deckOn: boolean;
+ factcheck: { verdicts?: Record<string, Partial<Verdict>>; stamp?: Record<string, unknown>; tally?: Record<string, unknown> } | null;
+ factcheckResolved: { verdicts: Record<string, Verdict>; stamp: { seconds: number; position: string }; tally: { show: boolean; position: string } };
+ verdicts: string[];
+ token: string | null;
+};
+
+const input =
+ "rounded border border-[var(--color-line)] bg-[var(--color-ink)] px-1.5 py-0.5 text-[12px] text-[var(--color-text)] outline-none focus:border-[var(--color-sel)]";
+const num = (v: string) => (v.trim() === "" ? null : Number(v));
+
+export default function StructureEditors({ project }: { project: string }) {
+ const router = useRouter();
+ const [doc, setDoc] = useState<Doc | null>(null);
+ const [error, setError] = useState<string | null>(null);
+ const [busy, setBusy] = useState(false);
+
+ const load = useCallback(async () => {
+ const r = await fetch(`/api/report/timeline?project=${encodeURIComponent(project)}`, { cache: "no-store" });
+ const j = await r.json();
+ if (r.ok) setDoc(j as Doc);
+ else setError(String(j.error ?? r.status));
+ }, [project]);
+
+ useEffect(() => {
+ void load();
+ const on = (e: Event) => {
+ if ((e as CustomEvent).detail?.project === project) void load();
+ };
+ window.addEventListener(MANIFEST_CHANGED, on);
+ return () => window.removeEventListener(MANIFEST_CHANGED, on);
+ }, [load, project]);
+
+ const run = async (op: string, args: Record<string, unknown>) => {
+ setBusy(true);
+ setError(null);
+ const res = await timelineOp(project, doc?.token ?? null, op, args);
+ setBusy(false);
+ if (!res.ok) {
+ setError(refusalOf(res));
+ if (res.json.stale) await load();
+ return false;
+ }
+ // Adopt the new token now: the reload the announcement starts may land
+ // after the next save is pressed.
+ if (typeof res.json.token === "string") setDoc((d) => (d ? { ...d, token: res.json.token as string } : d));
+ announceManifestChanged(project);
+ if (wroteNotes(res.json)) announceNotesChanged(project);
+ router.refresh();
+ return true;
+ };
+
+ if (!doc) return error ? <p className="text-[11px] text-[var(--color-bad)]">{error}</p> : null;
+ return (
+ <div data-testid="structure-editors" className="space-y-3">
+ {error && (
+ <p data-testid="structure-error" className="text-[11px] text-[var(--color-bad)]">
+ {error}
+ </p>
+ )}
+ {doc.teasers.map((t) => (
+ <TeaserEditor key={`${t.id}@${t.at}`} teaser={t} busy={busy} save={(patch) => run("teaser", { id: t.id, at: t.at, patch })} />
+ ))}
+ <PostsEditor posts={doc.posts} busy={busy} save={(post) => run("post", { post })} remove={(id) => run("post-remove", { id })} />
+ {doc.deckOn && <FactcheckEditor doc={doc} busy={busy} save={(factcheck) => run("factcheck", { factcheck })} />}
+ </div>
+ );
+}
+
+function TeaserEditor({ teaser: t, busy, save }: { teaser: Teaser; busy: boolean; save: (patch: Record<string, unknown>) => Promise<boolean> }) {
+ // Plain lines edit as one per row; a teaser whose lines carry settings
+ // (break, role, replace…) edits as the JSON it is, so nothing is dropped.
+ const plain = t.lines.every((l) => typeof l === "string");
+ const linesText = () => (plain ? (t.lines as string[]).join("\n") : JSON.stringify(t.lines, null, 2));
+ const [lines, setLines] = useState(linesText);
+ const [beat, setBeat] = useState(t.beat == null ? "" : String(t.beat));
+ const [tail, setTail] = useState(t.tail ?? "");
+ const [tailWait, setTailWait] = useState(t.tailWait == null ? "" : String(t.tailWait));
+ const [fade, setFade] = useState(t.dip ? String(t.dip.fade) : "");
+ const [black, setBlack] = useState(t.dip ? String(t.dip.black) : "");
+ const [local, setLocal] = useState<string | null>(null);
+ // Unsaved edits win over a reload: the re-read every structural write sets
+ // off can land after somebody has started typing again.
+ const dirty = useRef(false);
+ const edit = <T,>(set: (v: T) => void) => (v: T) => {
+ dirty.current = true;
+ set(v);
+ };
+ useEffect(() => {
+ if (dirty.current) return;
+ setLines(linesText());
+ setBeat(t.beat == null ? "" : String(t.beat));
+ setTail(t.tail ?? "");
+ setTailWait(t.tailWait == null ? "" : String(t.tailWait));
+ setFade(t.dip ? String(t.dip.fade) : "");
+ setBlack(t.dip ? String(t.dip.black) : "");
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [t]);
+
+ const submit = () => {
+ setLocal(null);
+ let parsed: unknown[];
+ try {
+ parsed = plain ? lines.split("\n").map((l) => l.trim()).filter(Boolean) : JSON.parse(lines);
+ } catch {
+ setLocal("the lines are not valid JSON");
+ return;
+ }
+ const dip = fade.trim() || black.trim() ? { fade: num(fade), black: num(black) } : null;
+ void save({ lines: parsed, beat: num(beat), tail: tail.trim() || null, tailWait: num(tailWait), dip }).then((ok) => {
+ if (ok) dirty.current = false;
+ });
+ };
+
+ return (
+ <div data-teaser-editor={t.id} className="space-y-1 rounded border border-[var(--color-line)] p-2">
+ <div className="micro">teaser · {t.id}</div>
+ <textarea data-testid={`teaser-lines-${t.id}`} rows={Math.min(8, Math.max(2, lines.split("\n").length))} value={lines} onChange={(e) => edit(setLines)(e.target.value)} className={`${input} w-full font-mono`} />
+ <div className="flex flex-wrap items-center gap-2 text-[11px] text-[var(--color-dim)]">
+ <label>beat <input data-testid={`teaser-beat-${t.id}`} value={beat} onChange={(e) => edit(setBeat)(e.target.value)} className={`${input} w-14`} /></label>
+ <label>tail <input value={tail} onChange={(e) => edit(setTail)(e.target.value)} className={`${input} w-16`} /></label>
+ <label>tail wait <input value={tailWait} onChange={(e) => edit(setTailWait)(e.target.value)} className={`${input} w-14`} /></label>
+ <label>dip fade <input value={fade} onChange={(e) => edit(setFade)(e.target.value)} className={`${input} w-14`} /></label>
+ <label>black <input value={black} onChange={(e) => edit(setBlack)(e.target.value)} className={`${input} w-14`} /></label>
+ <button type="button" data-testid={`teaser-save-${t.id}`} disabled={busy} onClick={submit} className={buttonVariants({ variant: "primary", size: "sm" })}>
+ Save teaser
+ </button>
+ {local && <span className="text-[var(--color-bad)]">{local}</span>}
+ </div>
+ </div>
+ );
+}
+
+const POST_FIELDS = ["platform", "author", "handle", "date", "url", "shot", "flag"] as const;
+
+function PostsEditor({
+ posts,
+ busy,
+ save,
+ remove,
+}: {
+ posts: Post[];
+ busy: boolean;
+ save: (post: Post) => Promise<boolean>;
+ remove: (id: string) => Promise<boolean>;
+}) {
+ const [adding, setAdding] = useState(false);
+ return (
+ <div data-testid="posts-editor" className="space-y-1">
+ <div className="flex items-center gap-2">
+ <span className="micro">posts — {posts.length}</span>
+ <button type="button" data-testid="post-add" onClick={() => setAdding((a) => !a)} className="text-[11px] text-[var(--color-sel)] hover:underline">
+ + post
+ </button>
+ </div>
+ {adding && <PostRow post={{ id: "", platform: "x" }} fresh busy={busy} save={async (p) => (await save(p)) && (setAdding(false), true)} />}
+ {posts.map((p) => (
+ <PostRow key={p.id} post={p} busy={busy} save={save} remove={() => void remove(p.id)} />
+ ))}
+ </div>
+ );
+}
+
+function PostRow({ post, fresh = false, busy, save, remove }: { post: Post; fresh?: boolean; busy: boolean; save: (p: Post) => Promise<boolean>; remove?: () => void }) {
+ const [d, setD] = useState<Record<string, string>>(() => {
+ const out: Record<string, string> = { id: post.id, text: String(post.text ?? "") };
+ for (const k of POST_FIELDS) out[k] = post[k] == null ? "" : String(post[k]);
+ return out;
+ });
+ const set = (k: string, v: string) => setD((x) => ({ ...x, [k]: v }));
+ return (
+ <div data-post-row={post.id || "new"} className="space-y-1 rounded border border-[var(--color-line)] p-2 text-[11px]">
+ <div className="flex flex-wrap items-center gap-1.5 text-[var(--color-dim)]">
+ {fresh ? (
+ <label>id <input data-testid="post-id" value={d.id} onChange={(e) => set("id", e.target.value)} className={`${input} w-28 font-mono`} /></label>
+ ) : (
+ <span className="font-mono text-[var(--color-text)]">{post.id}</span>
+ )}
+ <select value={d.platform} onChange={(e) => set("platform", e.target.value)} className={`${input} text-[11px]`}>
+ {["x", "bluesky", "web"].map((p) => (
+ <option key={p}>{p}</option>
+ ))}
+ </select>
+ {(["author", "handle", "date", "url", "shot", "flag"] as const).map((k) => (
+ <label key={k}>
+ {k} <input data-testid={`post-${k}`} value={d[k]} onChange={(e) => set(k, e.target.value)} className={`${input} ${k === "url" ? "w-56" : "w-28"}`} />
+ </label>
+ ))}
+ </div>
+ <textarea data-testid="post-text" rows={2} value={d.text} onChange={(e) => set("text", e.target.value)} className={`${input} w-full`} />
+ <div className="flex gap-2">
+ <button
+ type="button"
+ data-testid="post-save"
+ disabled={busy}
+ onClick={() => {
+ const p: Post = { id: d.id.trim(), text: d.text };
+ for (const k of POST_FIELDS) p[k] = d[k];
+ void save(p);
+ }}
+ className={buttonVariants({ variant: "primary", size: "sm" })}
+ >
+ {fresh ? "Add post" : "Save post"}
+ </button>
+ {remove && (
+ <button type="button" data-testid="post-remove" disabled={busy} onClick={remove} className={buttonVariants({ variant: "destructive", size: "sm" })}>
+ Remove
+ </button>
+ )}
+ </div>
+ </div>
+ );
+}
+
+function FactcheckEditor({ doc, busy, save }: { doc: Doc; busy: boolean; save: (f: Record<string, unknown> | null) => Promise<boolean> }) {
+ const given = doc.factcheck ?? {};
+ const [v, setV] = useState<Record<string, { label: string; color: string }>>(() =>
+ Object.fromEntries(doc.verdicts.map((k) => [k, { label: String(given.verdicts?.[k]?.label ?? ""), color: String(given.verdicts?.[k]?.color ?? "") }])),
+ );
+ const [seconds, setSeconds] = useState(given.stamp?.seconds == null ? "" : String(given.stamp.seconds));
+ const submit = () => {
+ const verdicts: Record<string, Record<string, string>> = {};
+ for (const [k, o] of Object.entries(v)) {
+ const one: Record<string, string> = {};
+ if (o.label.trim()) one.label = o.label.trim();
+ if (o.color.trim()) one.color = o.color.trim();
+ if (Object.keys(one).length) verdicts[k] = one;
+ }
+ // Only what differs from the defaults is written, as the deck's settings are.
+ const out: Record<string, unknown> = { ...given };
+ if (Object.keys(verdicts).length) out.verdicts = verdicts;
+ else delete out.verdicts;
+ const stamp = { ...(given.stamp ?? {}) } as Record<string, unknown>;
+ if (seconds.trim()) stamp.seconds = Number(seconds);
+ else delete stamp.seconds;
+ if (Object.keys(stamp).length) out.stamp = stamp;
+ else delete out.stamp;
+ void save(Object.keys(out).length ? out : null);
+ };
+ return (
+ <div data-testid="factcheck-editor" className="space-y-1 rounded border border-[var(--color-line)] p-2 text-[11px]">
+ <div className="micro">fact-check labels</div>
+ <div className="grid grid-cols-[max-content_1fr_max-content] items-center gap-x-2 gap-y-1">
+ {doc.verdicts.map((k) => {
+ const def = doc.factcheckResolved.verdicts[k];
+ return (
+ <div key={k} className="contents">
+ <span className="font-mono text-[var(--color-dim)]">{k}</span>
+ <input data-testid={`fc-label-${k}`} placeholder={def?.label} value={v[k]?.label ?? ""} onChange={(e) => setV((x) => ({ ...x, [k]: { ...x[k], label: e.target.value } }))} className={input} />
+ <span className="flex items-center gap-1">
+ <input data-testid={`fc-color-${k}`} placeholder={def?.color} value={v[k]?.color ?? ""} onChange={(e) => setV((x) => ({ ...x, [k]: { ...x[k], color: e.target.value } }))} className={`${input} w-20 font-mono`} />
+ <span className="inline-block h-3 w-3 rounded" style={{ background: v[k]?.color || def?.color }} />
+ </span>
+ </div>
+ );
+ })}
+ </div>
+ <div className="flex items-center gap-2 text-[var(--color-dim)]">
+ <label>stamp seconds <input value={seconds} placeholder={String(doc.factcheckResolved.stamp.seconds)} onChange={(e) => setSeconds(e.target.value)} className={`${input} w-14`} /></label>
+ <button type="button" data-testid="fc-save" disabled={busy} onClick={submit} className={buttonVariants({ variant: "primary", size: "sm" })}>
+ Save labels
+ </button>
+ </div>
+ </div>
+ );
+}
diff --git a/umtool/components/projects/TakesBench.tsx b/umtool/components/projects/TakesBench.tsx
@@ -1,6 +1,9 @@
"use client";
-import { useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
+import AnchoredNotes from "@/components/notes/AnchoredNotes";
+import { NotesProvider, useSharedNotes } from "@/components/notes/NotesProvider";
+import TimedNotes from "@/components/notes/TimedNotes";
import { badgeVariants, type BadgeVariants } from "@/components/ui/badge";
import { buttonVariants } from "@/components/ui/button";
import { fmtAgo } from "@/lib/format";
@@ -128,6 +131,7 @@ export default function TakesBench({
};
return (
+ <NotesProvider target={{ project }}>
<div className="space-y-6">
{groups.map((g) => {
const tally = VERDICTS.map((v) => [v.id, g.takes.filter((t) => verdicts[t.id]?.verdict === v.id).length] as const)
@@ -166,6 +170,7 @@ export default function TakesBench({
{g.takes.map((t) => (
<TakeCard
key={t.id}
+ project={project}
take={t}
src={previewSrc(project, t)}
verdict={verdicts[t.id] ?? null}
@@ -194,10 +199,12 @@ export default function TakesBench({
);
})}
</div>
+ </NotesProvider>
);
}
function TakeCard({
+ project,
take: t,
src,
verdict,
@@ -209,6 +216,7 @@ function TakeCard({
onUnmute,
save,
}: {
+ project: string;
take: Take;
src: string;
verdict: TakeVerdict | null;
@@ -220,6 +228,16 @@ function TakeCard({
onUnmute: () => void;
save: (patch: { verdict?: Verdict | null; note?: string }) => Promise<void>;
}) {
+ const notes = useSharedNotes({ project });
+ // The element, for the timed notes; the parent's callback (audio routing)
+ // still gets it. Stable, so it runs once per mount, not once per render.
+ const [el, setEl] = useState<HTMLVideoElement | null>(null);
+ const parentRef = useRef(videoRef);
+ parentRef.current = videoRef;
+ const bindVideo = useCallback((node: HTMLVideoElement | null) => {
+ parentRef.current(node);
+ setEl(node);
+ }, []);
const [note, setNote] = useState(verdict?.note ?? "");
const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null);
@@ -254,7 +272,7 @@ function TakeCard({
>
{t.previewSize != null ? (
<video
- ref={videoRef}
+ ref={bindVideo}
src={src}
controls
preload="metadata"
@@ -327,6 +345,14 @@ function TakeCard({
/>
</div>
{error && <span className="text-[11px] text-[var(--color-bad)]">{error}</span>}
+ {/* The conversation about this take: notes on the take as a whole, and
+ notes at a second of its preview, resolved to the entry on screen.
+ Both land in the project's notes.json, which the agent that made
+ the takes reads back (`umtool notes`) beside verdicts.json. */}
+ <AnchoredNotes notes={notes} pin={{ kind: "take", take: t.id }} label="take notes" />
+ {t.previewSize != null && (
+ <TimedNotes notes={notes} file={`takes/${t.id}/${t.preview}`} video={el} take={t.id} resolveProject={project} />
+ )}
</div>
</article>
);
diff --git a/umtool/components/projects/TimelineEditor.tsx b/umtool/components/projects/TimelineEditor.tsx
@@ -0,0 +1,205 @@
+"use client";
+
+import { useRouter } from "next/navigation";
+import { createContext, useCallback, useContext, useEffect, useRef, useState } from "react";
+import AnchoredNotes from "@/components/notes/AnchoredNotes";
+import { announceNotesChanged, useSharedNotes, wroteNotes } from "@/components/notes/NotesProvider";
+import { buttonVariants } from "@/components/ui/button";
+import { MANIFEST_CHANGED, announceManifestChanged, refusalOf, timelineOp } from "./timelineApi";
+
+// Re-ordering the cut, on the project page.
+//
+// The rows stay the server's (each `<li data-entry data-at>` the page always
+// drew); this list adds what moves them. Drag a row by its handle, or focus it
+// and press alt+↑ / alt+↓; each row's menu duplicates it, removes it, or
+// inserts a clip after it (`<channel>/<video>@<start>-<end>`). Every one is a
+// structural write (POST /api/report/timeline): snapshotted first, so "Undo"
+// restores the cut as it was before the last burst of edits.
+
+type Ctx = {
+ project: string;
+ busy: boolean;
+ run: (op: string, args: Record<string, unknown>) => Promise<boolean>;
+ dragging: React.MutableRefObject<{ id: string; at: number } | null>;
+};
+const TimelineCtx = createContext<Ctx | null>(null);
+
+const rowOf = (el: EventTarget | null) => (el instanceof Element ? (el.closest("li[data-entry][data-at]") as HTMLLIElement | null) : null);
+const rowKey = (li: HTMLLIElement) => ({ id: li.dataset.entry ?? "", at: Number(li.dataset.at) });
+
+export function TimelineList({ project, count, children }: { project: string; count: number; children: React.ReactNode }) {
+ const router = useRouter();
+ // The manifest's write token, in a ref as well as state: a key pressed before
+ // the first read came back must wait for it, not send no token at all.
+ const tokenRef = useRef<string | null>(null);
+ const [busy, setBusy] = useState(false);
+ const [error, setError] = useState<string | null>(null);
+ const [over, setOver] = useState<number | null>(null);
+ const dragging = useRef<{ id: string; at: number } | null>(null);
+
+ const loadToken = useCallback(async () => {
+ const r = await fetch(`/api/report/timeline?project=${encodeURIComponent(project)}`, { cache: "no-store" });
+ const j = await r.json().catch(() => ({}));
+ if (r.ok) tokenRef.current = typeof j.token === "string" ? j.token : null;
+ return tokenRef.current;
+ }, [project]);
+ useEffect(() => {
+ void loadToken();
+ const on = (e: Event) => {
+ if ((e as CustomEvent).detail?.project === project) void loadToken();
+ };
+ window.addEventListener(MANIFEST_CHANGED, on);
+ return () => window.removeEventListener(MANIFEST_CHANGED, on);
+ }, [loadToken, project]);
+
+ const run = useCallback(
+ async (op: string, args: Record<string, unknown>) => {
+ setBusy(true);
+ setError(null);
+ const res = await timelineOp(project, tokenRef.current ?? (await loadToken()), op, args);
+ setBusy(false);
+ if (!res.ok) {
+ setError(refusalOf(res));
+ if (res.json.stale) await loadToken();
+ return false;
+ }
+ tokenRef.current = typeof res.json.token === "string" ? res.json.token : null;
+ announceManifestChanged(project);
+ if (wroteNotes(res.json)) announceNotesChanged(project);
+ router.refresh();
+ return true;
+ },
+ [project, loadToken, router],
+ );
+
+ const onKeyDown = (e: React.KeyboardEvent) => {
+ if (!e.altKey || (e.key !== "ArrowUp" && e.key !== "ArrowDown")) return;
+ if (e.target instanceof HTMLElement && /^(INPUT|TEXTAREA|SELECT)$/.test(e.target.tagName)) return;
+ const li = rowOf(e.target);
+ if (!li || busy) return;
+ const { id, at } = rowKey(li);
+ const to = at + (e.key === "ArrowUp" ? -1 : 1);
+ if (to < 0 || to >= count) return;
+ e.preventDefault();
+ void run("move", { id, at, toIndex: to }).then((ok) => {
+ // Keep the moved row focused after the page redraws it.
+ if (ok) setTimeout(() => (document.querySelector(`li[data-at="${to}"]`) as HTMLElement | null)?.focus(), 400);
+ });
+ };
+
+ return (
+ <TimelineCtx.Provider value={{ project, busy, run, dragging }}>
+ <div className="mb-1.5 flex flex-wrap items-center gap-2 text-[11px]">
+ <button type="button" data-testid="timeline-undo" disabled={busy} onClick={() => void run("undo", {})} className={buttonVariants({ size: "sm" })}>
+ Undo last edit
+ </button>
+ <span className="text-[var(--color-dim)]">
+ drag ⠿ or <kbd>alt</kbd>+<kbd>↑</kbd>/<kbd>↓</kbd> to move
+ </span>
+ {busy && <span className="text-[var(--color-dim)]">saving…</span>}
+ {error && (
+ <span data-testid="timeline-error" className="text-[var(--color-bad)]">
+ {error}
+ </span>
+ )}
+ </div>
+ <ul
+ className="space-y-1"
+ data-testid="timeline-list"
+ data-drop-at={over ?? ""}
+ onKeyDown={onKeyDown}
+ onDragOver={(e) => {
+ if (!dragging.current) return;
+ const li = rowOf(e.target);
+ if (!li) return;
+ e.preventDefault();
+ setOver(rowKey(li).at);
+ }}
+ onDragLeave={() => setOver(null)}
+ onDrop={(e) => {
+ const from = dragging.current;
+ const li = rowOf(e.target);
+ dragging.current = null;
+ setOver(null);
+ if (!from || !li) return;
+ e.preventDefault();
+ const to = rowKey(li).at;
+ if (to !== from.at) void run("move", { id: from.id, at: from.at, toIndex: to });
+ }}
+ >
+ {children}
+ </ul>
+ </TimelineCtx.Provider>
+ );
+}
+
+/** Inside one row: the drag handle and the row's menu. */
+export function RowControls({ id, at }: { id: string; at: number }) {
+ const ctx = useContext(TimelineCtx);
+ const [menu, setMenu] = useState(false);
+ const [insert, setInsert] = useState<string | null>(null);
+ if (!ctx) return null;
+ const { busy, run, dragging } = ctx;
+ return (
+ <>
+ <span
+ draggable={!busy}
+ data-drag-handle={id}
+ title="drag to move"
+ onDragStart={(e) => {
+ dragging.current = { id, at };
+ e.dataTransfer.effectAllowed = "move";
+ e.dataTransfer.setData("text/plain", id);
+ }}
+ onDragEnd={() => (dragging.current = null)}
+ className="cursor-grab select-none text-[13px] leading-none text-[var(--color-dim)]"
+ >
+ ⠿
+ </span>
+ <span className="relative">
+ <button type="button" data-row-menu={id} onClick={() => setMenu((m) => !m)} className="px-1 text-[12px] text-[var(--color-dim)] hover:text-[var(--color-text)]" aria-label={`actions for ${id}`}>
+ ⋯
+ </button>
+ {menu && (
+ <span className="absolute right-0 z-10 mt-1 flex w-36 flex-col rounded border border-[var(--color-line)] bg-[var(--color-panel)] p-1 text-[11px] shadow">
+ <button type="button" data-row-action="duplicate" disabled={busy} onClick={() => (setMenu(false), void run("duplicate", { id, at }))} className="px-1 py-0.5 text-left hover:bg-[var(--color-panel-2)]">
+ duplicate
+ </button>
+ <button type="button" data-row-action="insert" disabled={busy} onClick={() => (setMenu(false), setInsert(""))} className="px-1 py-0.5 text-left hover:bg-[var(--color-panel-2)]">
+ insert clip after
+ </button>
+ <button type="button" data-row-action="remove" disabled={busy} onClick={() => (setMenu(false), void run("remove", { id, at }))} className="px-1 py-0.5 text-left text-[var(--color-bad)] hover:bg-[var(--color-panel-2)]">
+ remove
+ </button>
+ </span>
+ )}
+ </span>
+ {insert !== null && (
+ <input
+ autoFocus
+ data-testid={`insert-after-${id}`}
+ value={insert}
+ placeholder="<channel>/<video>@<start>-<end>"
+ onChange={(e) => setInsert(e.target.value)}
+ onKeyDown={(e) => {
+ if (e.key === "Escape") setInsert(null);
+ if (e.key === "Enter" && insert.trim()) {
+ void run("insert", { afterId: id, at, entry: insert.trim() }).then((ok) => ok && setInsert(null));
+ }
+ }}
+ className="w-64 rounded border border-[var(--color-line)] bg-[var(--color-ink)] px-1.5 py-0.5 font-mono text-[11px] text-[var(--color-text)] outline-none focus:border-[var(--color-sel)]"
+ />
+ )}
+ </>
+ );
+}
+
+/** Under one row: its notes (and the edit notes a generated manifest left on it). */
+export function RowNotes({ id, project }: { id: string; project: string }) {
+ const notes = useSharedNotes({ project });
+ return (
+ <div className="mt-1">
+ <AnchoredNotes notes={notes} pin={{ kind: "entry", entry: id }} />
+ </div>
+ );
+}
diff --git a/umtool/components/projects/timelineApi.ts b/umtool/components/projects/timelineApi.ts
@@ -0,0 +1,39 @@
+"use client";
+
+// The client's side of POST /api/report/timeline, and the event every part of
+// a project page listens to after the manifest changed under it.
+//
+// The timeline editor, the structure editors and the On-screen section each
+// hold a manifest TOKEN; a structural write changes the file, so after one
+// every other holder must re-read before its next save would 409.
+
+export const MANIFEST_CHANGED = "umtool:manifest-changed";
+
+export function announceManifestChanged(project: string) {
+ window.dispatchEvent(new CustomEvent(MANIFEST_CHANGED, { detail: { project } }));
+}
+
+export type TimelineResult = {
+ ok: boolean;
+ status: number;
+ json: Record<string, unknown> & { error?: string; errors?: string[]; stale?: boolean; token?: string };
+};
+
+export async function timelineOp(project: string, token: string | null, op: string, args: Record<string, unknown> = {}): Promise<TimelineResult> {
+ const r = await fetch("/api/report/timeline", {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ project, token, op, ...args }),
+ cache: "no-store",
+ });
+ const json = (await r.json().catch(() => ({ error: `HTTP ${r.status}` }))) as TimelineResult["json"];
+ return { ok: r.ok, status: r.status, json };
+}
+
+/** A refusal in words: the build's sentences when there are several. */
+export const refusalOf = (res: TimelineResult) =>
+ res.json.stale
+ ? "the manifest changed since this page read it — reloaded; try again"
+ : res.json.errors?.length
+ ? res.json.errors.join("; ")
+ : String(res.json.error ?? `HTTP ${res.status}`);
diff --git a/umtool/docs/README.md b/umtool/docs/README.md
@@ -93,6 +93,8 @@ defects that have already shipped in real videos: a manifest with no `siteOrigin
| [mix-from-a-project.md](mix-from-a-project.md) | Deep-linking a clip into `/mix` |
| [index.md](index.md) | The LMDB index, and why the filesystem stays the model |
| [cli.md](cli.md) | `umtool ls / show / check / build / window / …` |
+| [notes.md](notes.md) | Operator notes on articles and videos; `umtool notes`, the agent loop |
+| [sites.md](sites.md) | `/sites`: every site's articles, their media, workspaces and notes |
| [authoring.md](authoring.md) | Writing a manifest from a sweep report |
| [e2e.md](e2e.md) | The fixture, the stubs, the global queue |
| [quirks.md](quirks.md) | Everything that cost time to find out |
diff --git a/umtool/docs/notes.md b/umtool/docs/notes.md
@@ -0,0 +1,112 @@
+# Notes: the operator writes them in umtool, an agent acts on them
+
+A note is a sentence or two the operator leaves on an **article** (a report on a
+site, `/sites/<site>/<report>`) or on a **report-video project** (a timeline row, a
+take, a moment in a rendered cut). It is saved where the agent that made the thing
+can read it back, act on the right SOURCE file, and reply or resolve.
+
+```sh
+umtool notes --all --open # every notes file with an open note
+umtool notes candalyzer/polemic-israel # one article's open notes, as markdown
+umtool notes candace/polemic-israel # one video project's (plus every take's verdict)
+umtool notes reply n_mgk3x0a1b2 "Changed 'said' to 'claimed' in drafts/israel.json" --resolve
+umtool notes resolve | wontfix | reopen n_mgk3x0a1b2
+```
+
+Run it from the repo checkout (or with `TRANSCRIPTS_DIR`/`SITES_DIR` and
+`REPORTS_DIR` set): like `CHANNELS_DIR`, the sites directory is found by walking up
+from the working directory to the checkout.
+
+## The agent loop
+
+1. `umtool notes --all --open` — what is waiting.
+2. `umtool notes <target>` — each open note with its anchor resolved against what
+ is on disk now, and the **Source** line naming the file to edit.
+3. Edit the SOURCE, not the output. An article's `report.json` is regenerated from
+ a draft (`<workspace>/polemics/drafts/<slug>.json`) by a generator
+ (`make-site.py`, `polemics.py`); a generated video manifest (`generatedBy`) is
+ rewritten by its `make-videos.py`. An edit to the output is lost on the next run.
+4. Regenerate.
+5. `umtool notes reply <id> "<what changed>" --resolve` — or reply without
+ `--resolve` to ask a question. The operator sees agent replies badged "agent".
+
+**Never hand-edit `notes.json`.** The CLI and the app write through one store that
+validates, takes a lock, and refuses a stale write; a hand edit races the page and
+can lose a reply. If the recorded source is wrong, correct it:
+`umtool notes source <target> --draft <path> [--generator <path>] [--how <why>]`.
+
+The same digest is served as text at `GET /api/notes/context?article=<site>/<report>`
+(or `?project=<id>`); the page's "Copy agent brief" copies it.
+
+## Where notes live
+
+| Subject | File |
+|---|---|
+| An article | `transcripts/sites/<site>/reports/<report>/notes.json`, beside `report.json` |
+| A report-video project | `<project>/notes.json`, beside `video.manifest.json` |
+
+The article file is the one corpus file umtool writes, and only through
+`isCorpusNotesFile` (`lib/paths.mjs`): exactly `sites/<site>/reports/<id>/notes.json`,
+in an existing report directory that is not a symlink out of its site. The
+generators overwrite only `report.json`, `video.mp4` and `poster.jpg` in a report
+directory, so notes survive a regenerate; compose and the report history never read
+it (`common/publish/composeReports.test.ts` holds that), so a note is never published.
+
+## The file
+
+```jsonc
+{ "format": "umtool-notes", "version": 1,
+ "subject": { "kind": "article", "site": "candalyzer", "report": "polemic-israel" },
+ // | { "kind": "video-project", "project": "candace/polemic-israel" }
+ "source": { "draft": "~/reports/candace/polemics/drafts/israel.json",
+ "generator": "~/reports/candace/site/polemics.py",
+ "how": "draft matched by id polemic-israel; generator names candalyzer" },
+ "notes": [ { "id": "n_…", "status": "open", // open | resolved | wontfix
+ "author": "operator", "text": "…", // operator | agent
+ "at": "…", "updatedAt": "…",
+ "anchor": { … },
+ "replies": [ { "author": "agent", "text": "…", "at": "…" } ],
+ "resolvedAt": "…", "resolvedBy": "agent" } ] }
+```
+
+`source` is filled on the first write. For an article it comes from
+`lib/articles/sources.mjs`: a draft whose `id` or file name is the report id (allowing
+a `polemic-` prefix either way; the workspace whose generator names the site wins a
+tie), and the generator under `<workspace>/polemics` or `<workspace>/site` that names
+the site or the report. For a video project it is the manifest and, when it has
+`generatedBy`, the generator.
+
+The last note deleted deletes the file. A `notes.json` that does not parse is never
+overwritten — fix it by hand first.
+
+## Anchors
+
+| `kind` | Fields | Resolved for the agent as |
+|---|---|---|
+| `text` | `section` (a section id, or `title`/`subtitle`/`summary`/`method`), `quote`, `prefix`, `suffix` | the section title and the sentence holding the quote |
+| `cite` | `cite` (a citation id) | the citation's quote, speaker, date, `<channel>/<id>@start-end` |
+| `section` | `section` | the section title |
+| `whole` | — | the whole article or project |
+| `moment` | `file` (relative mp4), `t`, `take?`, `entry?`, `resolved?` | the time, and the entry, quote and source second it resolved to when written |
+| `entry` | `entry` (a timeline entry id) | the entry's title, quote and source span |
+| `take` | `take` (a take id) | the take's label and summary, and its verdict |
+| `edit` | `entry?`, `field`, `from`, `to` | an edit made in umtool to a GENERATED manifest — port it into the generator's inputs |
+
+A text anchor is a quote with 32 characters of context either side (the W3C
+TextQuoteSelector), never an offset, so it survives a regenerated `report.json`:
+`lib/annotations/anchor.mjs` finds the quote exactly, then with whitespace, quotes
+and case normalised, then — when the quote itself was rewritten — between its
+surviving context. A note it cannot place is **orphaned**: still shown, pinned to its
+section, with its original quote.
+
+## Code
+
+| Module | What |
+|---|---|
+| `lib/annotations/shape.mjs` | the contract: constants, validation, the ops (pure, client-safe) |
+| `lib/annotations/anchor.mjs` | re-anchoring (pure, client-safe) |
+| `lib/annotations/store.mjs` | read, lockfile (`notes.json.lock`, stale after 30 s), token, tmp+rename |
+| `lib/annotations/targets.mjs` | article / project → file, subject, source; `listNotesFiles` |
+| `lib/annotations/digest.mjs`, `cli.mjs` | the markdown digest and `umtool notes` |
+| `app/api/notes/route.ts` | GET / POST (operator-stamped; 409 on a stale token) |
+| `lib/annotations/useNotes.ts` | the page's hook |
diff --git a/umtool/docs/report-video.md b/umtool/docs/report-video.md
@@ -711,6 +711,66 @@ note empty is removed. A `verdicts.json` that does not parse is never
overwritten. The rules are `lib/report/takes.mjs`; the routes are
`/api/report/takes` (list), `/takes/preview` (ranges) and `/takes/verdict`.
+Each take also takes notes (a `take` anchor) and timed notes on its preview;
+the verdicts and those notes reach the agent through `umtool notes <project>`,
+which lists every take with its verdict and note ([notes.md](notes.md)).
+
+## Operator notes, timed notes and the generated-manifest guard
+
+Notes on a project live in `<project>/notes.json` (shape and CLI:
+[notes.md](notes.md)): **row notes** on a timeline entry (`entry` anchor, an open
+count per row on the project page and in the clip bench), **take notes**, **timed
+notes** and **edit notes**.
+
+**Timed notes.** On the built cut (On-screen → the built video), on every take
+preview and on an article's report video: `n` with the video focused, or Mark,
+writes a note at the playhead, and the tick strip under the player seeks. On a
+project the second is resolved against the build's `schedule.json` (the
+deliverable `out/<slug>.mp4` against `out/sourced/`, a variant's file against its
+own; a take's preview against `takes/<id>/out/<variant>/`) to the entry on screen:
+its title, quote, source second and archive link, stored in the note as
+`resolved`. A take's `preview.mp4` is that build's output copied, and its length
+equals the schedule's `total` exactly (all 9 takes of candace/polemic-israel), so
+a mark is exact when the lengths match within 0.25 s and `approx` when they
+differ, when the schedule is estimated, or when the second is a held frame past
+the clip's source. `lib/report/moments.mjs`, `GET /api/report/moment`.
+
+**Generated manifests.** A manifest with `generatedBy` shows one line on the
+project page, the clip bench and the On-screen section: "Generated by
+`<generatedBy>`; a rebuild of manifests overwrites edits made here." Edits are
+still allowed. Every manifest writer ROUTE goes through `withEditNotes`
+(`lib/report/guard.ts`), which diffs the manifest before and after and records
+each change as an `edit` note `{ entry?, field, from, to }`: repeated saves of one
+field coalesce into one note, and putting a field back deletes its note unless it
+has replies. The agent ports each change into the generator's inputs (BEATS,
+drafts), regenerates and resolves the note. `umtool window` is the agent's own
+tool and is not wrapped.
+
+## Structural edits
+
+The project page's timeline re-orders by drag or alt+↑/↓; each row's ⋯ menu
+duplicates it, removes it, or inserts a clip after it
+(`<channel>/<video>@<start>-<end>`). The On-screen section edits a teaser's lines,
+beat, tail, tail wait and dip; the posts (add, edit, remove); and each verdict's
+label and colour and the stamp seconds. An article's citation can be added to the
+end of a linked project's timeline ("Add to video", [sites.md](sites.md)).
+
+All of it is `POST /api/report/timeline` (`move`, `remove`, `duplicate`, `insert`,
+`teaser`, `post`, `post-remove`, `factcheck`, `undo`; `GET` gives the token), with
+the writers in `lib/report/manifest.mjs` beside the window writers: the mtime
+token, tmp + rename, 2 dp. Each op is checked by the build's own validators
+(`deck.mjs`, `factcheck.mjs`) and refused only for what it breaks, and snapshots
+the manifest first (`revisions/<stamp>-auto-before-<op>.manifest.json`, at most
+one per op every two minutes). **Undo** restores the newest such snapshot byte for
+byte (the current state is kept as `undo-saved`).
+
+A move recomputes `sectionEnter` — a clip enters when its `section` differs from
+the previous clip's, the first clip included (`lib/report/sections.mjs`) — and only
+in a manifest that already carries the flag; `section` itself never changes.
+report-to-video only READS the flag. Checked read-only on all 44 manifests under
+~/reports: no mismatch with the stored flags, and 3,490 move-and-back round trips
+byte-identical.
+
## Exports
`umtool export <project> --format toc-bbcode|toc-markdown|description|chapters`
diff --git a/umtool/docs/sites.md b/umtool/docs/sites.md
@@ -0,0 +1,92 @@
+# /sites — every site's articles
+
+Every report on every site under `transcripts/sites/` — published and drafts —
+read in place: no private site built, no port served. umtool **reads** the sites;
+the one file it writes there is a report's `notes.json` (docs/notes.md).
+
+| page | what |
+|---|---|
+| `/sites` | every site (private first), every article: status, updated, citations, open notes, poster, video project, source draft. Filters are links: `?site=`, `?status=published\|draft`, `?notes=open` |
+| `/sites/<site>` | its articles, its report videos (playable), the umtool projects they were cut in (takes, like/maybe/no), the workspace files they were written from (`?ws=&rel=` opens one) |
+| `/sites/<site>/<report>` | the article with its notes (below) |
+| `/sites/<site>/<report>?tab=source` | the article's workspace files, its own draft opened |
+| `/sites/<site>/<report>/evidence` | every citation, one per screen (`?c=<id>`) |
+
+## Where an article comes from
+
+- **Site and reports**: common's `listSites`/`getSite` (site.json, the editor's
+ defaults), `listReportDirs` (the enumerator the editor's Reports tab uses), each
+ `report.json` through the report validator. **Published** = named in site.json
+ `reports`, in that order; every other report directory is a **draft**.
+- **Source** (`lib/articles/sources.mjs`): the draft under
+ `~/reports/<ws>/polemics/drafts/` whose `id` or file name is the report id,
+ `polemic-` allowed on either side, preferring a workspace whose generator names
+ the site; the generator is the `*.py`/`*.mts` under `<ws>/polemics` or
+ `<ws>/site` that names the site or the report. Shown with its reason; written
+ into `notes.json` `source` on the first note. **Edit the draft, not
+ report.json** — the generator overwrites report.json.
+- **Video project** (`lib/articles/links.mjs`): a report-video manifest with a
+ top-level `"article": "<site>/<report>"` is linked to that article and nothing
+ else (build-video.mjs ignores the key). Otherwise a manifest `slug` equal to the
+ report id (± `polemic-`) in the draft's workspace, when exactly one matches;
+ several are listed as "possible".
+
+## Reading and noting
+
+The article renders in a ~70ch column with its notes in a rail (a drawer below
+1100px). Its video sits under the title (`components/articles/ArticleVideo.tsx`)
+and takes timed notes: `n` with the video focused, or **Mark**, notes the
+playhead as a `moment` anchor on the article's notes ([report-video.md](report-video.md)).
+
+| to note | do |
+|---|---|
+| a passage | select it → **Note** (or `n`) |
+| a section | **+ note** on its heading |
+| a citation | click it → the evidence panel → **+ note on this citation** |
+| the article | **+ whole article** |
+
+A passage note keeps its quote and 32 characters either side; it is found again
+in the text as it is now (`lib/annotations/anchor.mjs`: exact, then
+whitespace/quote/case-normalised, then by its surviving context) and marked.
+One whose quote is gone is listed **orphaned**. Keys, when not typing: `j`/`k`
+move, `r` resolves, `e` edits, `n` notes, `Esc` closes. `?status=` and `?note=`
+open the rail on a filter or a note (the decisions inbox links to the note).
+**Copy agent brief** copies `/api/notes/context` — the `umtool notes` digest.
+
+Open notes are `open-note` rows in `/browse/decisions` and on the dashboard.
+
+## Evidence
+
+A citation's panel: its quote, speaker, date, label and original link; ±6 cues
+of its record's `transcript.cues.json`, the cited ones marked (click one to seek);
+and media, best first:
+
+1. the site's **prepared** clip (`.export-index/sites/<site>/report-media/`);
+2. a clip **window** the editor fetched (`data/<id>/clips/`);
+3. a **saved** video or audio file in `data/<id>/` (through its media-tier link);
+4. otherwise the `fetch_clip` MCP line to fetch it through the editor. umtool
+ never runs yt-dlp, and its own fetch client needs a project manifest.
+
+A post shows its text and its capture screenshot. The walk
+(`/evidence`) is the same panel one citation at a time: `j`/`k` (or ←/→),
+`space` plays, `n` notes.
+
+**Add to video** (in the panel, when the article has a linked video project)
+puts the cited span at the end of that project's timeline as a clip
+(`/api/report/timeline` insert): the manifest is snapshotted first, so the
+project page's **Undo** takes it back, and on a generated manifest an `edit` note
+tells the agent to port it into the generator's inputs.
+
+## Routes (read-only)
+
+| route | serves |
+|---|---|
+| `/api/sites/media?site&report&file` | a file in the report dir (video.mp4, poster.jpg, stills/…), byte ranges |
+| `/api/sites/media?site&moment` | the prepared clip of a moment, via report-media/index.json |
+| `/api/sites/media?corpus=<abs>` | a media file lexically under CHANNELS_DIR |
+| `/api/sites/evidence?site&report&cite` | one citation's evidence (JSON) |
+| `/api/sites/workspace?ws&rel` | a listed workspace file; HTML under a CSP sandbox |
+
+`SITES_DIR` (else `TRANSCRIPTS_DIR/sites`, else the checkout's
+`transcripts/sites`) is where all of it is read; the e2e suite points it at its
+fixture (`e2e/fixtures/sites-fixture.mjs`).
diff --git a/umtool/e2e/article-evidence.spec.ts b/umtool/e2e/article-evidence.spec.ts
@@ -0,0 +1,71 @@
+import { test, expect } from "@playwright/test";
+import { rmSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { WARM_TIMEOUT, warm } from "./warm";
+
+// ---------------------------------------------------------------------------
+// A citation's evidence: the transcript around it, the window that plays it,
+// or -- with nothing on disk -- the fetch_clip line; and the one-per-screen
+// walk.
+//
+// priv/polemic-alpha c1 → sitechan/sv1, a clip window on disk
+// c2 → sitechan/sv2, cues only
+// ---------------------------------------------------------------------------
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const NOTES = path.join(HERE, "..", ".e2e-song", "sites", "priv", "reports", "polemic-alpha", "notes.json");
+
+
+test.beforeAll(async ({ playwright }) => {
+ test.setTimeout(WARM_TIMEOUT);
+ await warm(playwright, ["/sites/priv/polemic-alpha", "/sites/priv/polemic-alpha/evidence", "/api/sites/evidence?site=priv&report=polemic-alpha&cite=c1", "/api/notes?article=priv/polemic-alpha"]);
+});
+
+test.beforeEach(() => rmSync(NOTES, { force: true }));
+test.afterAll(() => rmSync(NOTES, { force: true }));
+
+test("a citation opens its evidence: the cited cue marked, the window playing from the span", async ({ page }) => {
+ await page.goto("/sites/priv/polemic-alpha");
+ await page.locator("button[data-cite='c1']").first().click();
+ const ev = page.locator("[data-evidence='c1']");
+ await expect(ev).toContainText("The vote was rigged and everybody knew.");
+ await expect(ev.locator("[data-play]")).toHaveAttribute("data-play", "window");
+ await expect(ev.locator("[data-cited='true']")).toHaveCount(1);
+ await expect(ev.locator("[data-cited='true']")).toContainText("The vote was rigged");
+ await expect(ev.locator("[data-cues] li")).toHaveCount(6);
+ const video = ev.locator("video");
+ await expect(video).toHaveAttribute("src", /corpus=.*sv1.*clips/);
+ await expect.poll(() => video.evaluate((v: HTMLVideoElement) => v.currentTime)).toBeGreaterThanOrEqual(0);
+});
+
+test("with nothing on disk it says so and offers the fetch_clip line, never yt-dlp", async ({ page }) => {
+ await page.goto("/sites/priv/polemic-alpha");
+ await page.locator("button[data-cite='c2']").first().click();
+ const ev = page.locator("[data-evidence='c2']");
+ await expect(ev.locator("[data-play]")).toHaveAttribute("data-play", "none");
+ await expect(ev).toContainText('fetch_clip {"channel":"sitechan","video":"sv2","start":8,"end":12');
+ await expect(ev).not.toContainText("yt-dlp");
+});
+
+test("the walk: one citation per screen, j/k, n notes it, the URL follows", async ({ page }) => {
+ await page.setViewportSize({ width: 1366, height: 768 });
+ await page.goto("/sites/priv/polemic-alpha/evidence");
+ await expect(page.locator("[data-walk='c1']")).toBeVisible();
+ await expect(page.locator("[data-walk-position]")).toHaveText("1 / 2");
+ await page.keyboard.press("j");
+ await expect(page.locator("[data-walk='c2']")).toBeVisible();
+ await expect(page).toHaveURL(/\?c=c2$/);
+ await page.keyboard.press("n");
+ await page.getByLabel("note text").fill("Needs a clip.");
+ await page.keyboard.press("Control+Enter");
+ await expect(page.locator("[data-walk='c2'] [data-note-card]")).toHaveCount(1);
+ await page.keyboard.press("k");
+ await expect(page.locator("[data-walk='c1'] [data-note-card]")).toHaveCount(0);
+
+ // It fits the laptop screen: no horizontal scroll.
+ expect(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth)).toBe(true);
+
+ await page.goto("/sites/priv/polemic-alpha/evidence?c=c2");
+ await expect(page.locator("[data-walk='c2'] [data-note-card]")).toContainText("Needs a clip.");
+});
diff --git a/umtool/e2e/article-integration.spec.ts b/umtool/e2e/article-integration.spec.ts
@@ -0,0 +1,106 @@
+import { test, expect, type Locator } from "@playwright/test";
+import { copyFileSync, existsSync, readFileSync, readdirSync, rmSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { WARM_TIMEOUT, warm } from "./warm";
+
+// ---------------------------------------------------------------------------
+// Where the two tracks meet on the article page:
+//
+// * the report's own video carries timed notes, written to the ARTICLE's
+// notes through the reader's one handle (a mark and a text note do not
+// 409 each other);
+// * "Add to video" puts a cited span at the end of the linked project's
+// timeline; the project is GENERATED, so an `edit` note is left for the
+// agent, and the change is undoable from the auto snapshot.
+//
+// priv/polemic-alpha video.mp4 (4 s); c1 → sitechan/sv1@3-6
+// sitews/polemic-alpha the linked project (generatedBy set), one clip a1
+// ---------------------------------------------------------------------------
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const FIX = path.join(HERE, "..", ".e2e-song");
+const NOTES = path.join(FIX, "sites", "priv", "reports", "polemic-alpha", "notes.json");
+const PROJ = path.join(FIX, "sitews", "polemic-alpha");
+const MANIFEST = path.join(PROJ, "video.manifest.json");
+const SAVED = path.join(PROJ, ".manifest.e2e-integration");
+const PROJ_NOTES = path.join(PROJ, "notes.json");
+
+const readJson = (f: string) => JSON.parse(readFileSync(f, "utf8"));
+
+async function seek(video: Locator, t: number) {
+ await video.evaluate(async (el: HTMLVideoElement, at: number) => {
+ if (el.readyState < 1) await new Promise((r) => el.addEventListener("loadedmetadata", r, { once: true }));
+ el.currentTime = at;
+ await new Promise((r) => el.addEventListener("seeked", r, { once: true }));
+ }, t);
+}
+
+function restoreProject() {
+ if (existsSync(SAVED)) {
+ copyFileSync(SAVED, MANIFEST);
+ rmSync(SAVED, { force: true });
+ }
+ rmSync(PROJ_NOTES, { force: true });
+ rmSync(`${MANIFEST}.bak`, { force: true });
+ const rev = path.join(PROJ, "revisions");
+ if (existsSync(rev)) for (const f of readdirSync(rev)) if (f.includes("auto-before")) rmSync(path.join(rev, f), { recursive: true, force: true });
+}
+
+test.beforeAll(async ({ playwright }) => {
+ test.setTimeout(WARM_TIMEOUT);
+ await warm(playwright, ["/sites/priv/polemic-alpha", "/api/notes?article=priv/polemic-alpha", "/api/report/timeline?project=sitews/polemic-alpha"]);
+});
+test.beforeEach(() => {
+ rmSync(NOTES, { force: true });
+ copyFileSync(MANIFEST, SAVED);
+});
+test.afterEach(() => {
+ rmSync(NOTES, { force: true });
+ restoreProject();
+});
+
+test("the article's video takes timed notes, on the article's notes, beside a text note", async ({ page }) => {
+ await page.goto("/sites/priv/polemic-alpha");
+ const video = page.getByTestId("article-video");
+ await expect(video).toBeVisible();
+ const scope = page.locator("[data-timed-notes='video.mp4']");
+ await seek(video, 1);
+ await scope.locator("[data-action='mark']").click();
+ const input = scope.getByTestId("timed-note-input");
+ await input.fill("the cut is late here");
+ await input.press("Enter");
+ await expect(scope.locator("[data-mark-at]")).toHaveCount(1);
+
+ // The rail's handle is the same one: a whole-article note right after does not 409.
+ await page.getByRole("button", { name: "+ whole article" }).click();
+ await page.getByLabel("note text").fill("Retitle it.");
+ await page.getByRole("button", { name: "save note" }).click();
+ await expect.poll(() => (existsSync(NOTES) ? readJson(NOTES).notes.length : 0)).toBe(2);
+ const doc = readJson(NOTES);
+ expect(doc.subject).toEqual({ kind: "article", site: "priv", report: "polemic-alpha" });
+ const moment = doc.notes.find((n: { anchor: { kind: string } }) => n.anchor.kind === "moment");
+ expect(moment.anchor).toMatchObject({ kind: "moment", file: "video.mp4", t: 1 });
+ expect(moment.author).toBe("operator");
+});
+
+test("Add to video: the cited span lands at the end of the linked project, with an edit note; undo restores it", async ({ page, request }) => {
+ const before = readJson(MANIFEST).timeline.length;
+ await page.goto("/sites/priv/polemic-alpha");
+ await page.locator("button[data-cite='c1']").first().click();
+ const add = page.locator("[data-evidence='c1'] [data-add-to-video]");
+ await expect(add).toContainText("sitews/polemic-alpha");
+ await add.getByRole("button", { name: "Add to video" }).click();
+ await expect(add.getByRole("status")).toContainText("cite-c1");
+
+ const m = readJson(MANIFEST);
+ expect(m.timeline.length).toBe(before + 1);
+ expect(m.timeline.at(-1)).toMatchObject({ type: "clip", id: "cite-c1", channel: "sitechan", video: "sv1", start: 3, end: 6 });
+ const notes = readJson(PROJ_NOTES).notes;
+ expect(notes.some((n: { anchor: { kind: string } }) => n.anchor.kind === "edit")).toBe(true);
+
+ const g = await (await request.get("/api/report/timeline?project=sitews/polemic-alpha")).json();
+ const undo = await request.post("/api/report/timeline", { data: { project: "sitews/polemic-alpha", token: g.token, op: "undo" } });
+ expect(undo.ok()).toBe(true);
+ expect(readJson(MANIFEST).timeline.length).toBe(before);
+});
diff --git a/umtool/e2e/article-notes.spec.ts b/umtool/e2e/article-notes.spec.ts
@@ -0,0 +1,186 @@
+import { test, expect, type Page } from "@playwright/test";
+import { existsSync, readFileSync, rmSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { WARM_TIMEOUT, warm } from "./warm";
+
+// ---------------------------------------------------------------------------
+// Notes on an article: written beside report.json (the one corpus file umtool
+// writes), anchored to a quote and found again, resolved, deleted -- and the
+// last delete removes the file.
+//
+// priv/polemic-alpha WRITES sites/priv/reports/polemic-alpha/notes.json
+// ---------------------------------------------------------------------------
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const NOTES = path.join(HERE, "..", ".e2e-song", "sites", "priv", "reports", "polemic-alpha", "notes.json");
+const PAGE = "/sites/priv/polemic-alpha";
+
+
+test.beforeAll(async ({ playwright }) => {
+ test.setTimeout(WARM_TIMEOUT);
+ await warm(playwright, ["/sites/priv/polemic-alpha", "/api/notes?article=priv/polemic-alpha", "/sites", "/browse/decisions?kind=open-note", "/api/sites/evidence?site=priv&report=polemic-alpha&cite=c1"]);
+});
+
+test.beforeEach(() => rmSync(NOTES, { force: true }));
+test.afterAll(() => rmSync(NOTES, { force: true }));
+
+async function select(page: Page, block: string, text: string) {
+ await page.evaluate(
+ ([block, text]) => {
+ const root = document.querySelector(`[data-block="${block}"]`)!;
+ const w = document.createTreeWalker(root, NodeFilter.SHOW_TEXT);
+ for (let n = w.nextNode() as Text | null; n; n = w.nextNode() as Text | null) {
+ const i = n.data.indexOf(text);
+ if (i < 0) continue;
+ const r = document.createRange();
+ r.setStart(n, i);
+ r.setEnd(n, i + text.length);
+ const s = getSelection()!;
+ s.removeAllRanges();
+ s.addRange(r);
+ return;
+ }
+ throw new Error(`no "${text}" in ${block}`);
+ },
+ [block, text],
+ );
+ await page.locator(`[data-block="${block}"]`).dispatchEvent("mouseup");
+}
+
+test("select text, Note, save: a mark on the quote, a note beside report.json", async ({ page }) => {
+ await page.goto(PAGE);
+ await select(page, "first", "Nobody checked the claim");
+ await page.getByRole("button", { name: "Note", exact: true }).click();
+ await page.getByLabel("note text").fill("Source this sentence.");
+ await page.getByRole("button", { name: "save note" }).click();
+
+ await expect(page.locator("mark[data-note]")).toHaveText("Nobody checked the claim");
+ const doc = JSON.parse(readFileSync(NOTES, "utf8"));
+ expect(doc.subject).toEqual({ kind: "article", site: "priv", report: "polemic-alpha" });
+ expect(doc.source.draft).toMatch(/sitews\/polemics\/drafts\/alpha\.json$/);
+ expect(doc.notes[0]).toMatchObject({ status: "open", author: "operator", text: "Source this sentence." });
+ expect(doc.notes[0].anchor).toMatchObject({ kind: "text", section: "first", quote: "Nobody checked the claim" });
+ expect(doc.notes[0].anchor.prefix).toContain("first stream.");
+
+ // Found again after a reload, and hovering the card lights the mark.
+ await page.reload();
+ const card = page.locator("[data-note-card]");
+ await expect(card).toHaveCount(1);
+ await card.hover();
+ await expect(page.locator("mark[data-note]")).toHaveAttribute("data-active", "true");
+});
+
+test("section, whole-article and citation notes; resolve, reopen, reply, filters", async ({ page }) => {
+ await page.goto(PAGE);
+ await page.getByRole("button", { name: "note on Later" }).click();
+ await page.getByLabel("note text").fill("Section note.");
+ await page.keyboard.press("Control+Enter");
+ await expect(page.locator("[data-note-card]")).toHaveCount(1);
+
+ await page.getByRole("button", { name: "+ whole article" }).click();
+ await page.getByLabel("note text").fill("Whole note.");
+ await page.getByRole("button", { name: "save note" }).click();
+ await expect(page.locator("[data-note-card]")).toHaveCount(2);
+
+ await page.locator("button[data-cite='c1']").first().click();
+ await expect(page.locator("[data-evidence='c1']")).toBeVisible();
+ await page.getByRole("button", { name: "+ note on this citation" }).click();
+ await page.getByLabel("note text").fill("Right second?");
+ await page.getByRole("button", { name: "save note" }).click();
+ await expect(page.locator("[data-evidence-rail] [data-note-card]")).toHaveCount(1);
+ await expect(page.locator("button[data-cite='c1'][data-has-note='true']").first()).toBeVisible();
+
+ const notes = JSON.parse(readFileSync(NOTES, "utf8")).notes;
+ expect(notes.map((n: { anchor: { kind: string } }) => n.anchor.kind).sort()).toEqual(["cite", "section", "whole"]);
+
+ // resolve the whole-article note; the open filter drops it
+ const whole = page.locator("[data-note-card]", { hasText: "Whole note." });
+ await whole.getByRole("button", { name: "resolve" }).click();
+ await expect(page.locator("[data-note-card]", { hasText: "Whole note." })).toHaveCount(0);
+ await page.getByRole("button", { name: /^resolved 1$/ }).click();
+ const resolved = page.locator("[data-note-card]", { hasText: "Whole note." });
+ await expect(resolved).toHaveAttribute("data-status", "resolved");
+ await resolved.getByRole("button", { name: "reopen" }).click();
+ await page.getByRole("button", { name: /^all 3$/ }).click();
+ const reopened = page.locator("[data-note-card]", { hasText: "Whole note." });
+ await expect(reopened).toHaveAttribute("data-status", "open");
+ await reopened.getByRole("button", { name: "reply" }).click();
+ await page.getByLabel("note text").fill("A reply.");
+ await reopened.getByRole("button", { name: "reply", exact: true }).click();
+ await expect(reopened.locator("[data-reply='operator']")).toContainText("A reply.");
+});
+
+test("keys: j selects, r resolves, n notes the selection", async ({ page }) => {
+ await page.goto(PAGE);
+ await select(page, "later", "on a different show");
+ await expect(page.getByRole("button", { name: "Note", exact: true })).toBeVisible();
+ await page.keyboard.press("n");
+ await page.getByLabel("note text").fill("Which show?");
+ await page.keyboard.press("Control+Enter");
+ await expect(page.locator("mark[data-note]")).toHaveText("on a different show");
+ await page.locator("main").click({ position: { x: 5, y: 5 } });
+ await page.keyboard.press("j");
+ await expect(page.locator("[data-note-card]")).toHaveAttribute("aria-current", "true");
+ await page.keyboard.press("r");
+ await expect(page.locator("[data-note-card]")).toHaveCount(0);
+ expect(JSON.parse(readFileSync(NOTES, "utf8")).notes[0].status).toBe("resolved");
+});
+
+test("a quote that is gone is still listed, flagged orphaned", async ({ page, request }) => {
+ const res = await request.post("/api/notes?article=priv/polemic-alpha", {
+ data: { op: { op: "add", text: "About a sentence the agent rewrote.", anchor: { kind: "text", section: "first", quote: "a sentence that is not in the article", prefix: "", suffix: "" } } },
+ });
+ expect(res.status()).toBe(200);
+ await page.goto(PAGE);
+ await expect(page.locator("[data-note-card][data-orphaned='true']")).toHaveCount(1);
+ await expect(page.locator("mark[data-note]")).toHaveCount(0);
+});
+
+test("a stale token is a 409 and the page re-reads; the last delete removes notes.json", async ({ page, request }) => {
+ await page.goto(PAGE);
+ await page.getByRole("button", { name: "+ whole article" }).click();
+ await page.getByLabel("note text").fill("Mine.");
+ await page.getByRole("button", { name: "save note" }).click();
+ await expect(page.locator("[data-note-card]")).toHaveCount(1);
+
+ // An agent writes in between (no token: the CLI's way).
+ const id = JSON.parse(readFileSync(NOTES, "utf8")).notes[0].id;
+ const stale = await request.post("/api/notes?article=priv/polemic-alpha", { data: { token: "absent", op: { op: "status", id, status: "resolved" } } });
+ expect(stale.status()).toBe(409);
+ await request.post("/api/notes?article=priv/polemic-alpha", { data: { op: { op: "reply", id, text: "seen" } } });
+
+ await page.locator("[data-note-card]").getByRole("button", { name: "resolve" }).click();
+ await expect(page.getByText(/reloaded; check and try again/)).toBeVisible();
+ await expect(page.locator("[data-note-card] [data-reply]")).toContainText("seen");
+
+ page.on("dialog", (d) => d.accept());
+ await page.locator("[data-note-card]").getByRole("button", { name: "delete" }).click();
+ await expect(page.locator("[data-note-card]")).toHaveCount(0);
+ expect(existsSync(NOTES)).toBe(false);
+});
+
+test("open notes are open decisions, linked to the note; ?note= opens on it", async ({ page, request }) => {
+ await request.post("/api/notes?article=priv/polemic-alpha", { data: { op: { op: "add", text: "For the inbox.", anchor: { kind: "section", section: "later" } } } });
+ const id = JSON.parse(readFileSync(NOTES, "utf8")).notes[0].id;
+
+ await page.goto("/sites?notes=open");
+ await expect(page.locator("[data-article='priv/polemic-alpha'] [data-open-notes]")).toHaveAttribute("data-open-notes", "1");
+
+ await page.goto("/browse/decisions?kind=open-note");
+ const row = page.locator("[data-project='sites/priv/polemic-alpha'] [data-decision='open-note']");
+ await expect(row).toContainText("For the inbox.");
+ await row.getByRole("link").click();
+ await expect(page).toHaveURL(new RegExp(`/sites/priv/polemic-alpha\\?note=${id}`));
+ await expect(page.locator(`[data-note-card='${id}']`)).toHaveAttribute("aria-current", "true");
+});
+
+test("writes outside a report are refused", async ({ request }) => {
+ for (const q of ["article=priv/../pub", "article=priv/nope", "article=priv", "article=../x/y"]) {
+ const r = await request.post(`/api/notes?${q}`, { data: { op: { op: "add", text: "x", anchor: { kind: "whole" } } } });
+ expect([400, 403, 404]).toContain(r.status());
+ }
+ const bad = await request.post("/api/notes?article=priv/polemic-alpha", { data: { op: { op: "add", text: "x", anchor: { kind: "moment", file: "../../x.mp4", t: 1 } } } });
+ expect(bad.status()).toBe(400);
+ expect(existsSync(NOTES)).toBe(false);
+});
diff --git a/umtool/e2e/fixtures/make-fixture.mjs b/umtool/e2e/fixtures/make-fixture.mjs
@@ -22,6 +22,7 @@ import { spawnSync } from "node:child_process";
import path from "node:path";
import { SONG_DATA } from "../../song/paths.mjs";
import { songCapabilities } from "./song-capabilities.mjs";
+import { makeSitesFixture } from "./sites-fixture.mjs";
const CODE = path.resolve(path.dirname(new URL(import.meta.url).pathname), "..", "..", "song");
const dest = path.resolve(process.argv[2] ?? path.join(process.cwd(), ".e2e-song"));
@@ -1814,7 +1815,78 @@ take("intro-a", { group: "opening", order: 5, label: "Cold open", kind: "similar
take("bad-kind", { group: "finale", order: 4, label: "Bad", kind: "maybe" });
mkdirSync(path.join(TAKES, "takes", "current", "out"), { recursive: true });
+// -- THE VIDEO-NOTES AND TIMELINE FIXTURES ------------------------------------
+//
+// video-notes-fixture is a GENERATED manifest (`generatedBy`) with a built cut
+// and its schedule, and a take with its own: timed notes resolve against them,
+// and every edit made to it leaves an `edit` note (video-notes.spec.ts).
+// timeline-fixture is hand-written, with a teaser, three clips, a post riding
+// on the second and the deck on: the structural edits' subject
+// (timeline-edit.spec.ts). Both are written by specs; nothing else reads them.
+const twoSeconds = (file, hz) =>
+ ff([
+ "-f", "lavfi", "-i", "testsrc=size=320x180:rate=15:duration=2",
+ "-f", "lavfi", "-i", `sine=frequency=${hz}:duration=2`,
+ "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-shortest",
+ "-movflags", "+faststart",
+ file,
+ ]);
+const NOTES_SCHEDULE = {
+ version: 1,
+ kind: "deck",
+ estimated: false,
+ fps: 15,
+ transition: 0,
+ total: 2,
+ segments: [
+ { id: "k1", type: "card", start: 0, duration: 0.5, end: 0.5, title: "Opening" },
+ { id: "n01", type: "clip", start: 0.5, duration: 0.7, end: 1.2, title: "The first claim" },
+ { id: "n02", type: "clip", start: 1.2, duration: 0.8, end: 2, title: "The second claim" },
+ ],
+};
+const VNOTES = writeProject("video-notes-fixture", {
+ ...manifest("video-notes-fixture", "The Video Notes Fixture", { siteOrigin: "https://archive.example" }, [
+ { type: "card", id: "k1", heading: "Opening" },
+ { type: "clip", id: "n01", video: "vid1", start: 0, end: 3, cite: 0, section: 0, lock: true, quote: "This is a complete sentence." },
+ { type: "clip", id: "n02", video: "vid1", start: 9, end: 12, cite: 9, section: 0, lock: true, quote: "Another whole sentence entirely." },
+ ]),
+ generatedBy: "polemics/video/make-videos.py",
+});
+{
+ // The deck on: the On-screen section shows the built cut (and its timed
+ // notes) only under it.
+ const m = JSON.parse(readFileSync(path.join(VNOTES, "video.manifest.json"), "utf8"));
+ m.render.chrome = { engine: "hyperframes", layout: "deck" };
+ writeFileSync(path.join(VNOTES, "video.manifest.json"), JSON.stringify(m, null, 2) + "\n");
+}
+mkdirSync(path.join(VNOTES, "out", "sourced"), { recursive: true });
+twoSeconds(path.join(VNOTES, "out", "video-notes-fixture.mp4"), 300);
+writeFileSync(path.join(VNOTES, "out", "sourced", "schedule.json"), JSON.stringify(NOTES_SCHEDULE, null, 2));
+{
+ const dir = path.join(VNOTES, "takes", "alt");
+ mkdirSync(path.join(dir, "out", "sourced"), { recursive: true });
+ writeFileSync(
+ path.join(dir, "take.json"),
+ JSON.stringify({ id: "alt", group: "cut", order: 1, label: "Alternate", kind: "similar", summary: "Tighter.", preview: "preview.mp4", seconds: 2 }, null, 2),
+ );
+ twoSeconds(path.join(dir, "preview.mp4"), 360);
+ writeFileSync(path.join(dir, "out", "sourced", "schedule.json"), JSON.stringify(NOTES_SCHEDULE, null, 2));
+}
+writeProject("timeline-fixture", {
+ ...manifest("timeline-fixture", "The Timeline Fixture", { siteOrigin: "https://archive.example" }, [
+ { type: "teaser", id: "t1", lines: ["THE PROMISE"] },
+ { type: "clip", id: "a01", video: "vid1", start: 0, end: 3, cite: 0, section: 1, sectionEnter: true, lock: true, quote: "one" },
+ { type: "clip", id: "a02", video: "vid1", start: 9, end: 12, cite: 9, section: 1, lock: true, quote: "two" },
+ { type: "clip", id: "a03", video: "vid1", start: 12, end: 15, cite: 12, section: 2, sectionEnter: true, lock: true, quote: "three" },
+ ]),
+ posts: [{ id: "p1", platform: "x", author: "Someone", handle: "@someone", date: "2024-01-02", text: "A post.", url: "https://x.com/someone/status/1", attachTo: "a02" }],
+});
+
+const { sites: SITES } = makeSitesFixture({ dest, reports, channels: CHANNELS });
+
console.log(`fixture at ${dest}`);
+console.log(` SITES_DIR=${SITES}`);
+console.log(` video-notes-fixture (generated, built, schedule + take alt), timeline-fixture (teaser, a01-a03, post p1)`);
if (planned) console.log(` planned clip (used in a build): ${planned}`);
console.log(` videos/: alpha (4 cuts, 3 variants), beta (2 cuts), deck (1 cut, 2 variants)`);
console.log(` deck: 1 spec error, 1 stale recipe, 1 unjudged variant, 1 judged one`);
diff --git a/umtool/e2e/fixtures/sites-fixture.mjs b/umtool/e2e/fixtures/sites-fixture.mjs
@@ -0,0 +1,171 @@
+// The fixture's SITES_DIR (`<dest>/sites`), which the e2e server reads instead
+// of the real transcripts/sites (playwright.config.ts). The suite WRITES notes
+// there -- a report's notes.json is the one corpus file umtool writes -- so it
+// must never be the real one.
+//
+// Called by make-fixture.mjs after the projects and the channels exist, so a
+// report here can cite the fixture's channels and link its projects.
+//
+// sites/priv private, cited-only. Reports:
+// polemic-alpha PUBLISHED: summary, two sections, cites sitechan/sv1
+// (a clip window on disk: plays) and sitechan/sv2 (cues,
+// no media: the fetch_clip line); video.mp4 + poster.jpg
+// polemic-beta a DRAFT (not in site.json's reports)
+// sites/pub public. Reports:
+// gamma PUBLISHED, one section, no citations
+// <dest>/sitews/ the WORKSPACE (a direct child of REPORTS_ROOT, which the
+// e2e server takes as <dest>): polemics/drafts/{alpha,beta}.json,
+// polemics/make-site.py naming `priv`, polemics/out/alpha.{md,html},
+// NOTES.md
+// <dest>/sitews/polemic-alpha/
+// the article's report-video project: a manifest whose slug
+// is polemic-alpha, two takes, one verdict
+// channels/sitechan/data/{sv1,sv2} the cited records
+import { spawnSync } from "node:child_process";
+import { mkdirSync, writeFileSync } from "node:fs";
+import path from "node:path";
+
+const put = (file, value) => {
+ mkdirSync(path.dirname(file), { recursive: true });
+ writeFileSync(file, typeof value === "string" ? value : JSON.stringify(value, null, 2) + "\n");
+};
+
+const ff = (args) => {
+ const r = spawnSync("ffmpeg", ["-nostdin", "-v", "error", "-y", ...args], { encoding: "utf8" });
+ if (r.status !== 0) throw new Error(`ffmpeg failed: ${r.stderr || r.status}`);
+};
+
+const clip = (out, seconds, hz) => {
+ mkdirSync(path.dirname(out), { recursive: true });
+ ff([
+ "-f", "lavfi", "-i", `testsrc=size=320x180:rate=15:duration=${seconds}`,
+ "-f", "lavfi", "-i", `sine=frequency=${hz}:duration=${seconds}`,
+ "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-shortest", "-movflags", "+faststart", out,
+ ]);
+};
+
+export const SITE_CUES = {
+ sv1: [
+ [0, 3, "Opening line of the first source."],
+ [3, 6, "The vote was rigged and everybody knew."],
+ [6, 9, "Nobody checked it at the time."],
+ [9, 12, "Later the story changed again."],
+ [12, 15, "A fifth line for context."],
+ [15, 18, "And a sixth to close."],
+ ],
+ sv2: [
+ [0, 4, "The second source starts here."],
+ [4, 8, "She said it on a different show."],
+ [8, 12, "Then she said the opposite."],
+ ],
+};
+
+const ALPHA_BODY_1 =
+ "She said [the vote was rigged](cite:c1) in the first stream. Nobody checked the claim at the time, and the clip shows it.\n\nThe second paragraph repeats the claim for the record.";
+const ALPHA_BODY_2 = "Two years later [she said the opposite](cite:c2), on a different show.";
+
+const siteJson = (siteId, title, extra) => ({
+ siteId,
+ siteTitle: title,
+ groups: [{ id: "default", name: "All channels", selectedByDefault: true, order: 0, inline: true }],
+ defaultGroupId: "default",
+ channels: [{ slug: "sitechan", order: 0 }],
+ ...extra,
+});
+
+/**
+ * @param {{ dest: string, reports: string, channels: string }} at
+ */
+export function makeSitesFixture({ dest, channels }) {
+ const sites = path.join(dest, "sites");
+ mkdirSync(sites, { recursive: true });
+
+ // ---- the cited records ----------------------------------------------------
+ for (const [vid, rows] of Object.entries(SITE_CUES)) {
+ put(path.join(channels, "sitechan", "data", vid, "transcript.cues.json"), {
+ title: `Site source ${vid}`,
+ uploadDate: "20240315",
+ channel: "Site Channel",
+ webpageUrl: `https://www.youtube.com/watch?v=${vid}`,
+ duration: rows[rows.length - 1][1],
+ cues: rows.map(([start, end, text]) => ({ start, end, text })),
+ });
+ }
+ // sv1 has a clip window the editor fetched: [0, 12] holds the cited [3, 6].
+ clip(path.join(channels, "sitechan", "data", "sv1", "clips", "0.00-12.00.mp4"), 12, 523);
+
+ // ---- the sites ------------------------------------------------------------
+ put(path.join(sites, "priv", "site.json"), siteJson("priv", "Private Fixture", { audience: "private", search: false, reports: ["polemic-alpha"] }));
+ put(path.join(sites, "pub", "site.json"), siteJson("pub", "Public Fixture", { reports: ["gamma"] }));
+
+ const alphaDir = path.join(sites, "priv", "reports", "polemic-alpha");
+ put(path.join(alphaDir, "report.json"), {
+ format: "archilyzer-report",
+ version: 1,
+ id: "polemic-alpha",
+ kind: "sweep",
+ series: "Polemics",
+ title: "Alpha: the rigged vote",
+ subtitle: "What she said, and when",
+ summary: "She said one thing in 2020 and the opposite later.",
+ published: "2026-10-01",
+ updated: "2026-10-07",
+ video: { src: "video.mp4", poster: "poster.jpg" },
+ citations: {
+ c1: { kind: "video", channel: "sitechan", id: "sv1", start: 3, end: 6, quote: "The vote was rigged and everybody knew.", speaker: "Site Channel", date: "2024-03-15" },
+ c2: { kind: "video", channel: "sitechan", id: "sv2", start: 8, end: 12, quote: "Then she said the opposite.", speaker: "Site Channel", date: "2024-03-15" },
+ },
+ sections: [
+ { id: "first", title: "The first claim", body: ALPHA_BODY_1 },
+ { id: "later", title: "Later", body: ALPHA_BODY_2 },
+ ],
+ });
+ clip(path.join(alphaDir, "video.mp4"), 4, 440);
+ ff(["-f", "lavfi", "-i", "testsrc=size=320x180:rate=1:duration=1", "-frames:v", "1", path.join(alphaDir, "poster.jpg")]);
+
+ put(path.join(sites, "priv", "reports", "polemic-beta", "report.json"), {
+ format: "archilyzer-report",
+ version: 1,
+ id: "polemic-beta",
+ kind: "sweep",
+ title: "Beta: a draft",
+ summary: "A draft nobody has published.",
+ sections: [{ id: "only", title: "Only section", body: "The draft's only paragraph." }],
+ });
+
+ put(path.join(sites, "pub", "reports", "gamma", "report.json"), {
+ format: "archilyzer-report",
+ version: 1,
+ id: "gamma",
+ kind: "sweep",
+ title: "Gamma on the public site",
+ published: "2026-09-01",
+ sections: [{ id: "g", title: "Gamma", body: "A public article with nothing cited." }],
+ });
+
+ // ---- the workspace they were written in ------------------------------------
+ const ws = path.join(dest, "sitews");
+ put(path.join(ws, "polemics", "drafts", "alpha.json"), { id: "polemic-alpha", title: "Alpha: the rigged vote", sections: [] });
+ put(path.join(ws, "polemics", "drafts", "beta.json"), { id: "polemic-beta", title: "Beta: a draft", sections: [] });
+ put(path.join(ws, "polemics", "make-site.py"), 'SITE = "priv"\nfor f in DRAFTS.glob("drafts/*.json"):\n rid = f"polemic-{f.stem}"\n');
+ put(path.join(ws, "polemics", "out", "alpha.md"), "# Alpha\n\nThe rendered draft of **alpha**.\n");
+ put(path.join(ws, "polemics", "out", "alpha.html"), "<!doctype html><h1>Alpha html</h1><script>document.title='ran'</script>\n");
+ put(path.join(ws, "NOTES.md"), "# Workspace notes\n\n- one\n- two\n");
+
+ // ---- the article's video project -------------------------------------------
+ const proj = path.join(ws, "polemic-alpha");
+ put(path.join(proj, "video.manifest.json"), {
+ schemaVersion: 1,
+ slug: "polemic-alpha",
+ title: "Alpha: the rigged vote",
+ generatedBy: "polemics/video/make-videos.py",
+ provenance: { channelSlug: "sitechan", siteOrigin: "https://priv.example" },
+ timeline: [{ type: "clip", id: "a1", video: "sv1", channel: "sitechan", start: 3, end: 6, quote: "The vote was rigged" }],
+ });
+ for (const [id, order] of [["deck", 1], ["tight", 2]]) {
+ put(path.join(proj, "takes", id, "take.json"), { id, group: "cut", order, label: id, kind: order === 1 ? "reference" : "similar", preview: "preview.mp4" });
+ }
+ put(path.join(proj, "takes", "verdicts.json"), { deck: { verdict: "like", note: "", at: "2026-10-07T00:00:00Z" } });
+
+ return { sites, workspace: ws, project: proj };
+}
diff --git a/umtool/e2e/sites.spec.ts b/umtool/e2e/sites.spec.ts
@@ -0,0 +1,88 @@
+import { test, expect } from "@playwright/test";
+import { WARM_TIMEOUT, warm } from "./warm";
+
+// ---------------------------------------------------------------------------
+// /sites: every site's articles, a site's page, its media and workspace files.
+//
+// priv/polemic-alpha published, a video + poster, linked to the project
+// sitews/polemic-alpha (slug in the draft's workspace)
+// priv/polemic-beta a draft
+// pub/gamma published on the public site
+// (e2e/fixtures/sites-fixture.mjs)
+// ---------------------------------------------------------------------------
+
+
+test.beforeAll(async ({ playwright }) => {
+ test.setTimeout(WARM_TIMEOUT);
+ await warm(playwright, ["/sites", "/sites/priv", "/sites/priv/polemic-alpha", "/api/sites/media?site=priv&report=polemic-alpha&file=poster.jpg", "/api/sites/workspace?ws=sitews&rel=NOTES.md"]);
+});
+
+test("/sites lists every site, private first, with published and draft articles", async ({ page }) => {
+ const res = await page.goto("/sites");
+ expect(res?.status()).toBe(200);
+ await expect(page.getByRole("link", { name: "sites", exact: true }).first()).toHaveAttribute("aria-current", "page");
+
+ const sites = page.locator("[data-site]");
+ await expect(sites).toHaveCount(2);
+ await expect(sites.nth(0)).toHaveAttribute("data-site", "priv");
+ await expect(sites.nth(1)).toHaveAttribute("data-site", "pub");
+ await expect(page.locator("[data-site='priv'] [data-audience]")).toHaveAttribute("data-audience", "private");
+ await expect(page.locator("[data-site='priv'] [data-counts]")).toHaveAttribute("data-counts", "1/1");
+
+ const alpha = page.locator("[data-article='priv/polemic-alpha']");
+ await expect(alpha).toHaveAttribute("data-status", "published");
+ await expect(alpha).toContainText("Alpha: the rigged vote");
+ await expect(alpha.locator("[data-project-link='sitews/polemic-alpha']")).toBeVisible();
+ await expect(alpha).toContainText("alpha.json");
+ await expect(alpha.locator("img")).toHaveCount(1);
+ await expect(page.locator("[data-article='priv/polemic-beta']")).toHaveAttribute("data-status", "draft");
+});
+
+test("the filters are links, and compose", async ({ page }) => {
+ await page.goto("/sites");
+ await page.getByRole("link", { name: /^draft \d+$/ }).click();
+ await expect(page).toHaveURL(/status=draft/);
+ await expect(page.locator("[data-article]")).toHaveCount(1);
+ await expect(page.locator("[data-article='priv/polemic-beta']")).toBeVisible();
+
+ await page.goto("/sites?site=pub");
+ await expect(page.locator("[data-site]")).toHaveCount(1);
+ await expect(page.locator("[data-article='pub/gamma']")).toBeVisible();
+
+ await page.goto("/sites?notes=open");
+ await expect(page.locator("[data-article]")).toHaveCount(0);
+});
+
+test("a site's page plays its report videos, lists its project's takes and its workspace", async ({ page, request }) => {
+ await page.goto("/sites/priv");
+ await expect(page.locator("[data-report-video='polemic-alpha'] video")).toHaveAttribute("src", /file=video\.mp4/);
+ const project = page.locator("[data-video-project='sitews/polemic-alpha']");
+ await expect(project.locator("[data-takes]")).toHaveAttribute("data-takes", "2");
+ await expect(project).toContainText("like 1");
+
+ // media: ranged, read-only, and nothing outside the report dir
+ const v = await request.get("/api/sites/media?site=priv&report=polemic-alpha&file=video.mp4", { headers: { range: "bytes=0-99" } });
+ expect(v.status()).toBe(206);
+ expect(v.headers()["content-type"]).toBe("video/mp4");
+ expect((await request.get("/api/sites/media?site=priv&report=polemic-alpha&file=../polemic-beta/report.json")).status()).toBe(404);
+ expect((await request.get("/api/sites/media?site=priv&report=polemic-alpha&file=report.json")).status()).toBe(400);
+ expect((await request.get("/api/sites/media?corpus=/etc/passwd")).status()).toBe(400);
+
+ // workspace files: markdown renders, a draft folds, HTML is sandboxed
+ const files = page.locator("[data-section='files']");
+ await files.locator("[data-file='NOTES.md']").click();
+ await expect(page.locator("[data-opened='md']")).toContainText("Workspace notes");
+ await page.locator("[data-file='polemics/drafts/alpha.json']").click();
+ await expect(page.locator("[data-opened='json']")).toContainText("polemic-alpha");
+ await page.locator("[data-file='polemics/out/alpha.html']").click();
+ await expect(page.locator("iframe[sandbox='']")).toHaveCount(1);
+ const html = await request.get("/api/sites/workspace?ws=sitews&rel=polemics/out/alpha.html");
+ expect(html.headers()["content-security-policy"]).toContain("sandbox");
+ expect((await request.get("/api/sites/workspace?ws=sitews&rel=../sites/priv/site.json")).status()).toBe(404);
+ expect((await request.get("/api/sites/workspace?ws=..&rel=NOTES.md")).status()).toBe(404);
+});
+
+test("an unknown site or article is a 404", async ({ page }) => {
+ expect((await page.goto("/sites/nope"))?.status()).toBe(404);
+ expect((await page.goto("/sites/priv/nope"))?.status()).toBe(404);
+});
diff --git a/umtool/e2e/timeline-edit.spec.ts b/umtool/e2e/timeline-edit.spec.ts
@@ -0,0 +1,136 @@
+import { test, expect, type Page } from "@playwright/test";
+import { copyFileSync, existsSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+// ---------------------------------------------------------------------------
+// Structural edits to a report video (lib/report/manifest.mjs "STRUCTURE",
+// POST /api/report/timeline):
+//
+// timeline-fixture hand-written: teaser t1, clips a01 a02 a03, post p1 on
+// a02. WRITES its manifest and revisions/; every test
+// starts from the fixture's manifest.
+//
+// Re-order with alt+↓ and by dragging, undo, the row menu (duplicate, remove,
+// insert after), a refusal in the build's words, and the teaser, posts and
+// fact-check editors.
+// ---------------------------------------------------------------------------
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const DIR = path.join(HERE, "..", ".e2e-song", "reports", "timeline-fixture");
+const MANIFEST = path.join(DIR, "video.manifest.json");
+const PRISTINE = path.join(DIR, "video.manifest.pristine");
+const PAGE = "/browse/reports/timeline-fixture";
+
+type Entry = Record<string, unknown> & { id: string };
+const manifest = () => JSON.parse(readFileSync(MANIFEST, "utf8")) as { timeline: Entry[]; posts?: Entry[]; render: Record<string, unknown> };
+const order = () => manifest().timeline.map((e) => e.id);
+const rows = (page: Page) => page.locator("li[data-entry][data-kind]");
+const shownOrder = (page: Page) => rows(page).evaluateAll((els) => els.map((e) => e.getAttribute("data-entry")));
+
+test.beforeAll(() => {
+ if (!existsSync(PRISTINE)) copyFileSync(MANIFEST, PRISTINE);
+});
+test.beforeEach(() => {
+ copyFileSync(PRISTINE, MANIFEST);
+ rmSync(path.join(DIR, "revisions"), { recursive: true, force: true });
+ rmSync(path.join(DIR, "notes.json"), { force: true });
+});
+
+async function open(page: Page) {
+ await page.goto(PAGE);
+ await expect(page.getByTestId("timeline-undo")).toBeEnabled();
+ // The list has read its token once the page is hydrated.
+ await page.waitForLoadState("networkidle");
+}
+
+test("alt+↓ moves a row, recomputes sectionEnter, and Undo puts it back byte for byte", async ({ page }) => {
+ const before = readFileSync(MANIFEST, "utf8");
+ await open(page);
+ await page.locator("li[data-entry='a01'][data-kind]").focus();
+ await page.keyboard.press("Alt+ArrowDown");
+ await expect.poll(order).toEqual(["t1", "a02", "a01", "a03"]);
+ await expect.poll(() => shownOrder(page)).toEqual(["t1", "a02", "a01", "a03"]);
+ const m = manifest();
+ expect(m.timeline[1].sectionEnter).toBe(true);
+ expect("sectionEnter" in m.timeline[2]).toBe(false);
+ expect(readdirSync(path.join(DIR, "revisions")).some((n) => n.includes("auto-before-move"))).toBe(true);
+
+ await page.getByTestId("timeline-undo").click();
+ await expect.poll(() => readFileSync(MANIFEST, "utf8")).toBe(before);
+ await expect.poll(() => shownOrder(page)).toEqual(["t1", "a01", "a02", "a03"]);
+});
+
+test("a row dragged by its handle lands where it is dropped", async ({ page }) => {
+ await open(page);
+ await page.locator("[data-drag-handle='a03']").dragTo(page.locator("li[data-entry='a01'][data-kind]"));
+ await expect.poll(order).toEqual(["t1", "a03", "a01", "a02"]);
+});
+
+test("the row menu duplicates, removes, inserts — and a refusal is in the build's words", async ({ page }) => {
+ await open(page);
+ await page.locator("[data-row-menu='a03']").click();
+ await page.locator("[data-row-action='duplicate']").click();
+ await expect.poll(order).toEqual(["t1", "a01", "a02", "a03", "a03-copy"]);
+
+ await expect(page.locator("[data-row-menu='a03-copy']")).toBeVisible();
+ await page.locator("[data-row-menu='a03-copy']").click();
+ await page.locator("[data-row-action='remove']").click();
+ await expect.poll(order).toEqual(["t1", "a01", "a02", "a03"]);
+
+ await page.locator("[data-row-menu='a03']").click();
+ await page.locator("[data-row-action='insert']").click();
+ await page.getByTestId("insert-after-a03").fill("testchan/vid1@3-6");
+ await page.getByTestId("insert-after-a03").press("Enter");
+ await expect.poll(order).toEqual(["t1", "a01", "a02", "a03", "vid1-3"]);
+ expect(manifest().timeline[4]).toEqual({ type: "clip", id: "vid1-3", channel: "testchan", video: "vid1", start: 3, end: 6 });
+
+ // p1 rides on a02: removing it is refused, and nothing is written.
+ const was = readFileSync(MANIFEST, "utf8");
+ await page.locator("[data-row-menu='a02']").click();
+ await page.locator("[data-row-action='remove']").click();
+ await expect(page.getByTestId("timeline-error")).toContainText("attachTo");
+ expect(readFileSync(MANIFEST, "utf8")).toBe(was);
+});
+
+test("the teaser's lines and beat, edited in place; an empty teaser is refused", async ({ page }) => {
+ await open(page);
+ await page.getByTestId("structure-folded").locator("summary").click();
+ await page.getByTestId("teaser-lines-t1").fill("THE PROMISE\nAND WHAT HAPPENED");
+ await page.getByTestId("teaser-beat-t1").fill("1.2");
+ await page.getByTestId("teaser-save-t1").click();
+ await expect.poll(() => manifest().timeline[0].lines).toEqual(["THE PROMISE", "AND WHAT HAPPENED"]);
+ expect(manifest().timeline[0].beat).toBe(1.2);
+
+ await page.getByTestId("teaser-lines-t1").fill("");
+ await page.getByTestId("teaser-save-t1").click();
+ await expect(page.getByTestId("structure-error")).toContainText("lines must be a list");
+});
+
+test("a post added, then removed; the fact-check's labels once the deck is on", async ({ page }) => {
+ // The fact-check is drawn by the deck: turn it on in the fixture first.
+ const m = manifest();
+ m.render.chrome = { engine: "hyperframes", layout: "deck" };
+ writeFileSync(MANIFEST, JSON.stringify(m, null, 2) + "\n");
+
+ await open(page);
+ await page.getByTestId("structure-folded").locator("summary").click();
+ await page.getByTestId("post-add").click();
+ const fresh = page.locator("[data-post-row='new']");
+ await fresh.getByTestId("post-id").fill("p2");
+ await fresh.getByTestId("post-date").fill("2024-02-03");
+ await fresh.getByTestId("post-url").fill("https://x.com/someone/status/2");
+ await fresh.getByTestId("post-text").fill("Another post.");
+ await fresh.getByTestId("post-save").click();
+ await expect.poll(() => (manifest().posts ?? []).map((p) => p.id)).toEqual(["p1", "p2"]);
+
+ await page.locator("[data-post-row='p2']").getByTestId("post-remove").click();
+ await expect.poll(() => (manifest().posts ?? []).map((p) => p.id)).toEqual(["p1"]);
+
+ await page.getByTestId("fc-label-CONTRADICTED").fill("NOPE");
+ await page.getByTestId("fc-color-CONTRADICTED").fill("#ff0000");
+ await page.getByTestId("fc-save").click();
+ await expect
+ .poll(() => (manifest().render.chrome as { factcheck?: unknown }).factcheck)
+ .toEqual({ verdicts: { CONTRADICTED: { label: "NOPE", color: "#ff0000" } } });
+});
diff --git a/umtool/e2e/video-notes.spec.ts b/umtool/e2e/video-notes.spec.ts
@@ -0,0 +1,151 @@
+import { test, expect, type Locator, type Page } from "@playwright/test";
+import { copyFileSync, existsSync, readFileSync, rmSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+// ---------------------------------------------------------------------------
+// Notes on a report video (lib/annotations, components/notes):
+//
+// video-notes-fixture a GENERATED manifest (`generatedBy`) with a built cut,
+// its schedule, and one take (`alt`) with its own. WRITES
+// its notes.json; every test starts from the fixture's
+// manifest and no notes.
+//
+// What is proved: a timed note at a second of the built cut and of a take's
+// preview, resolved to the entry on screen; take notes and row notes; and that
+// an edit made here to a generated manifest leaves an `edit` note, coalesced,
+// and gone again when the edit is put back.
+// ---------------------------------------------------------------------------
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const DIR = path.join(HERE, "..", ".e2e-song", "reports", "video-notes-fixture");
+const MANIFEST = path.join(DIR, "video.manifest.json");
+const PRISTINE = path.join(DIR, "video.manifest.pristine");
+const NOTES = path.join(DIR, "notes.json");
+const PROJECT = "reports/video-notes-fixture";
+const PAGE = `/browse/${PROJECT}`;
+
+type Note = { id: string; status: string; author: string; text: string; anchor: Record<string, unknown>; replies: unknown[] };
+const notesOnDisk = (): Note[] => (existsSync(NOTES) ? JSON.parse(readFileSync(NOTES, "utf8")).notes : []);
+
+test.beforeAll(() => {
+ if (!existsSync(PRISTINE)) copyFileSync(MANIFEST, PRISTINE);
+});
+test.beforeEach(() => {
+ copyFileSync(PRISTINE, MANIFEST);
+ rmSync(NOTES, { force: true });
+ rmSync(path.join(DIR, "revisions"), { recursive: true, force: true });
+});
+
+/** Seek a <video> to `t` and wait for it to land. */
+async function seek(video: Locator, t: number) {
+ await video.evaluate(async (el: HTMLVideoElement, at: number) => {
+ if (el.readyState < 1) await new Promise((r) => el.addEventListener("loadedmetadata", r, { once: true }));
+ el.currentTime = at;
+ await new Promise((r) => el.addEventListener("seeked", r, { once: true }));
+ }, t);
+}
+
+async function addTimedNote(page: Page, scope: Locator, text: string, via: "button" | "key") {
+ if (via === "button") await scope.locator("[data-action='mark']").click();
+ else {
+ await scope.locator("video").focus();
+ await page.keyboard.press("n");
+ }
+ const input = scope.getByTestId("timed-note-input");
+ await input.fill(text);
+ await input.press("Enter");
+}
+
+test("the generated banner is on the project page and the clip bench", async ({ page }) => {
+ await page.goto(PAGE);
+ await expect(page.getByTestId("generated-banner").first()).toContainText(
+ "Generated by polemics/video/make-videos.py; a rebuild of manifests overwrites edits made here.",
+ );
+ await page.goto(`${PAGE}/clip/n01`);
+ await expect(page.getByTestId("generated-banner")).toContainText("polemics/video/make-videos.py");
+});
+
+test("a timed note on the built cut resolves to the entry on screen and its source", async ({ page }) => {
+ await page.goto(PAGE);
+ const video = page.getByTestId("onscreen-final-video");
+ await expect(video).toBeVisible();
+ const scope = page.locator("[data-timed-notes='out/video-notes-fixture.mp4']");
+ await seek(video, 0.8);
+ await addTimedNote(page, scope, "the claim card is late", "button");
+
+ await expect(scope.locator("[data-mark-entry='n01']")).toContainText("the claim card is late");
+ await expect(scope.locator("[data-mark-entry='n01']")).toContainText("The first claim");
+ await expect(scope.locator("[data-tick]")).toHaveCount(1);
+
+ const [n] = notesOnDisk();
+ expect(n.author).toBe("operator");
+ expect(n.anchor).toMatchObject({ kind: "moment", file: "out/video-notes-fixture.mp4", t: 0.8, entry: "n01" });
+ expect(n.anchor.resolved).toMatchObject({ title: "The first claim", channel: "testchan", video: "vid1", sourceT: 0.3 });
+ expect(String((n.anchor.resolved as { url: string }).url)).toBe("https://archive.example/?v=testchan%2Fvid1&t=0");
+ expect((n.anchor.resolved as { approx?: boolean }).approx).toBeUndefined();
+
+ // Delete it: the last note takes the file with it.
+ await scope.locator(`[data-note-id='${n.id}'] [data-note-action='delete']`).click();
+ await expect(scope.locator("[data-mark-at]")).toHaveCount(0);
+ await expect.poll(() => existsSync(NOTES)).toBe(false);
+});
+
+test("a take: notes on the take, and a timed note on its preview with `n`", async ({ page }) => {
+ await page.goto(`${PAGE}/takes`);
+ const card = page.locator("[data-take='alt']");
+ await seek(card.locator("video"), 1.5);
+ await addTimedNote(page, card, "second claim runs long", "key");
+ await expect(card.locator("[data-mark-entry='n02']")).toContainText("second claim runs long");
+
+ await card.locator("[data-anchored-notes='alt'] [data-action='toggle-notes']").click();
+ await card.getByTestId("note-input-alt").fill("prefer this one, but tighter");
+ await card.getByTestId("note-input-alt").press("Enter");
+ await expect(card.locator("[data-anchored-notes='alt']")).toHaveAttribute("data-open-notes", "1");
+
+ const notes = notesOnDisk();
+ expect(notes.map((n) => n.anchor.kind).sort()).toEqual(["moment", "take"]);
+ expect(notes.find((n) => n.anchor.kind === "moment")!.anchor).toMatchObject({ file: "takes/alt/preview.mp4", take: "alt", entry: "n02" });
+ expect(notes.find((n) => n.anchor.kind === "take")!.anchor).toEqual({ kind: "take", take: "alt" });
+
+ // Resolve the take note; it stays, shown as resolved.
+ const takeNote = notes.find((n) => n.anchor.kind === "take")!;
+ await card.locator(`[data-note-id='${takeNote.id}'] [data-note-action='resolve']`).click();
+ await expect(card.locator(`[data-note-id='${takeNote.id}']`)).toHaveAttribute("data-note-status", "resolved");
+ await expect(card.locator("[data-anchored-notes='alt']")).toHaveAttribute("data-open-notes", "0");
+ expect(notesOnDisk().find((n) => n.id === takeNote.id)!.status).toBe("resolved");
+});
+
+test("an edit to a generated manifest leaves one edit note, coalesced, gone when put back", async ({ request }) => {
+ const token = async () => (await (await request.get(`/api/report/clip?project=${PROJECT}&clip=n01`)).json()).token as string;
+ const put = async (title: string) =>
+ request.put("/api/report/window", { data: { project: PROJECT, clip: "n01", token: await token(), title } });
+
+ let r = await (await put("A new title")).json();
+ expect(r.editNotes).toMatchObject({ generatedBy: "polemics/video/make-videos.py", added: 1 });
+ r = await (await put("A newer title")).json();
+ expect(r.editNotes).toMatchObject({ added: 0, updated: 1 });
+ const [n] = notesOnDisk();
+ expect(notesOnDisk()).toHaveLength(1);
+ expect(n.anchor).toEqual({ kind: "edit", entry: "n01", field: "title", from: null, to: "A newer title" });
+ expect(n.text).toContain("polemics/video/make-videos.py");
+
+ const doc = JSON.parse(readFileSync(NOTES, "utf8"));
+ expect(doc.source).toMatchObject({ manifest: expect.stringContaining("video-notes-fixture/video.manifest.json") });
+
+ r = await (await put("")).json();
+ expect(r.editNotes).toMatchObject({ deleted: 1 });
+ expect(existsSync(NOTES)).toBe(false);
+});
+
+test("a row note on the project page, counted on the row and kept across a reload", async ({ page }) => {
+ await page.goto(PAGE);
+ const row = page.locator("[data-anchored-notes='n02']");
+ await row.locator("[data-action='toggle-notes']").click();
+ await page.getByTestId("note-input-n02").fill("check the date on this one");
+ await page.getByTestId("note-input-n02").press("Enter");
+ await expect(row).toHaveAttribute("data-open-notes", "1");
+ await page.reload();
+ await expect(page.locator("[data-anchored-notes='n02']")).toHaveAttribute("data-open-notes", "1");
+ expect(notesOnDisk()[0].anchor).toEqual({ kind: "entry", entry: "n02" });
+});
diff --git a/umtool/e2e/warm.ts b/umtool/e2e/warm.ts
@@ -0,0 +1,17 @@
+import { test, type PlaywrightWorkerArgs } from "@playwright/test";
+
+// The e2e server is `next dev`: the first request to a route COMPILES it, and
+// the article page pulls in a large graph (common's report views, markdown,
+// the evidence resolver). On a loaded machine that first compile alone has
+// taken over 30 s -- a test's whole budget -- so each spec file that visits
+// these routes compiles them first, in a beforeAll with its own timeout.
+export const WARM_TIMEOUT = 180_000;
+
+export async function warm(playwright: PlaywrightWorkerArgs["playwright"], urls: string[]) {
+ const ctx = await playwright.request.newContext({ baseURL: test.info().project.use.baseURL });
+ try {
+ for (const u of urls) await ctx.get(u, { timeout: 170_000 });
+ } finally {
+ await ctx.dispose();
+ }
+}
diff --git a/umtool/lib/annotations/anchor.mjs b/umtool/lib/annotations/anchor.mjs
@@ -0,0 +1,176 @@
+// Finding a text anchor again in text that may have changed.
+//
+// A note on an article is anchored by its QUOTE plus 32 characters of context
+// either side (the W3C TextQuoteSelector), never by an offset: report.json is
+// regenerated from a draft, and an offset into the old text points at nothing
+// after the agent edits the paragraph above it.
+//
+// PURE and client-safe: no node imports. The article page runs it against the
+// rendered DOM text of a section; `umtool notes` runs it against the section's
+// plain text; the unit test runs it against both.
+
+export const CONTEXT = 32;
+
+/** Common-suffix length of `a` and `b` (how much of the prefix still precedes). */
+function suffixMatch(a, b) {
+ let n = 0;
+ while (n < a.length && n < b.length && a[a.length - 1 - n] === b[b.length - 1 - n]) n += 1;
+ return n;
+}
+/** Common-prefix length of `a` and `b` (how much of the suffix still follows). */
+function prefixMatch(a, b) {
+ let n = 0;
+ while (n < a.length && n < b.length && a[n] === b[n]) n += 1;
+ return n;
+}
+
+function allIndexes(hay, needle) {
+ const out = [];
+ if (!needle) return out;
+ for (let i = hay.indexOf(needle); i !== -1; i = hay.indexOf(needle, i + 1)) out.push(i);
+ return out;
+}
+
+/** Of several hits, the one whose surroundings best match prefix/suffix. Ties: the first. */
+function best(hay, hits, len, prefix, suffix) {
+ let top = hits[0];
+ let topScore = -1;
+ for (const i of hits) {
+ const score =
+ suffixMatch(hay.slice(Math.max(0, i - prefix.length), i), prefix) +
+ prefixMatch(hay.slice(i + len, i + len + suffix.length), suffix);
+ if (score > topScore) {
+ top = i;
+ topScore = score;
+ }
+ }
+ return top;
+}
+
+/**
+ * Whitespace collapsed (and typographic quotes/dashes folded), with a map from
+ * each normalised index back to the original one. `lower` also lowercases.
+ */
+export function normalise(text, { lower = false } = {}) {
+ const chars = [];
+ const map = [];
+ let space = false;
+ for (let i = 0; i < text.length; i += 1) {
+ let c = text[i];
+ if (/\s/.test(c)) {
+ if (space || chars.length === 0) continue;
+ space = true;
+ chars.push(" ");
+ map.push(i);
+ continue;
+ }
+ space = false;
+ if (c === "‘" || c === "’") c = "'";
+ else if (c === "“" || c === "”") c = '"';
+ else if (c === "–" || c === "—") c = "-";
+ else if (c === "…") c = ".";
+ if (lower) c = c.toLowerCase();
+ chars.push(c);
+ map.push(i);
+ }
+ if (chars[chars.length - 1] === " ") {
+ chars.pop();
+ map.pop();
+ }
+ map.push(text.length);
+ return { text: chars.join(""), map };
+}
+
+const norm = (s, lower) => normalise(s ?? "", { lower }).text;
+
+/**
+ * Locate a text anchor. `{ found: true, start, end, how }` with offsets into
+ * `text`, or `{ found: false }` -- an ORPHANED note, still shown, pinned to its
+ * section. `how`: "exact", "normalised" (whitespace/quotes/case differ), or
+ * "context" (the quote itself was edited, but what came before and after it is
+ * still there, close together).
+ *
+ * @param {string} text
+ * @param {{ quote: string, prefix?: string, suffix?: string }} anchor
+ */
+export function locateQuote(text, anchor) {
+ const quote = anchor?.quote ?? "";
+ const prefix = anchor?.prefix ?? "";
+ const suffix = anchor?.suffix ?? "";
+ if (!text || !quote) return { found: false };
+
+ const exact = allIndexes(text, quote);
+ if (exact.length) {
+ const i = best(text, exact, quote.length, prefix, suffix);
+ return { found: true, start: i, end: i + quote.length, how: "exact" };
+ }
+
+ for (const lower of [false, true]) {
+ const n = normalise(text, { lower });
+ const q = norm(quote, lower);
+ if (!q) continue;
+ const hits = allIndexes(n.text, q);
+ if (hits.length) {
+ const i = best(n.text, hits, q.length, norm(prefix, lower), norm(suffix, lower));
+ return { found: true, start: n.map[i], end: n.map[i + q.length - 1] + 1, how: "normalised" };
+ }
+ }
+
+ // The quote was rewritten. If the context on BOTH sides survives, close
+ // together, the span between them is where it was.
+ const n = normalise(text, { lower: true });
+ const p = norm(prefix, true);
+ const s = norm(suffix, true);
+ if (p.length >= 8 && s.length >= 8) {
+ const q = norm(quote, true);
+ for (const pi of allIndexes(n.text, p)) {
+ const from = pi + p.length;
+ const si = n.text.indexOf(s, from);
+ if (si === -1) continue;
+ const span = si - from;
+ if (span <= 0 || span > Math.max(q.length * 2, q.length + 80)) continue;
+ // Trim the separator spaces the normalised text keeps around the span.
+ let a = from;
+ let b = si;
+ while (a < b && n.text[a] === " ") a += 1;
+ while (b > a && n.text[b - 1] === " ") b -= 1;
+ if (a >= b) continue;
+ return { found: true, start: n.map[a], end: n.map[b - 1] + 1, how: "context" };
+ }
+ }
+ return { found: false };
+}
+
+/**
+ * The anchor for a selection `[start, end)` of `text`: the quote (trimmed of
+ * surrounding whitespace) and up to CONTEXT characters either side.
+ *
+ * @param {string} text
+ * @param {number} start
+ * @param {number} end
+ * @param {number} [context]
+ */
+export function quoteAnchor(text, start, end, context = CONTEXT) {
+ let a = Math.max(0, Math.min(start, end));
+ let b = Math.min(text.length, Math.max(start, end));
+ while (a < b && /\s/.test(text[a])) a += 1;
+ while (b > a && /\s/.test(text[b - 1])) b -= 1;
+ return {
+ quote: text.slice(a, b),
+ prefix: text.slice(Math.max(0, a - context), a),
+ suffix: text.slice(b, b + context),
+ };
+}
+
+/** The sentence of `text` around `[start, end)`, for a reader with no page open. */
+export function sentenceAround(text, start, end, max = 400) {
+ const before = text.slice(0, start);
+ const after = text.slice(end);
+ const boundary = /[.?!]\s+|\n/g;
+ let from = 0;
+ for (let m = boundary.exec(before); m; m = boundary.exec(before)) from = m.index + m[0].length;
+ const m = after.search(/[.?!](\s|$)|\n/);
+ const to = m === -1 ? text.length : end + m + (after[m] === "\n" ? 0 : 1);
+ const out = text.slice(from, to).replace(/\s+/g, " ").trim();
+ return out.length > max ? `${out.slice(0, max - 1)}…` : out;
+}
diff --git a/umtool/lib/annotations/anchor.test.mjs b/umtool/lib/annotations/anchor.test.mjs
@@ -0,0 +1,65 @@
+// Re-anchoring a quote in text that changed.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import test from "node:test";
+import { locateQuote, normalise, quoteAnchor, sentenceAround } from "./anchor.mjs";
+
+const P1 = "She said the vote was rigged in 2020. Nobody checked the claim at the time.";
+const P2 = "Two years later she said the vote was rigged again, on a different show.";
+const TEXT = `${P1}\n\n${P2}`;
+
+test("an exact quote is found, and the context picks between two copies", () => {
+ const second = TEXT.indexOf("the vote was rigged", P1.length);
+ const a = quoteAnchor(TEXT, second, second + "the vote was rigged".length);
+ assert.equal(a.quote, "the vote was rigged");
+ const r = locateQuote(TEXT, a);
+ assert.deepEqual([r.found, r.start, r.how], [true, second, "exact"]);
+ const first = locateQuote(TEXT, quoteAnchor(TEXT, TEXT.indexOf("the vote"), TEXT.indexOf("the vote") + 19));
+ assert.equal(first.start, TEXT.indexOf("the vote"));
+});
+
+test("a paragraph edited ABOVE the quote does not move the note off it", () => {
+ const start = TEXT.indexOf("on a different show");
+ const a = quoteAnchor(TEXT, start, start + "on a different show".length);
+ const edited = `A new opening paragraph the agent added.\n\n${P1.replace("Nobody checked", "No outlet checked")}\n\n${P2}`;
+ const r = locateQuote(edited, a);
+ assert.equal(r.found, true);
+ assert.equal(edited.slice(r.start, r.end), "on a different show");
+});
+
+test("whitespace, typographic quotes and case still match (normalised)", () => {
+ const a = { quote: "it's the\nclaim", prefix: "", suffix: "" };
+ const t = "And then: It’s the claim, again.";
+ const r = locateQuote(t, a);
+ assert.equal(r.found, true);
+ assert.equal(r.how, "normalised");
+ assert.equal(t.slice(r.start, r.end), "It’s the claim");
+});
+
+test("a rewritten quote is found by its surviving context; a vanished one is orphaned", () => {
+ const start = TEXT.indexOf("Nobody checked the claim");
+ const a = quoteAnchor(TEXT, start, start + "Nobody checked the claim".length);
+ const rewritten = TEXT.replace("Nobody checked the claim", "No outlet verified it");
+ const r = locateQuote(rewritten, a);
+ assert.equal(r.found, true);
+ assert.equal(r.how, "context");
+ assert.equal(rewritten.slice(r.start, r.end), "No outlet verified it");
+
+ const gone = locateQuote("A completely different article.", a);
+ assert.deepEqual(gone, { found: false });
+ assert.deepEqual(locateQuote("", a), { found: false });
+});
+
+test("normalise maps back to the original offsets", () => {
+ const n = normalise(" a \n\n b—c ");
+ assert.equal(n.text, "a b-c");
+ assert.deepEqual(n.map.slice(0, 5), [2, 3, 7, 8, 9]);
+});
+
+test("sentenceAround gives the sentence holding the quote", () => {
+ const s = TEXT.indexOf("Nobody checked");
+ assert.equal(sentenceAround(TEXT, s, s + 6), "Nobody checked the claim at the time.");
+ const f = TEXT.indexOf("Two years");
+ assert.equal(sentenceAround(TEXT, f, f + 3), P2);
+});
diff --git a/umtool/lib/annotations/cli.mjs b/umtool/lib/annotations/cli.mjs
@@ -0,0 +1,140 @@
+// `umtool notes` -- the agent's side of the notes the operator writes in the
+// app. It reads and writes through the same store (./store.mjs) and targets
+// (./targets.mjs) the app does, and stamps every write `author: agent`. An
+// agent never hand-edits notes.json: this validates, locks, and keeps the
+// operator's page from losing a reply (docs/notes.md).
+//
+// umtool notes [--all] every notes file, with open counts
+// umtool notes <site>/<report> | <project> the digest (open notes)
+// [--open | --resolved | --all-status] [--json]
+// umtool notes reply <id> "<text>" [--resolve] [--in <target>]
+// umtool notes resolve | wontfix | reopen <id> [--in <target>]
+// umtool notes source <target> [--draft P] [--generator P] [--how T]
+import { REPORTS_ROOT, SITES_DIR } from "../paths.mjs";
+import { digest } from "./digest.mjs";
+import { readNotes } from "./store.mjs";
+import { articleTarget, listNotesFiles, projectTarget, resolveTarget, writeNote } from "./targets.mjs";
+
+const USAGE = [
+ "usage: umtool notes [--all] every notes file and its open count",
+ " umtool notes <site>/<report> | <project> the notes, as markdown (open ones)",
+ " [--open | --resolved | --all-status] [--json]",
+ " umtool notes reply <id> \"<text>\" [--resolve] answer a note (and close it)",
+ " umtool notes resolve | wontfix | reopen <id>",
+ " umtool notes source <target> [--draft P] [--generator P] [--how T]",
+ "",
+ "Notes are the operator's, written in umtool (/sites, a video project). Act on one",
+ "by editing its SOURCE file and regenerating; never edit notes.json by hand.",
+].join("\n");
+
+class CliError extends Error {}
+
+/**
+ * @param {string[]} args everything after `notes`
+ * @param {{ sitesDir?: string, reportsRoot?: string, log?: (s: string) => void }} [opts]
+ * @returns {Promise<number>} exit code
+ */
+export async function notesCommand(args, { sitesDir = SITES_DIR, reportsRoot = REPORTS_ROOT, log = console.log } = {}) {
+ const flags = new Set(args.filter((a) => a.startsWith("--")));
+ const val = (n) => {
+ const i = args.indexOf(n);
+ return i >= 0 && i + 1 < args.length ? args[i + 1] : undefined;
+ };
+ const VALUED = new Set(["--in", "--draft", "--generator", "--how"]);
+ const pos = args.filter((a, i) => !a.startsWith("--") && !(i > 0 && VALUED.has(args[i - 1])));
+ const json = flags.has("--json");
+ const opts = { sitesDir, reportsRoot };
+ const print = (v) => log(json ? JSON.stringify(v, null, 2) : v);
+
+ try {
+ const verb = pos[0];
+ if (flags.has("--help") || verb === "help") {
+ log(USAGE);
+ return 0;
+ }
+ if (verb === "reply" || verb === "resolve" || verb === "wontfix" || verb === "reopen") {
+ const id = pos[1];
+ if (!id) throw new CliError(`which note? \`umtool notes ${verb} <id>\``);
+ const where = await findNote(id, val("--in"), opts);
+ let op;
+ if (verb === "reply") {
+ const text = pos[2];
+ if (!text) throw new CliError('what reply? `umtool notes reply <id> "<text>" [--resolve]`');
+ op = { op: "reply", id, text, resolve: flags.has("--resolve") };
+ } else {
+ op = { op: "status", id, status: verb === "reopen" ? "open" : verb === "wontfix" ? "wontfix" : "resolved" };
+ }
+ const r = await writeNote(where.target, op, { by: "agent" });
+ if (json) print({ ok: true, target: where.target.id, file: where.target.file, note: r.note });
+ else log(`${id} in ${where.target.id}: ${r.note?.status}${verb === "reply" ? `, ${r.note?.replies.length} repl${r.note?.replies.length === 1 ? "y" : "ies"}` : ""}`);
+ return 0;
+ }
+ if (verb === "source") {
+ const spec = pos[1];
+ if (!spec) throw new CliError("which target? `umtool notes source <site>/<report> --draft P`");
+ const target = await resolveTarget(spec, opts);
+ const cur = await readNotes(target.file);
+ if (!cur.doc) throw new CliError(`${target.id} has no notes; the source is recorded with the first note`);
+ const source = { ...(cur.doc.source ?? {}) };
+ for (const k of ["draft", "generator", "how"]) if (val(`--${k}`) !== undefined) source[k] = val(`--${k}`);
+ const r = await writeNote(target, { op: "source", source }, { by: "agent" });
+ print(json ? { ok: true, source: r.doc?.source ?? null } : `${target.id}: source ${JSON.stringify(r.doc?.source ?? {})}`);
+ return 0;
+ }
+
+ const status = flags.has("--all-status") ? "all" : flags.has("--resolved") ? "resolved" : "open";
+ if (!verb || flags.has("--all")) {
+ const files = await listNotesFiles(opts);
+ if (json) {
+ print(files.map((f) => ({ kind: f.kind, id: f.id, file: f.file, open: f.doc ? f.doc.notes.filter((n) => n.status === "open").length : null, total: f.doc?.notes.length ?? null, error: f.error })));
+ return 0;
+ }
+ if (!files.length) {
+ log(`no notes under ${sitesDir} or ${reportsRoot}`);
+ return 0;
+ }
+ const w = Math.max(...files.map((f) => f.id.length));
+ let open = 0;
+ for (const f of files) {
+ const o = f.doc ? f.doc.notes.filter((n) => n.status === "open").length : 0;
+ open += o;
+ if (status === "open" && !o && !f.error) continue;
+ log(`${f.id.padEnd(w)} ${f.kind === "article" ? "article" : "video "} ${f.error ? `UNREADABLE: ${f.error}` : `${o} open / ${f.doc.notes.length}`}`);
+ }
+ log(`\n${open} open note(s) in ${files.length} file(s). \`umtool notes <id>\` for one.`);
+ return 0;
+ }
+
+ const target = await resolveTarget(verb, opts);
+ const read = await readNotes(target.file);
+ const entry = { kind: target.kind, id: target.id, file: target.file, doc: read.doc, ...(read.error ? { error: read.error } : {}) };
+ if (json) {
+ print({ ...entry, source: read.doc?.source ?? (await target.source()) ?? null });
+ return 0;
+ }
+ if (!read.doc && !read.error) {
+ log(`no notes on ${target.id} (${target.file})`);
+ return 0;
+ }
+ log(await digest(entry, { status, sitesDir, reportsRoot, projectDir: target.dir }));
+ return 0;
+ } catch (err) {
+ console.error(err instanceof Error ? err.message : String(err));
+ return err instanceof CliError || err?.name === "NoteError" || err?.name === "TargetError" ? 2 : 1;
+ }
+}
+
+/** The notes file holding note `id`: the one named by --in, else a search of every file. */
+async function findNote(id, inSpec, opts) {
+ if (inSpec) {
+ const target = await resolveTarget(inSpec, opts);
+ const r = await readNotes(target.file);
+ if (!r.doc?.notes.some((n) => n.id === id)) throw new CliError(`no note ${id} in ${target.id}`);
+ return { target };
+ }
+ const hits = (await listNotesFiles(opts)).filter((f) => f.doc?.notes.some((n) => n.id === id));
+ if (!hits.length) throw new CliError(`no note ${id} (see \`umtool notes --all\`)`);
+ if (hits.length > 1) throw new CliError(`${id} is in ${hits.length} files; say which with --in: ${hits.map((h) => h.id).join(", ")}`);
+ const hit = hits[0];
+ return { target: hit.kind === "article" ? await articleTarget(hit.id, opts) : await projectTarget(hit.id, opts) };
+}
diff --git a/umtool/lib/annotations/cli.test.mjs b/umtool/lib/annotations/cli.test.mjs
@@ -0,0 +1,103 @@
+// `umtool notes`: the agent's loop, end to end, against a temp SITES_DIR and
+// REPORTS_DIR -- read the digest, reply and resolve, reopen, list.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { notesCommand } from "./cli.mjs";
+import { articleTarget, projectTarget, writeNote } from "./targets.mjs";
+import { clearSourcesCache } from "../articles/sources.mjs";
+
+const REPORT = {
+ format: "archilyzer-report",
+ version: 1,
+ id: "polemic-x",
+ kind: "sweep",
+ title: "On X",
+ summary: "She said **X** twice. Then she [denied it](cite:c1).",
+ citations: { c1: { kind: "video", channel: "ch", id: "v1", start: 10, end: 20, quote: "I never said X", speaker: "Her" } },
+ sections: [{ id: "s1", title: "The first time", body: "In 2019 she said X on a podcast. Nobody noticed." }],
+};
+
+async function fixture() {
+ const root = await mkdtemp(path.join(tmpdir(), "umtool-notes-cli-"));
+ const sites = path.join(root, "sites");
+ const reports = path.join(root, "reports");
+ await mkdir(path.join(sites, "priv", "reports", "polemic-x"), { recursive: true });
+ await writeFile(path.join(sites, "priv", "site.json"), "{}");
+ await writeFile(path.join(sites, "priv", "reports", "polemic-x", "report.json"), JSON.stringify(REPORT));
+ await mkdir(path.join(reports, "ws", "polemics", "drafts"), { recursive: true });
+ await writeFile(path.join(reports, "ws", "polemics", "drafts", "x.json"), JSON.stringify({ id: "polemic-x" }));
+ await writeFile(path.join(reports, "ws", "polemics", "make-site.py"), 'SITE = "priv" # drafts\n');
+ const proj = path.join(reports, "ws", "polemic-x");
+ await mkdir(proj, { recursive: true });
+ await writeFile(
+ path.join(proj, "video.manifest.json"),
+ JSON.stringify({ schemaVersion: 1, slug: "polemic-x", generatedBy: "polemics/make-site.py", timeline: [{ type: "clip", id: "e1", channel: "ch", video: "v1", start: 10, end: 20, quote: "I never said X", onscreen: { title: "Denial" } }] }),
+ );
+ clearSourcesCache();
+ return { root, sites, reports, opts: { sitesDir: sites, reportsRoot: reports } };
+}
+
+async function run(args, opts) {
+ const out = [];
+ const code = await notesCommand(args, { ...opts, log: (s) => out.push(String(s)) });
+ return { code, text: out.join("\n") };
+}
+
+test("the agent loop: digest, reply --resolve, reopen; every write is the agent's", async () => {
+ const { root, sites, opts } = await fixture();
+ const t = await articleTarget("priv/polemic-x", opts);
+ const a = await writeNote(t, { op: "add", text: "Too strong; say 'claimed'.", anchor: { kind: "text", section: "s1", quote: "she said X", prefix: "In 2019 ", suffix: " on a podcast" } }, { by: "operator" });
+ await writeNote(t, { op: "add", text: "Is this the right clip?", anchor: { kind: "cite", cite: "c1" } }, { by: "operator" });
+
+ const d = await run(["priv/polemic-x"], opts);
+ assert.equal(d.code, 0);
+ assert.match(d.text, /# Notes on priv\/polemic-x — On X/);
+ assert.match(d.text, /edit `.*ws\/polemics\/drafts\/x\.json`; `report\.json` is regenerated by `.*make-site\.py`/);
+ assert.match(d.text, /“she said X” in “The first time”/);
+ assert.match(d.text, /> In 2019 she said X on a podcast\./);
+ assert.match(d.text, /citation `c1` — ch\/v1@10-20 \(Her\)/);
+ assert.match(d.text, /2 open, 0 closed/);
+
+ const r = await run(["reply", a.note.id, "Changed to 'claimed' in drafts/x.json", "--resolve"], opts);
+ assert.equal(r.code, 0, r.text);
+ const doc = JSON.parse(await readFile(path.join(sites, "priv", "reports", "polemic-x", "notes.json"), "utf8"));
+ const n = doc.notes.find((x) => x.id === a.note.id);
+ assert.equal(n.status, "resolved");
+ assert.equal(n.resolvedBy, "agent");
+ assert.deepEqual(n.replies.map((x) => x.author), ["agent"]);
+
+ assert.match((await run(["priv/polemic-x"], opts)).text, /1 open, 1 closed/);
+ assert.match((await run(["priv/polemic-x", "--resolved"], opts)).text, /Changed to 'claimed'/);
+ assert.equal((await run(["reopen", a.note.id], opts)).code, 0);
+ const list = await run(["--all"], opts);
+ assert.match(list.text, /priv\/polemic-x\s+article\s+2 open \/ 2/);
+
+ // refusals exit 2 and write nothing
+ assert.equal((await run(["reply", "n_nosuchnote"], opts)).code, 2);
+ assert.equal((await run(["reply", a.note.id], opts)).code, 2);
+ assert.equal((await run(["priv/../etc"], opts)).code, 2);
+ await rm(root, { recursive: true });
+});
+
+test("a video project's digest names the generator and lists the takes' verdicts", async () => {
+ const { root, reports, opts } = await fixture();
+ const proj = path.join(reports, "ws", "polemic-x");
+ await mkdir(path.join(proj, "takes", "deck"), { recursive: true });
+ await writeFile(path.join(proj, "takes", "deck", "take.json"), JSON.stringify({ id: "deck", group: "open", order: 1, label: "Deck first", kind: "similar", preview: "preview.mp4" }));
+ await writeFile(path.join(proj, "takes", "verdicts.json"), JSON.stringify({ deck: { verdict: "like", note: "keep the beat", at: "x" } }));
+ const t = await projectTarget("ws/polemic-x", opts);
+ await writeNote(t, { op: "add", text: "Cut this.", anchor: { kind: "entry", entry: "e1" } }, { by: "operator" });
+ await writeNote(t, { op: "add", text: "quote changed", anchor: { kind: "edit", entry: "e1", field: "quote", from: "a", to: "b" } }, { by: "operator" });
+ const d = await run(["ws/polemic-x"], opts);
+ assert.equal(d.code, 0, d.text);
+ assert.match(d.text, /video\.manifest\.json` is regenerated by `.*ws\/polemics\/make-site\.py`/);
+ assert.match(d.text, /timeline entry `e1` \(clip\) “Denial”/);
+ assert.match(d.text, /`e1\.quote`: "a" → "b"/);
+ assert.match(d.text, /- `deck` “Deck first” \(open, similar\): like — keep the beat/);
+ await rm(root, { recursive: true });
+});
diff --git a/umtool/lib/annotations/digest.mjs b/umtool/lib/annotations/digest.mjs
@@ -0,0 +1,207 @@
+// Notes as an AGENT reads them: markdown, every anchor resolved to something a
+// reader with no page open can act on, and the file to edit named first.
+//
+// `umtool notes <target>` prints this; GET /api/notes/context serves the same
+// text, so "Copy agent brief" on a page and an agent's CLI hand over the same
+// words. An anchor is resolved against what is on disk NOW:
+//
+// text the section's title and the sentence holding the quote, found
+// again with the same re-anchoring the page uses (ORPHANED when the
+// quote is gone -- the note still prints, with its quote)
+// cite the citation's quote, speaker, date and `<channel>/<id>@start-end`
+// moment the time, and the entry/source it resolved to when it was written
+// entry the timeline entry's title and quote
+// take the take's label and summary, and the operator's verdict on it
+// edit what changed, from → to, to port into the generator's inputs
+//
+// A video project's digest also lists every take with its verdict and note
+// (takes/verdicts.json): the agent that rendered the takes reads them here.
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { REPORTS_ROOT, SITES_DIR } from "../paths.mjs";
+import { listTakes, readVerdicts } from "../report/takes.mjs";
+import { locateQuote, sentenceAround } from "./anchor.mjs";
+
+const readJson = (file) => readFile(/* turbopackIgnore: true */ file, "utf8").then(JSON.parse, () => null);
+
+/** Markdown to the plain text a reader sees: links to their labels, emphasis and code marks dropped. */
+export function plainText(md) {
+ return String(md ?? "")
+ .replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1")
+ .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
+ .replace(/^#{1,6}\s+/gm, "")
+ .replace(/^\s*>\s?/gm, "")
+ .replace(/^\s*[-*+]\s+/gm, "")
+ .replace(/(\*\*|__|\*|_|`)/g, "");
+}
+
+/**
+ * The plain text of one block of a report, as the page renders it: a section's
+ * title, body and claims; or the title/subtitle/summary/method.
+ */
+export function blockText(report, block) {
+ if (!report) return { title: block, text: "" };
+ if (["title", "subtitle", "summary", "method"].includes(block)) {
+ return { title: block, text: plainText(report[block] ?? "") };
+ }
+ const s = (report.sections ?? []).find((x) => x.id === block);
+ if (!s) return null;
+ const parts = [s.title, plainText(s.body ?? "")];
+ for (const c of s.claims ?? []) parts.push(c.title ?? "", plainText(c.text), plainText(c.findings ?? ""));
+ return { title: s.title, text: parts.filter(Boolean).join("\n") };
+}
+
+const clip = (s, n = 300) => {
+ const t = String(s ?? "").replace(/\s+/g, " ").trim();
+ return t.length > n ? `${t.slice(0, n - 1)}…` : t;
+};
+const quoteLine = (s) => `> ${clip(s, 400)}`;
+const hms = (t) => {
+ const s = Math.max(0, Math.round(Number(t) || 0));
+ const h = Math.floor(s / 3600);
+ const m = Math.floor((s % 3600) / 60);
+ const ss = String(s % 60).padStart(2, "0");
+ return h ? `${h}:${String(m).padStart(2, "0")}:${ss}` : `${m}:${ss}`;
+};
+
+/** One anchor, resolved, as markdown lines. */
+function anchorLines(a, ctx) {
+ const { report, manifest, takes, verdicts } = ctx;
+ switch (a.kind) {
+ case "whole":
+ return [ctx.kind === "article" ? "**On:** the whole article" : "**On:** the whole project"];
+ case "section": {
+ const b = blockText(report, a.section);
+ return [`**On:** section “${b?.title ?? a.section}”${b ? "" : " (section no longer exists)"}`];
+ }
+ case "text": {
+ const b = blockText(report, a.section);
+ if (!b) return [`**On:** text in section \`${a.section}\` — ORPHANED (section no longer exists)`, quoteLine(a.quote)];
+ const hit = locateQuote(b.text, a);
+ if (!hit.found) return [`**On:** text in “${b.title}” — ORPHANED (quote no longer in the section)`, quoteLine(a.quote)];
+ const exact = b.text.slice(hit.start, hit.end);
+ return [
+ `**On:** “${clip(exact, 200)}” in “${b.title}”${hit.how === "exact" ? "" : ` (found ${hit.how})`}`,
+ quoteLine(sentenceAround(b.text, hit.start, hit.end)),
+ ];
+ }
+ case "cite": {
+ const c = report?.citations?.[a.cite];
+ if (!c) return [`**On:** citation \`${a.cite}\` (no longer in the report)`];
+ const who = [c.speaker, c.date].filter(Boolean).join(", ");
+ const where =
+ c.kind === "video" || c.kind === "audio"
+ ? `${c.channel}/${c.id}@${c.start}-${c.end}`
+ : c.kind === "post"
+ ? `post ${c.channel}/${c.id}`
+ : c.kind === "page"
+ ? c.url
+ : `source ${c.source}`;
+ return [`**On:** citation \`${a.cite}\`${c.label ? ` “${c.label}”` : ""} — ${where}${who ? ` (${who})` : ""}`, quoteLine(c.quote)];
+ }
+ case "moment": {
+ const r = a.resolved ?? {};
+ const take = a.take ? ` of take \`${a.take}\`` : "";
+ const lines = [`**On:** ${hms(a.t)} in \`${a.file}\`${take}${r.approx ? " (approximate)" : ""}`];
+ const entry = a.entry ?? r.entry;
+ if (entry || r.title) lines.push(`entry \`${entry ?? "?"}\`${r.title ? ` “${clip(r.title, 160)}”` : ""}`);
+ if (r.quote) lines.push(quoteLine(r.quote));
+ if (r.channel && r.video) lines.push(`source ${r.channel}/${r.video}${r.sourceT !== undefined ? ` @ ${hms(r.sourceT)}` : ""}${r.url ? ` — ${r.url}` : ""}`);
+ return lines;
+ }
+ case "entry": {
+ const e = (manifest?.timeline ?? []).find((x) => x.id === a.entry);
+ if (!e) return [`**On:** timeline entry \`${a.entry}\` (no longer in the manifest)`];
+ const title = e.title ?? e.onscreen?.title;
+ const lines = [`**On:** timeline entry \`${a.entry}\` (${e.type ?? "entry"})${title ? ` “${clip(title, 160)}”` : ""}`];
+ if (e.quote) lines.push(quoteLine(e.quote));
+ if (e.type === "clip" && e.channel && e.video) lines.push(`source ${e.channel}/${e.video}@${e.start}-${e.end}`);
+ return lines;
+ }
+ case "take": {
+ const t = takes?.find((x) => x.id === a.take);
+ const v = verdicts?.[a.take];
+ const lines = [`**On:** take \`${a.take}\`${t ? ` “${t.label}” (${t.group})` : " (no longer in takes/)"}${v?.verdict ? ` — verdict: ${v.verdict}` : ""}`];
+ if (t?.summary) lines.push(`summary: ${clip(t.summary, 300)}`);
+ return lines;
+ }
+ case "edit":
+ return [
+ `**On:** an edit made in umtool to a GENERATED manifest — port it into the generator's inputs`,
+ `\`${a.entry ? `${a.entry}.` : ""}${a.field}\`: ${clip(JSON.stringify(a.from), 300)} → ${clip(JSON.stringify(a.to), 300)}`,
+ ];
+ default:
+ return [`**On:** ${JSON.stringify(a)}`];
+ }
+}
+
+/** "Edit X; Y is regenerated by Z" -- the line an agent most needs. */
+export function sourceLine(kind, source) {
+ if (!source) return kind === "article" ? "**Source:** unknown — find the draft before editing report.json, which a generator may overwrite." : "**Source:** the manifest.";
+ if (kind === "article") {
+ if (source.draft && source.generator) return `**Source:** edit \`${source.draft}\`; \`report.json\` is regenerated by \`${source.generator}\`.`;
+ if (source.draft) return `**Source:** edit \`${source.draft}\`.`;
+ if (source.generator) return `**Source:** \`report.json\` is written by \`${source.generator}\`; edit its inputs.`;
+ } else {
+ if (source.generator) return `**Source:** \`${source.manifest}\` is regenerated by \`${source.generator}\`; edit its inputs (BEATS, drafts), not the manifest.`;
+ if (source.manifest) return `**Source:** edit \`${source.manifest}\`.`;
+ }
+ return `**Source:** ${source.how ?? "unknown"}`;
+}
+
+const STATUS = { open: (n) => n.status === "open", resolved: (n) => n.status !== "open", all: () => true };
+
+/**
+ * The digest of one notes file.
+ *
+ * @param {{ kind: "article" | "video-project", id: string, file: string, doc: any, error?: string }} entry
+ * @param {{ status?: "open" | "resolved" | "all", sitesDir?: string, reportsRoot?: string, projectDir?: string }} [opts]
+ */
+export async function digest(entry, { status = "open", sitesDir = SITES_DIR, reportsRoot = REPORTS_ROOT, projectDir } = {}) {
+ const lines = [];
+ const doc = entry.doc;
+ const ctx = { kind: entry.kind, report: null, manifest: null, takes: null, verdicts: null };
+ let heading;
+ if (entry.kind === "article") {
+ const [site, report] = entry.id.split("/");
+ ctx.report = await readJson(path.join(/* turbopackIgnore: true */ sitesDir, site, "reports", report, "report.json"));
+ heading = `# Notes on ${entry.id}${ctx.report?.title ? ` — ${ctx.report.title}` : ""}`;
+ } else {
+ const dir = projectDir ?? path.dirname(/* turbopackIgnore: true */ entry.file);
+ ctx.manifest = await readJson(path.join(/* turbopackIgnore: true */ dir, "video.manifest.json"));
+ const t = await listTakes(dir);
+ ctx.takes = t.takes;
+ ctx.verdicts = await readVerdicts(dir);
+ heading = `# Notes on ${entry.id}${ctx.manifest?.title ? ` — ${ctx.manifest.title}` : ""}`;
+ }
+ lines.push(heading, "");
+ lines.push(`file: \`${entry.file}\``);
+ if (entry.error) {
+ lines.push("", `**notes.json does not parse:** ${entry.error}. Nothing here may write it until it is fixed by hand.`);
+ return lines.join("\n");
+ }
+ lines.push(sourceLine(entry.kind, doc?.source));
+ if (doc?.source?.how) lines.push(`(${doc.source.how})`);
+ const all = doc?.notes ?? [];
+ const shown = all.filter(STATUS[status] ?? STATUS.open);
+ const open = all.filter((n) => n.status === "open").length;
+ lines.push("", `${open} open, ${all.length - open} closed${status === "all" ? "" : `; showing ${status}`}.`);
+ for (const n of shown) {
+ lines.push("", `## ${n.id} — ${n.status}${n.status !== "open" && n.resolvedBy ? ` by ${n.resolvedBy}` : ""} (${n.author}, ${n.at.slice(0, 16).replace("T", " ")})`);
+ lines.push(...anchorLines(n.anchor, ctx));
+ lines.push("", n.text);
+ for (const r of n.replies) lines.push("", `- **${r.author}** (${r.at.slice(0, 16).replace("T", " ")}): ${r.text.replace(/\n/g, "\n ")}`);
+ }
+ if (entry.kind === "video-project" && ctx.takes?.length) {
+ lines.push("", "## Takes (takes/verdicts.json)");
+ for (const t of ctx.takes) {
+ const v = ctx.verdicts?.[t.id];
+ lines.push(`- \`${t.id}\` “${t.label}” (${t.group}, ${t.kind}): ${v?.verdict ?? "no verdict"}${v?.note ? ` — ${clip(v.note, 400)}` : ""}`);
+ }
+ }
+ if (shown.length) {
+ lines.push("", "Act on a note by editing the SOURCE above and regenerating, then:");
+ lines.push(`\`umtool notes reply <id> "what you changed" --resolve\` (or \`umtool notes reply <id> "question"\` to ask).`);
+ }
+ return lines.join("\n");
+}
diff --git a/umtool/lib/annotations/server.ts b/umtool/lib/annotations/server.ts
@@ -0,0 +1,27 @@
+import { projectRef } from "@/lib/projects";
+import { articleTarget, projectTargetFor, TargetError } from "./targets.mjs";
+
+// The app's side of lib/annotations/targets.mjs: a request's `article` or
+// `project` parameter to a target, using the app's memoised project walk.
+// Everything about WHERE a note may be written is decided in targets.mjs.
+
+export type Target = Awaited<ReturnType<typeof articleTarget>> | Awaited<ReturnType<typeof projectTargetFor>>;
+
+export async function targetFrom(params: { article?: string | null; project?: string | null }): Promise<Target> {
+ if (params.article && params.project) throw new TargetError("pass article or project, not both");
+ if (params.article) return articleTarget(params.article);
+ if (params.project) {
+ const p = await projectRef(params.project);
+ if (!p) throw new TargetError(`no project ${params.project}`, 404);
+ return projectTargetFor(p);
+ }
+ throw new TargetError("pass ?article=<site>/<report> or ?project=<id>");
+}
+
+/** A thrown error as a response: typed refusals keep their status, the rest are 500s. */
+export function errorResponse(err: unknown): Response {
+ const status = typeof (err as { status?: unknown })?.status === "number" ? (err as { status: number }).status : null;
+ const name = (err as Error)?.name;
+ const code = status ?? (name === "NoteError" ? 400 : 500);
+ return Response.json({ error: err instanceof Error ? err.message : String(err) }, { status: code });
+}
diff --git a/umtool/lib/annotations/shape.mjs b/umtool/lib/annotations/shape.mjs
@@ -0,0 +1,302 @@
+// notes.json: its constants, its validation and the edits made to it -- PURE
+// (no node imports), so the store (./store.mjs, node), the CLI and a client
+// component all hold the same rules. ./types.ts gives them types.
+//
+// { "format": "umtool-notes", "version": 1,
+// "subject": { kind: "article", site, report } | { kind: "video-project", project },
+// "source": { draft?, generator?, manifest?, how? }, which file to EDIT
+// "notes": [ { id, status, author, text, at, updatedAt, anchor, replies,
+// resolvedAt?, resolvedBy? } ] }
+//
+// docs/notes.md is the prose.
+
+export const NOTES_FORMAT = "umtool-notes";
+export const NOTES_VERSION = 1;
+export const NOTE_TEXT_LIMIT = 8000;
+export const NOTE_STATUSES = ["open", "resolved", "wontfix"];
+export const NOTE_AUTHORS = ["operator", "agent"];
+export const ANCHOR_KINDS = ["text", "cite", "section", "whole", "moment", "entry", "take", "edit"];
+export const REPORT_BLOCKS = ["title", "subtitle", "summary", "method"];
+
+const ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+const TAKE_ID = /^[a-z0-9][a-z0-9-]{0,63}$/;
+const NOTE_ID = /^n_[a-z0-9]{4,32}$/;
+const isStr = (v) => typeof v === "string";
+const short = (v, max) => isStr(v) && v.length <= max;
+
+export const isNoteId = (v) => isStr(v) && NOTE_ID.test(v);
+
+/** A fresh note id: time then randomness, base36. */
+export function newNoteId(now = Date.now()) {
+ return `n_${now.toString(36)}${Math.random().toString(36).slice(2, 6).padEnd(4, "0")}`;
+}
+
+/** A moment's rel path: relative, no `..`, no empty segment, no backslash. */
+function relFile(v) {
+ return short(v, 512) && v.length > 0 && !v.startsWith("/") && !/[\\\0]/.test(v) && !v.split("/").some((s) => s === ".." || s === "" || s === ".");
+}
+
+/** A JSON value small enough to keep: an edit's from/to. */
+function smallJson(v) {
+ if (v === undefined) return true;
+ try {
+ return JSON.stringify(v).length <= 8000;
+ } catch {
+ return false;
+ }
+}
+
+const RESOLVED_STR = ["entry", "title", "quote", "channel", "video", "url"];
+
+/**
+ * An anchor as stored, or `{ error }`. Extra keys are dropped; a text anchor's
+ * prefix/suffix default to "".
+ *
+ * @param {unknown} raw
+ * @returns {{ anchor: Record<string, unknown> } | { error: string }}
+ */
+export function validateAnchor(raw) {
+ if (!raw || typeof raw !== "object" || Array.isArray(raw)) return { error: "anchor is not an object" };
+ const a = /** @type {Record<string, unknown>} */ (raw);
+ switch (a.kind) {
+ case "whole":
+ return { anchor: { kind: "whole" } };
+ case "section":
+ if (!short(a.section, 128) || !ID.test(a.section)) return { error: "section anchor needs a section id" };
+ return { anchor: { kind: "section", section: a.section } };
+ case "text": {
+ if (!short(a.section, 128) || !ID.test(a.section)) return { error: "text anchor needs a section id" };
+ if (!short(a.quote, 4000) || !a.quote.trim()) return { error: "text anchor needs a quote" };
+ if (a.prefix !== undefined && !short(a.prefix, 256)) return { error: "prefix is not a short string" };
+ if (a.suffix !== undefined && !short(a.suffix, 256)) return { error: "suffix is not a short string" };
+ return { anchor: { kind: "text", section: a.section, quote: a.quote, prefix: a.prefix ?? "", suffix: a.suffix ?? "" } };
+ }
+ case "cite":
+ if (!short(a.cite, 128) || !ID.test(a.cite)) return { error: "cite anchor needs a citation id" };
+ return { anchor: { kind: "cite", cite: a.cite } };
+ case "entry":
+ if (!short(a.entry, 128) || !ID.test(a.entry)) return { error: "entry anchor needs an entry id" };
+ return { anchor: { kind: "entry", entry: a.entry } };
+ case "take":
+ if (!isStr(a.take) || !TAKE_ID.test(a.take)) return { error: "take anchor needs a take id" };
+ return { anchor: { kind: "take", take: a.take } };
+ case "moment": {
+ if (!relFile(a.file)) return { error: "moment anchor needs a relative file" };
+ const t = Number(a.t);
+ if (!Number.isFinite(t) || t < 0) return { error: "moment anchor needs t ≥ 0" };
+ const out = { kind: "moment", file: a.file, t: Number(t.toFixed(2)) };
+ if (a.take !== undefined) {
+ if (!isStr(a.take) || !TAKE_ID.test(a.take)) return { error: "moment take is not a take id" };
+ out.take = a.take;
+ }
+ if (a.entry !== undefined) {
+ if (!short(a.entry, 128) || !ID.test(a.entry)) return { error: "moment entry is not an entry id" };
+ out.entry = a.entry;
+ }
+ if (a.resolved !== undefined) {
+ if (!a.resolved || typeof a.resolved !== "object" || Array.isArray(a.resolved)) return { error: "resolved is not an object" };
+ const r = /** @type {Record<string, unknown>} */ (a.resolved);
+ const res = {};
+ for (const k of RESOLVED_STR) if (short(r[k], 2000)) res[k] = r[k];
+ if (typeof r.sourceT === "number" && Number.isFinite(r.sourceT)) res.sourceT = Number(r.sourceT.toFixed(2));
+ if (r.approx === true) res.approx = true;
+ out.resolved = res;
+ }
+ return { anchor: out };
+ }
+ case "edit": {
+ if (!short(a.field, 128) || !a.field) return { error: "edit anchor needs a field" };
+ if (a.entry !== undefined && (!short(a.entry, 128) || !ID.test(a.entry))) return { error: "edit entry is not an entry id" };
+ if (!smallJson(a.from) || !smallJson(a.to)) return { error: "edit from/to too large" };
+ const out = { kind: "edit", field: a.field, from: a.from ?? null, to: a.to ?? null };
+ if (a.entry !== undefined) out.entry = a.entry;
+ return { anchor: out };
+ }
+ default:
+ return { error: `anchor kind must be one of ${ANCHOR_KINDS.join(", ")}` };
+ }
+}
+
+/** @returns {{ subject: Record<string, string> } | { error: string }} */
+export function validateSubject(raw) {
+ if (!raw || typeof raw !== "object") return { error: "subject is not an object" };
+ const s = /** @type {Record<string, unknown>} */ (raw);
+ if (s.kind === "article" && isStr(s.site) && TAKE_ID.test(s.site) && isStr(s.report) && TAKE_ID.test(s.report)) {
+ return { subject: { kind: "article", site: s.site, report: s.report } };
+ }
+ if (s.kind === "video-project" && short(s.project, 512) && s.project.length > 0) {
+ return { subject: { kind: "video-project", project: s.project } };
+ }
+ return { error: "subject must be an article {site, report} or a video-project {project}" };
+}
+
+function cleanSource(raw) {
+ if (!raw || typeof raw !== "object" || Array.isArray(raw)) return undefined;
+ const out = {};
+ for (const k of ["draft", "generator", "manifest", "how"]) if (short(raw[k], 1000) && raw[k]) out[k] = raw[k];
+ return Object.keys(out).length ? out : undefined;
+}
+
+function cleanText(v) {
+ if (!isStr(v)) throw new NoteError("text must be a string");
+ const t = v.replace(/\s+$/, "");
+ if (!t.trim()) throw new NoteError("text is empty");
+ if (t.length > NOTE_TEXT_LIMIT) throw new NoteError(`text is over ${NOTE_TEXT_LIMIT} characters`);
+ return t;
+}
+
+/** A refused edit: the caller's fault, a 400. */
+export class NoteError extends Error {
+ constructor(message) {
+ super(message);
+ this.name = "NoteError";
+ }
+}
+
+/**
+ * A parsed notes.json, checked. `{ doc }` or `{ error }`. A note that is not
+ * the contract's shape is an error for the whole file -- the file is never
+ * "repaired" by dropping it, because the next write would erase it.
+ */
+export function parseNotesDoc(raw) {
+ if (!raw || typeof raw !== "object" || Array.isArray(raw)) return { error: "not an object" };
+ if (raw.format !== NOTES_FORMAT) return { error: `format is not ${NOTES_FORMAT}` };
+ if (raw.version !== NOTES_VERSION) return { error: `version ${raw.version} is not ${NOTES_VERSION}` };
+ const subject = validateSubject(raw.subject);
+ if ("error" in subject) return subject;
+ if (!Array.isArray(raw.notes)) return { error: "notes is not a list" };
+ const notes = [];
+ for (const [i, n] of raw.notes.entries()) {
+ const where = `notes[${i}]`;
+ if (!n || typeof n !== "object") return { error: `${where} is not an object` };
+ if (!isNoteId(n.id)) return { error: `${where}.id is not a note id` };
+ if (!NOTE_STATUSES.includes(n.status)) return { error: `${where}.status` };
+ if (!NOTE_AUTHORS.includes(n.author)) return { error: `${where}.author` };
+ if (!isStr(n.text) || !isStr(n.at) || !isStr(n.updatedAt)) return { error: `${where} text/at/updatedAt` };
+ const a = validateAnchor(n.anchor);
+ if ("error" in a) return { error: `${where}.anchor: ${a.error}` };
+ if (!Array.isArray(n.replies)) return { error: `${where}.replies is not a list` };
+ const replies = [];
+ for (const r of n.replies) {
+ if (!r || !NOTE_AUTHORS.includes(r.author) || !isStr(r.text) || !isStr(r.at)) return { error: `${where}.replies` };
+ replies.push({ author: r.author, text: r.text, at: r.at });
+ }
+ const note = { id: n.id, status: n.status, author: n.author, text: n.text, at: n.at, updatedAt: n.updatedAt, anchor: a.anchor, replies };
+ if (isStr(n.resolvedAt)) note.resolvedAt = n.resolvedAt;
+ if (NOTE_AUTHORS.includes(n.resolvedBy)) note.resolvedBy = n.resolvedBy;
+ notes.push(note);
+ }
+ const doc = { format: NOTES_FORMAT, version: NOTES_VERSION, subject: subject.subject };
+ const source = cleanSource(raw.source);
+ if (source) doc.source = source;
+ doc.notes = notes;
+ return { doc };
+}
+
+export function emptyDoc(subject, source) {
+ const doc = { format: NOTES_FORMAT, version: NOTES_VERSION, subject };
+ const s = cleanSource(source);
+ if (s) doc.source = s;
+ doc.notes = [];
+ return doc;
+}
+
+function find(doc, id) {
+ const note = doc.notes.find((n) => n.id === id);
+ if (!note) throw new NoteError(`no note ${id}`);
+ return note;
+}
+
+function setStatus(note, status, by, now) {
+ if (!NOTE_STATUSES.includes(status)) throw new NoteError(`status must be one of ${NOTE_STATUSES.join(", ")}`);
+ note.status = status;
+ note.updatedAt = now;
+ if (status === "open") {
+ delete note.resolvedAt;
+ delete note.resolvedBy;
+ } else {
+ note.resolvedAt = now;
+ note.resolvedBy = by;
+ }
+}
+
+/**
+ * Apply one op to a doc, in place. Returns the note touched (null on delete).
+ * `by` is who is writing: the app stamps "operator", the CLI "agent".
+ *
+ * { op: "add", text, anchor } a new open note
+ * { op: "edit", id, text?, anchor? } rewrite it (its author only)
+ * { op: "status", id, status } open | resolved | wontfix
+ * { op: "reply", id, text, resolve? } a threaded reply, optionally resolving
+ * { op: "delete", id } remove the note
+ * { op: "delete-reply", id, index } remove one reply
+ * { op: "source", source } correct which file to edit
+ *
+ * @param {Record<string, any>} doc
+ * @param {Record<string, any>} op
+ * @param {"operator" | "agent"} by
+ * @param {string} [now]
+ */
+export function applyOp(doc, op, by, now = new Date().toISOString()) {
+ if (!NOTE_AUTHORS.includes(by)) throw new NoteError("author must be operator or agent");
+ switch (op?.op) {
+ case "add": {
+ const a = validateAnchor(op.anchor);
+ if ("error" in a) throw new NoteError(a.error);
+ let id = newNoteId();
+ while (doc.notes.some((n) => n.id === id)) id = newNoteId();
+ const note = { id, status: "open", author: by, text: cleanText(op.text), at: now, updatedAt: now, anchor: a.anchor, replies: [] };
+ doc.notes.push(note);
+ return note;
+ }
+ case "edit": {
+ const note = find(doc, op.id);
+ if (note.author !== by) throw new NoteError(`only the ${note.author} edits this note; reply instead`);
+ if (op.text !== undefined) note.text = cleanText(op.text);
+ if (op.anchor !== undefined) {
+ const a = validateAnchor(op.anchor);
+ if ("error" in a) throw new NoteError(a.error);
+ note.anchor = a.anchor;
+ }
+ note.updatedAt = now;
+ return note;
+ }
+ case "status": {
+ const note = find(doc, op.id);
+ setStatus(note, op.status, by, now);
+ return note;
+ }
+ case "reply": {
+ const note = find(doc, op.id);
+ note.replies.push({ author: by, text: cleanText(op.text), at: now });
+ note.updatedAt = now;
+ if (op.resolve) setStatus(note, "resolved", by, now);
+ return note;
+ }
+ case "delete": {
+ const i = doc.notes.findIndex((n) => n.id === op.id);
+ if (i === -1) throw new NoteError(`no note ${op.id}`);
+ doc.notes.splice(i, 1);
+ return null;
+ }
+ case "delete-reply": {
+ const note = find(doc, op.id);
+ const i = Number(op.index);
+ if (!Number.isInteger(i) || i < 0 || i >= note.replies.length) throw new NoteError("no such reply");
+ if (note.replies[i].author !== by) throw new NoteError(`only the ${note.replies[i].author} deletes that reply`);
+ note.replies.splice(i, 1);
+ note.updatedAt = now;
+ return note;
+ }
+ case "source": {
+ const s = cleanSource(op.source);
+ if (s) doc.source = s;
+ else delete doc.source;
+ return null;
+ }
+ default:
+ throw new NoteError("op must be add, edit, status, reply, delete, delete-reply or source");
+ }
+}
+
+export const openNotes = (doc) => (doc ? doc.notes.filter((n) => n.status === "open") : []);
diff --git a/umtool/lib/annotations/store.mjs b/umtool/lib/annotations/store.mjs
@@ -0,0 +1,157 @@
+// notes.json on disk: read, lock, apply one op, write. Shared by the app's
+// /api/notes and `umtool notes`, so the operator's page and an agent's CLI go
+// through ONE writer with one set of rules (./shape.mjs).
+//
+// Three writers can race on one file -- the page, an agent's `umtool notes
+// reply`, a second agent -- and they are different PROCESSES, so the in-process
+// queues lib/state.ts and lib/report/manifest.mjs use are not enough here:
+//
+// * a LOCKFILE (`notes.json.lock`, created O_EXCL) serialises
+// read-modify-write across processes; one left by a dead writer is stale
+// after 30 s and is taken over;
+// * the write is tmp + rename, so a reader never sees half a file;
+// * every write from the page carries the TOKEN it read (the file's mtime in
+// ns, or "absent"); a stale one is a 409, never a silent overwrite of a
+// reply an agent wrote in between;
+// * a notes.json that exists and does not parse is NEVER overwritten (the
+// takes.mjs rule) -- writing over it would erase every note in it;
+// * the last note deleted deletes the file: an empty notes.json says nothing.
+//
+// WHERE a write may land is the caller's to decide BEFORE it gets here
+// (lib/annotations/targets.mjs: isCorpusNotesFile for an article, a project
+// directory for a video) -- `writeOp` takes the file it is given.
+import { open, readFile, rename, stat, unlink, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { NoteError, applyOp, emptyDoc, parseNotesDoc } from "./shape.mjs";
+
+export { NoteError };
+export const LOCK_STALE_MS = 30_000;
+const LOCK_WAIT_MS = 10_000;
+
+/** A write whose token no longer matches the file: someone wrote in between. */
+export class StaleNotes extends Error {
+ constructor(expected, got) {
+ super(`the notes changed since you read them (${got} vs ${expected})`);
+ this.name = "StaleNotes";
+ this.status = 409;
+ }
+}
+
+/** A notes.json that exists and is not one: refused, never overwritten. */
+export class NotesUnreadable extends Error {
+ constructor(file, why) {
+ super(`${path.basename(file)} does not parse (${why}); not overwriting it`);
+ this.name = "NotesUnreadable";
+ this.status = 409;
+ }
+}
+
+/** The file's identity for a write guard: mtime in ns plus size, or "absent". */
+export async function notesToken(file) {
+ const st = await stat(/* turbopackIgnore: true */ file, { bigint: true }).catch(() => null);
+ return st ? `${st.mtimeNs}-${st.size}` : "absent";
+}
+
+/**
+ * `{ doc, token }` -- doc null when there is no file. `{ error }` too when the
+ * file is there and is not a notes doc (the page shows it; writes refuse).
+ *
+ * @param {string} file
+ * @returns {Promise<{ doc: import("./types").NotesDoc | null, token: string, error?: string }>}
+ */
+export async function readNotes(file) {
+ const token = await notesToken(file);
+ let text;
+ try {
+ text = await readFile(/* turbopackIgnore: true */ file, "utf8");
+ } catch (err) {
+ if (/** @type {NodeJS.ErrnoException} */ (err).code === "ENOENT") return { doc: null, token: "absent" };
+ return { doc: null, token, error: String(/** @type {Error} */ (err).message ?? err) };
+ }
+ let raw;
+ try {
+ raw = JSON.parse(text);
+ } catch (err) {
+ return { doc: null, token, error: `not JSON: ${/** @type {Error} */ (err).message}` };
+ }
+ const r = parseNotesDoc(raw);
+ if ("error" in r) return { doc: null, token, error: r.error };
+ return { doc: r.doc, token };
+}
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+/**
+ * Run `fn` holding `<file>.lock`. A lock older than LOCK_STALE_MS is a dead
+ * writer's and is removed; otherwise wait (up to 10 s) and retry.
+ *
+ * @template T
+ * @param {string} file
+ * @param {() => Promise<T>} fn
+ * @param {{ waitMs?: number, staleMs?: number }} [opts]
+ * @returns {Promise<T>}
+ */
+export async function withNotesLock(file, fn, { waitMs = LOCK_WAIT_MS, staleMs = LOCK_STALE_MS } = {}) {
+ const lock = `${file}.lock`;
+ const deadline = Date.now() + waitMs;
+ let delay = 15;
+ for (;;) {
+ try {
+ const h = await open(/* turbopackIgnore: true */ lock, "wx");
+ await h.writeFile(`${process.pid} ${new Date().toISOString()}\n`).catch(() => {});
+ await h.close();
+ break;
+ } catch (err) {
+ if (/** @type {NodeJS.ErrnoException} */ (err).code !== "EEXIST") throw err;
+ const st = await stat(/* turbopackIgnore: true */ lock).catch(() => null);
+ if (st && Date.now() - st.mtimeMs > staleMs) {
+ await unlink(/* turbopackIgnore: true */ lock).catch(() => {});
+ continue;
+ }
+ if (Date.now() > deadline) throw new Error(`${path.basename(lock)} is held; try again`);
+ await sleep(delay);
+ delay = Math.min(delay * 2, 250);
+ }
+ }
+ try {
+ return await fn();
+ } finally {
+ await unlink(/* turbopackIgnore: true */ lock).catch(() => {});
+ }
+}
+
+/**
+ * Apply one op to the notes at `file` and write it back. `init` is the
+ * subject (and source) a NEW file starts with; an existing file keeps its own
+ * subject, and its `source` unless the op is `source`. `token` (when given)
+ * must match the file as it is now, or StaleNotes. `by` is "operator" (the
+ * app) or "agent" (the CLI).
+ *
+ * Returns the doc as written (null when the file was deleted), the new token,
+ * and the note the op touched.
+ *
+ * @param {string} file
+ * @param {{ subject: Record<string, string>, source?: Record<string, string> | null }} init
+ * @param {Record<string, any>} op
+ * @param {{ by: "operator" | "agent", token?: string | null }} opts
+ * @returns {Promise<{ doc: import("./types").NotesDoc | null, token: string, note: import("./types").Note | null }>}
+ */
+export async function writeOp(file, init, op, { by, token = null }) {
+ return withNotesLock(file, async () => {
+ const current = await readNotes(file);
+ if (current.error) throw new NotesUnreadable(file, current.error);
+ if (token !== null && token !== undefined && token !== current.token) throw new StaleNotes(token, current.token);
+ const doc = /** @type {any} */ (current.doc ?? emptyDoc(init.subject, init.source ?? undefined));
+ const note = applyOp(doc, op, by);
+ if (doc.notes.length === 0) {
+ await unlink(/* turbopackIgnore: true */ file).catch((err) => {
+ if (err.code !== "ENOENT") throw err;
+ });
+ return { doc: null, token: "absent", note };
+ }
+ const tmp = `${file}.tmp-${process.pid}-${Math.random().toString(36).slice(2, 8)}`;
+ await writeFile(/* turbopackIgnore: true */ tmp, JSON.stringify(doc, null, 2) + "\n", "utf8");
+ await rename(/* turbopackIgnore: true */ tmp, file);
+ return { doc, token: await notesToken(file), note };
+ });
+}
diff --git a/umtool/lib/annotations/store.test.mjs b/umtool/lib/annotations/store.test.mjs
@@ -0,0 +1,206 @@
+// notes.json: the store, the corpus write predicate and the targets.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, readFile, rm, stat, symlink, utimes, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { corpusNotesFile, isCorpusNotesFile } from "../paths.mjs";
+import { NoteError, applyOp, emptyDoc, parseNotesDoc, validateAnchor } from "./shape.mjs";
+import { NotesUnreadable, StaleNotes, readNotes, withNotesLock, writeOp } from "./store.mjs";
+import { articleTarget, listNotesFiles, writeNote } from "./targets.mjs";
+
+const SUBJECT = { kind: "article", site: "s1", report: "r1" };
+
+async function tmp() {
+ return mkdtemp(path.join(tmpdir(), "umtool-notes-"));
+}
+
+test("add, reply, resolve, reopen, delete: round-trip, and the last delete removes the file", async () => {
+ const dir = await tmp();
+ const file = path.join(dir, "notes.json");
+ const a = await writeOp(file, { subject: SUBJECT, source: { draft: "~/d.json" } }, { op: "add", text: "fix this", anchor: { kind: "whole" } }, { by: "operator" });
+ assert.equal(a.doc.notes.length, 1);
+ assert.equal(a.doc.source.draft, "~/d.json");
+ const id = a.note.id;
+ assert.match(id, /^n_[a-z0-9]+$/);
+
+ const back = await readNotes(file);
+ assert.deepEqual(back.doc, a.doc);
+ assert.equal(back.token, a.token);
+
+ const r = await writeOp(file, { subject: SUBJECT }, { op: "reply", id, text: "done in drafts/x.json", resolve: true }, { by: "agent", token: a.token });
+ assert.equal(r.note.status, "resolved");
+ assert.equal(r.note.resolvedBy, "agent");
+ assert.equal(r.note.replies[0].author, "agent");
+
+ const o = await writeOp(file, { subject: SUBJECT }, { op: "status", id, status: "open" }, { by: "operator" });
+ assert.equal(o.note.status, "open");
+ assert.equal(o.note.resolvedAt, undefined);
+
+ const d = await writeOp(file, { subject: SUBJECT }, { op: "delete", id }, { by: "operator" });
+ assert.equal(d.doc, null);
+ await assert.rejects(stat(file), /ENOENT/);
+ await rm(dir, { recursive: true });
+});
+
+test("a stale token is a 409 and the file is untouched", async () => {
+ const dir = await tmp();
+ const file = path.join(dir, "notes.json");
+ const a = await writeOp(file, { subject: SUBJECT }, { op: "add", text: "one", anchor: { kind: "whole" } }, { by: "operator", token: "absent" });
+ await writeOp(file, { subject: SUBJECT }, { op: "add", text: "two (agent)", anchor: { kind: "whole" } }, { by: "agent" });
+ const before = await readFile(file, "utf8");
+ await assert.rejects(
+ writeOp(file, { subject: SUBJECT }, { op: "add", text: "three", anchor: { kind: "whole" } }, { by: "operator", token: a.token }),
+ (err) => err instanceof StaleNotes && err.status === 409,
+ );
+ assert.equal(await readFile(file, "utf8"), before);
+ // "absent" against a file that exists is stale too
+ await assert.rejects(
+ writeOp(file, { subject: SUBJECT }, { op: "add", text: "x", anchor: { kind: "whole" } }, { by: "operator", token: "absent" }),
+ StaleNotes,
+ );
+ await rm(dir, { recursive: true });
+});
+
+test("an unparseable notes.json is never overwritten", async () => {
+ const dir = await tmp();
+ const file = path.join(dir, "notes.json");
+ await writeFile(file, "{ not json");
+ assert.match((await readNotes(file)).error, /not JSON/);
+ await assert.rejects(writeOp(file, { subject: SUBJECT }, { op: "add", text: "x", anchor: { kind: "whole" } }, { by: "operator" }), NotesUnreadable);
+ assert.equal(await readFile(file, "utf8"), "{ not json");
+ // a parseable file with a bad note is refused the same way
+ await writeFile(file, JSON.stringify({ format: "umtool-notes", version: 1, subject: SUBJECT, notes: [{ id: "bad" }] }));
+ await assert.rejects(writeOp(file, { subject: SUBJECT }, { op: "add", text: "x", anchor: { kind: "whole" } }, { by: "operator" }), NotesUnreadable);
+ await rm(dir, { recursive: true });
+});
+
+test("lock contention: parallel writers all land; a held lock waits; a stale lock is taken over", async () => {
+ const dir = await tmp();
+ const file = path.join(dir, "notes.json");
+ await Promise.all(
+ Array.from({ length: 12 }, (_, i) =>
+ writeOp(file, { subject: SUBJECT }, { op: "add", text: `n${i}`, anchor: { kind: "whole" } }, { by: i % 2 ? "agent" : "operator" }),
+ ),
+ );
+ assert.equal((await readNotes(file)).doc.notes.length, 12);
+
+ // Another process holds it (a fresh lock file): we wait, then give up.
+ await writeFile(`${file}.lock`, "999999 now\n");
+ await assert.rejects(withNotesLock(file, async () => 1, { waitMs: 120 }), /held/);
+ // The same lock, 31 s old: a dead writer's, taken over.
+ const old = new Date(Date.now() - 31_000);
+ await utimes(`${file}.lock`, old, old);
+ assert.equal(await withNotesLock(file, async () => 2, { waitMs: 120 }), 2);
+ await assert.rejects(stat(`${file}.lock`), /ENOENT/);
+ await rm(dir, { recursive: true });
+});
+
+test("ops refuse what the contract does not allow", () => {
+ const doc = emptyDoc(SUBJECT);
+ assert.throws(() => applyOp(doc, { op: "add", text: " ", anchor: { kind: "whole" } }, "operator"), NoteError);
+ assert.throws(() => applyOp(doc, { op: "add", text: "x", anchor: { kind: "nope" } }, "operator"), NoteError);
+ assert.throws(() => applyOp(doc, { op: "add", text: "x".repeat(8001), anchor: { kind: "whole" } }, "operator"), NoteError);
+ const n = applyOp(doc, { op: "add", text: "x", anchor: { kind: "whole" } }, "operator");
+ assert.throws(() => applyOp(doc, { op: "edit", id: n.id, text: "agent rewrites it" }, "agent"), /reply instead/);
+ assert.throws(() => applyOp(doc, { op: "status", id: n.id, status: "done" }, "agent"), NoteError);
+ assert.throws(() => applyOp(doc, { op: "reply", id: "n_missing0", text: "x" }, "agent"), /no note/);
+ assert.throws(() => applyOp(doc, { op: "frobnicate" }, "agent"), NoteError);
+ assert.throws(() => applyOp(doc, { op: "add", text: "x", anchor: { kind: "whole" } }, "someone"), NoteError);
+ // source: set, then cleared
+ applyOp(doc, { op: "source", source: { draft: "~/x.json", junk: 1 } }, "agent");
+ assert.deepEqual(doc.source, { draft: "~/x.json" });
+ assert.ok(parseNotesDoc(JSON.parse(JSON.stringify(doc))).doc);
+});
+
+test("anchors: every kind validates; bad ones refuse", () => {
+ const ok = [
+ { kind: "whole" },
+ { kind: "section", section: "s-2" },
+ { kind: "text", section: "summary", quote: "the claim", prefix: "before ", suffix: " after" },
+ { kind: "cite", cite: "c12" },
+ { kind: "moment", file: "takes/deck/preview.mp4", t: 12.345, take: "deck", entry: "e3", resolved: { title: "T", sourceT: 81.234, approx: true, bogus: 1 } },
+ { kind: "entry", entry: "clip-4" },
+ { kind: "take", take: "cold-open" },
+ { kind: "edit", entry: "e3", field: "quote", from: "a", to: "b" },
+ ];
+ for (const a of ok) assert.ok("anchor" in validateAnchor(a), JSON.stringify(a));
+ assert.equal(validateAnchor(ok[4]).anchor.t, 12.35);
+ assert.deepEqual(validateAnchor(ok[4]).anchor.resolved, { title: "T", sourceT: 81.23, approx: true });
+ assert.equal(validateAnchor({ kind: "text", section: "s", quote: "q" }).anchor.prefix, "");
+ const bad = [
+ null,
+ { kind: "text", section: "s" },
+ { kind: "moment", file: "../x.mp4", t: 1 },
+ { kind: "moment", file: "/abs.mp4", t: 1 },
+ { kind: "moment", file: "a.mp4", t: -1 },
+ { kind: "take", take: "Bad Id" },
+ { kind: "section", section: "has space" },
+ { kind: "edit", field: "" },
+ ];
+ for (const a of bad) assert.ok("error" in validateAnchor(a), JSON.stringify(a));
+});
+
+async function sitesFixture() {
+ const root = await tmp();
+ const sites = path.join(root, "sites");
+ await mkdir(path.join(sites, "s1", "reports", "r1"), { recursive: true });
+ await writeFile(path.join(sites, "s1", "site.json"), "{}");
+ await writeFile(path.join(sites, "s1", "reports", "r1", "report.json"), "{}");
+ const outside = path.join(root, "outside", "r2");
+ await mkdir(outside, { recursive: true });
+ await symlink(outside, path.join(sites, "s1", "reports", "r2"));
+ return { root, sites };
+}
+
+test("isCorpusNotesFile: exactly sites/<site>/reports/<id>/notes.json, and nothing else", async () => {
+ const { root, sites } = await sitesFixture();
+ const opt = { sitesDir: sites };
+ const good = path.join(sites, "s1", "reports", "r1", "notes.json");
+ assert.equal(corpusNotesFile("s1", "r1", opt), good);
+ assert.equal(await isCorpusNotesFile(good, opt), true);
+ // traversal, spelled several ways
+ assert.equal(await isCorpusNotesFile(path.join(sites, "s1", "reports", "r1", "..", "r1", "notes.json").replace(/\/r1\/notes/, "/../r1/r1/notes"), opt), false);
+ assert.equal(await isCorpusNotesFile(`${sites}/s1/reports/../reports/r1/notes.json`, opt), false);
+ assert.equal(await isCorpusNotesFile(`${sites}/s1/reports//r1/notes.json`, opt), false);
+ assert.equal(await isCorpusNotesFile("s1/reports/r1/notes.json", opt), false);
+ // wrong name, wrong depth, wrong middle segment, bad ids
+ assert.equal(await isCorpusNotesFile(path.join(sites, "s1", "reports", "r1", "report.json"), opt), false);
+ assert.equal(await isCorpusNotesFile(path.join(sites, "s1", "reports", "r1", "x", "notes.json"), opt), false);
+ assert.equal(await isCorpusNotesFile(path.join(sites, "s1", "stills", "r1", "notes.json"), opt), false);
+ assert.equal(await isCorpusNotesFile(path.join(sites, "S1", "reports", "r1", "notes.json"), opt), false);
+ assert.equal(corpusNotesFile("s1", "../r1", opt), null);
+ // a report dir that is a symlink out of the site
+ assert.equal(await isCorpusNotesFile(path.join(sites, "s1", "reports", "r2", "notes.json"), opt), false);
+ // a report that does not exist: a note never creates its directory
+ assert.equal(await isCorpusNotesFile(path.join(sites, "s1", "reports", "r9", "notes.json"), opt), false);
+ // a notes.json that is itself a symlink
+ await writeFile(path.join(root, "elsewhere.json"), "{}");
+ await symlink(path.join(root, "elsewhere.json"), good);
+ assert.equal(await isCorpusNotesFile(good, opt), false);
+ await rm(root, { recursive: true });
+});
+
+test("articleTarget: writes land beside report.json with the discovered source; refusals are typed", async () => {
+ const { root, sites } = await sitesFixture();
+ const reports = path.join(root, "reports");
+ await mkdir(path.join(reports, "ws", "polemics", "drafts"), { recursive: true });
+ await writeFile(path.join(reports, "ws", "polemics", "drafts", "r1.json"), JSON.stringify({ id: "r1" }));
+ await writeFile(path.join(reports, "ws", "polemics", "make-site.py"), 'OUT = "sites/s1/reports"\nfor d in drafts: pass\n');
+ const opt = { sitesDir: sites, reportsRoot: reports };
+
+ const t = await articleTarget("s1/r1", opt);
+ const w = await writeNote(t, { op: "add", text: "x", anchor: { kind: "whole" } }, { by: "operator" });
+ assert.equal(w.doc.source.draft.endsWith(path.join("ws", "polemics", "drafts", "r1.json")), true);
+ assert.equal(w.doc.source.generator.endsWith("make-site.py"), true);
+ const list = await listNotesFiles(opt);
+ assert.deepEqual(list.map((l) => [l.kind, l.id, l.doc.notes.length]), [["article", "s1/r1", 1]]);
+
+ await assert.rejects(articleTarget("s1/r9", opt), (e) => e.status === 404);
+ await assert.rejects(articleTarget("s1/r2", opt), (e) => e.status === 403);
+ await assert.rejects(articleTarget("s1/../r1", opt), (e) => e.status === 400);
+ await assert.rejects(articleTarget("s1", opt), (e) => e.status === 400);
+ await rm(root, { recursive: true });
+});
diff --git a/umtool/lib/annotations/targets.mjs b/umtool/lib/annotations/targets.mjs
@@ -0,0 +1,185 @@
+// What a note is ON, and therefore which notes.json it lives in -- decided
+// here, once, for the app's /api/notes and for `umtool notes`.
+//
+// article `SITES_DIR/<site>/reports/<report>/notes.json`, beside
+// report.json. The generators that write a report dir
+// overwrite only report.json, video.mp4 and poster.jpg, so it
+// survives a regenerate; the compose stage and the report
+// history never read it (common/publish tests hold that), so
+// it is never published. The ONE corpus file umtool writes,
+// and only through isCorpusNotesFile.
+// video-project `<project>/notes.json`, beside video.manifest.json. A
+// report-video project under REPORTS_ROOT, found by the same
+// walk `umtool ls` uses.
+import { readdir, readFile, realpath, stat } from "node:fs/promises";
+import path from "node:path";
+import { NOTES_FILENAME, REPORTS_ROOT, SEGMENT_RE, SITES_DIR, corpusNotesFile, inside, isCorpusNotesFile } from "../paths.mjs";
+import { projectRefs, resolveProject } from "../projects/core.mjs";
+import { kindTakesNotes } from "../projects/kinds.mjs";
+import { sourceFor, tildify } from "../articles/sources.mjs";
+import { readNotes, writeOp } from "./store.mjs";
+
+const exists = (p) => stat(/* turbopackIgnore: true */ p).then(() => true, () => false);
+
+/** A refused target: the caller's fault (400/404). */
+export class TargetError extends Error {
+ constructor(message, status = 400) {
+ super(message);
+ this.name = "TargetError";
+ this.status = status;
+ }
+}
+
+/**
+ * An article target, checked: the site and report ids, the report directory
+ * on disk, and the notes path through isCorpusNotesFile.
+ *
+ * @param {string} spec `<site>/<report>`
+ * @param {{ sitesDir?: string, reportsRoot?: string }} [opts]
+ */
+export async function articleTarget(spec, { sitesDir = SITES_DIR, reportsRoot = REPORTS_ROOT } = {}) {
+ const [site, report, ...rest] = String(spec ?? "").split("/");
+ if (rest.length || !SEGMENT_RE.test(site ?? "") || !SEGMENT_RE.test(report ?? "")) {
+ throw new TargetError(`not an article: ${JSON.stringify(spec)} (want <site>/<report>)`);
+ }
+ const file = corpusNotesFile(site, report, { sitesDir });
+ if (!file || !(await exists(path.dirname(/* turbopackIgnore: true */ file)))) throw new TargetError(`no report ${site}/${report}`, 404);
+ if (!(await isCorpusNotesFile(file, { sitesDir }))) throw new TargetError(`refusing to write ${file}`, 403);
+ return {
+ kind: "article",
+ id: `${site}/${report}`,
+ file,
+ subject: { kind: "article", site, report },
+ // Lazy: the scan reads every workspace's generators, and a read of an
+ // existing file never needs it.
+ source: async () => sourceFor(site, report, { reportsRoot }),
+ };
+}
+
+/**
+ * A video-project target: a report-video project under REPORTS_ROOT, by id,
+ * unique name or directory.
+ *
+ * @param {string} spec
+ * @param {{ reportsRoot?: string }} [opts]
+ */
+export async function projectTarget(spec, { reportsRoot = REPORTS_ROOT } = {}) {
+ const r = await resolveProject(String(spec ?? ""), reportsRoot);
+ if (r.ambiguous) throw new TargetError(`${spec} names ${r.ambiguous.length} projects: ${r.ambiguous.map((p) => p.id).join(", ")}`);
+ if (!r.project) throw new TargetError(`no project ${spec}`, 404);
+ return projectTargetFor(r.project, { reportsRoot });
+}
+
+/**
+ * The target for a project ref already in hand (the app's memoised walk).
+ *
+ * @param {{ id: string, dir: string, kind: string }} p
+ * @param {{ reportsRoot?: string }} [opts]
+ */
+export async function projectTargetFor(p, { reportsRoot = REPORTS_ROOT } = {}) {
+ if (!kindTakesNotes(p.kind)) throw new TargetError(`${p.id} is a ${p.kind} project; notes are for report videos`);
+ const [realRoot, realDir] = await Promise.all([
+ realpath(/* turbopackIgnore: true */ reportsRoot).catch(() => null),
+ realpath(/* turbopackIgnore: true */ p.dir).catch(() => null),
+ ]);
+ if (!realRoot || !realDir || !inside(realRoot, realDir) || realRoot === realDir) {
+ throw new TargetError(`${p.id} is not under the reports root`, 403);
+ }
+ const file = path.join(/* turbopackIgnore: true */ p.dir, NOTES_FILENAME);
+ return {
+ kind: "video-project",
+ id: p.id,
+ dir: p.dir,
+ file,
+ subject: { kind: "video-project", project: p.id },
+ source: async () => projectSource(p.dir, reportsRoot),
+ };
+}
+
+/**
+ * A report-video project's source: its manifest, and -- when the manifest is
+ * generated -- the generator, resolved against the project's workspace (the
+ * first directory under REPORTS_ROOT) when that file exists.
+ */
+export async function projectSource(dir, reportsRoot = REPORTS_ROOT) {
+ const manifest = path.join(/* turbopackIgnore: true */ dir, "video.manifest.json");
+ const out = { manifest: tildify(manifest) };
+ let generatedBy = null;
+ try {
+ const m = JSON.parse(await readFile(/* turbopackIgnore: true */ manifest, "utf8"));
+ if (typeof m.generatedBy === "string" && m.generatedBy.trim()) generatedBy = m.generatedBy.trim();
+ } catch {
+ // no manifest, or not JSON: the manifest path is still the place to look
+ }
+ if (!generatedBy) {
+ out.how = "hand-edited manifest";
+ return out;
+ }
+ const rel = path.relative(/* turbopackIgnore: true */ reportsRoot, dir);
+ const ws = rel && !rel.startsWith("..") ? path.join(/* turbopackIgnore: true */ reportsRoot, rel.split(path.sep)[0]) : null;
+ const candidates = [ws && path.join(/* turbopackIgnore: true */ ws, generatedBy), path.join(/* turbopackIgnore: true */ dir, generatedBy)].filter(Boolean);
+ let gen = null;
+ for (const c of candidates) {
+ if (await exists(c)) {
+ gen = c;
+ break;
+ }
+ }
+ out.generator = gen ? tildify(gen) : generatedBy;
+ out.how = `manifest is generated by ${generatedBy}; edit its inputs, then regenerate`;
+ return out;
+}
+
+/**
+ * Resolve what the CLI was handed: `<site>/<report>` when that report exists,
+ * else a project.
+ */
+export async function resolveTarget(spec, opts = {}) {
+ const parts = String(spec ?? "").split("/");
+ if (parts.length === 2 && parts.every((s) => SEGMENT_RE.test(s))) {
+ const dir = path.join(/* turbopackIgnore: true */ opts.sitesDir ?? SITES_DIR, parts[0], "reports", parts[1]);
+ if (await exists(dir)) return articleTarget(spec, opts);
+ }
+ return projectTarget(spec, opts);
+}
+
+/**
+ * Every notes.json there is: articles under SITES_DIR, projects under
+ * REPORTS_ROOT. `{ target, file, doc, error? }` each; sorted by id.
+ *
+ * @param {{ sitesDir?: string, reportsRoot?: string }} [opts]
+ */
+export async function listNotesFiles({ sitesDir = SITES_DIR, reportsRoot = REPORTS_ROOT } = {}) {
+ const out = [];
+ for (const site of await readdir(/* turbopackIgnore: true */ sitesDir).catch(() => [])) {
+ if (!SEGMENT_RE.test(site)) continue;
+ for (const report of await readdir(/* turbopackIgnore: true */ path.join(/* turbopackIgnore: true */ sitesDir, site, "reports")).catch(() => [])) {
+ if (!SEGMENT_RE.test(report)) continue;
+ const file = path.join(/* turbopackIgnore: true */ sitesDir, site, "reports", report, NOTES_FILENAME);
+ if (!(await exists(file))) continue;
+ out.push({ kind: "article", id: `${site}/${report}`, file, ...(await readNotes(file)) });
+ }
+ }
+ for (const p of await projectRefs(reportsRoot)) {
+ if (!kindTakesNotes(p.kind)) continue;
+ const file = path.join(/* turbopackIgnore: true */ p.dir, NOTES_FILENAME);
+ if (!(await exists(file))) continue;
+ out.push({ kind: "video-project", id: p.id, projectKind: p.kind, file, ...(await readNotes(file)) });
+ }
+ return out.sort((a, b) => a.kind.localeCompare(b.kind) || a.id.localeCompare(b.id));
+}
+
+/**
+ * One write to a target's notes. A file that does not exist yet starts with
+ * the target's subject and its discovered source (which the agent may correct
+ * later with a `source` op); an existing file keeps both.
+ *
+ * @param {{ file: string, subject: Record<string, string>, source: () => Promise<Record<string, string> | null> }} target
+ * @param {Record<string, any>} op
+ * @param {{ by: "operator" | "agent", token?: string | null }} opts
+ */
+export async function writeNote(target, op, opts) {
+ const now = await readNotes(target.file);
+ const source = now.doc || now.error ? undefined : ((await target.source()) ?? undefined);
+ return writeOp(target.file, { subject: target.subject, source }, op, opts);
+}
diff --git a/umtool/lib/annotations/types.ts b/umtool/lib/annotations/types.ts
@@ -0,0 +1,98 @@
+// The notes shapes, with NO server imports (the lib/note-types.ts rule): a
+// client component may take a value from here without dragging node:fs into
+// the browser bundle. The store that reads and writes them is ./store.mjs,
+// shared by the app and `umtool notes`; docs/notes.md is the prose.
+
+// The values are ./shape.mjs's (pure, shared with the store and the CLI);
+// this file adds the types.
+export {
+ ANCHOR_KINDS,
+ NOTE_AUTHORS,
+ NOTE_STATUSES,
+ NOTE_TEXT_LIMIT,
+ NOTES_FORMAT,
+ NOTES_VERSION,
+ REPORT_BLOCKS,
+} from "./shape.mjs";
+export { CONTEXT as ANCHOR_CONTEXT } from "./anchor.mjs";
+
+export type NoteStatus = "open" | "resolved" | "wontfix";
+export type NoteAuthor = "operator" | "agent";
+
+/** What a moment resolved to when it was written, for the agent reading it back. */
+export type MomentResolved = {
+ entry?: string;
+ title?: string;
+ quote?: string;
+ channel?: string;
+ video?: string;
+ sourceT?: number;
+ url?: string;
+ /** The schedule did not match this file exactly (a preview, not out/). */
+ approx?: boolean;
+};
+
+export type Anchor =
+ | { kind: "text"; section: string; quote: string; prefix: string; suffix: string }
+ | { kind: "cite"; cite: string }
+ | { kind: "section"; section: string }
+ | { kind: "whole" }
+ | { kind: "moment"; file: string; t: number; take?: string; entry?: string; resolved?: MomentResolved }
+ | { kind: "entry"; entry: string }
+ | { kind: "take"; take: string }
+ | { kind: "edit"; entry?: string; field: string; from: unknown; to: unknown };
+
+export type AnchorKind = Anchor["kind"];
+
+export type NoteReply = { author: NoteAuthor; text: string; at: string };
+
+export type Note = {
+ id: string;
+ status: NoteStatus;
+ author: NoteAuthor;
+ text: string;
+ at: string;
+ updatedAt: string;
+ anchor: Anchor;
+ replies: NoteReply[];
+ resolvedAt?: string;
+ resolvedBy?: NoteAuthor;
+};
+
+export type NotesSubject =
+ | { kind: "article"; site: string; report: string }
+ | { kind: "video-project"; project: string };
+
+/** Which file an agent should edit to act on a note. Filled by umtool, correctable. */
+export type NotesSource = { draft?: string; generator?: string; manifest?: string; how?: string };
+
+export type NotesDoc = {
+ format: "umtool-notes";
+ version: 1;
+ subject: NotesSubject;
+ source?: NotesSource;
+ notes: Note[];
+};
+
+/** What GET /api/notes returns. `token` goes back on every write (409 when stale). */
+export type NotesRead = {
+ subject: NotesSubject;
+ file: string;
+ token: string;
+ doc: NotesDoc | null;
+ source: NotesSource | null;
+ error?: string;
+};
+
+/** One write. The server stamps author "operator" on everything the UI sends. */
+export type NoteOp =
+ | { op: "add"; text: string; anchor: Anchor }
+ | { op: "edit"; id: string; text?: string; anchor?: Anchor }
+ | { op: "status"; id: string; status: NoteStatus }
+ | { op: "reply"; id: string; text: string; resolve?: boolean }
+ | { op: "delete"; id: string }
+ | { op: "delete-reply"; id: string; index: number }
+ | { op: "source"; source: NotesSource };
+
+export const openCount = (doc: NotesDoc | null | undefined): number =>
+ doc ? doc.notes.filter((n) => n.status === "open").length : 0;
diff --git a/umtool/lib/annotations/useNotes.ts b/umtool/lib/annotations/useNotes.ts
@@ -0,0 +1,92 @@
+"use client";
+
+import { useCallback, useEffect, useRef, useState } from "react";
+import type { NoteOp, NotesRead, Note, NotesDoc } from "./types";
+
+// The page's handle on one notes.json: read it, write one op at a time with the
+// token it read, and on a 409 re-read and say so rather than retrying blind --
+// an agent may have replied in between, and the operator should see that reply
+// before their edit lands on top of it.
+
+export type NotesTarget = { article: string } | { project: string };
+
+export function notesQuery(target: NotesTarget): string {
+ return "article" in target ? `article=${encodeURIComponent(target.article)}` : `project=${encodeURIComponent(target.project)}`;
+}
+
+export type UseNotes = {
+ doc: NotesDoc | null;
+ notes: Note[];
+ source: NotesRead["source"];
+ error: string | null;
+ loading: boolean;
+ busy: boolean;
+ /** Apply one op. Resolves to the note touched (null on delete), or null after an error (shown in `error`). */
+ write: (op: NoteOp) => Promise<Note | null>;
+ reload: () => Promise<void>;
+};
+
+export function useNotes(target: NotesTarget | null, { initial }: { initial?: NotesRead | null } = {}): UseNotes {
+ const [read, setRead] = useState<NotesRead | null>(initial ?? null);
+ const [error, setError] = useState<string | null>(initial?.error ?? null);
+ const [loading, setLoading] = useState(!initial && !!target);
+ const [busy, setBusy] = useState(false);
+ const token = useRef<string | null>(initial?.token ?? null);
+ const query = target ? notesQuery(target) : null;
+
+ const reload = useCallback(async () => {
+ if (!query) return;
+ setLoading(true);
+ try {
+ const res = await fetch(`/api/notes?${query}`, { cache: "no-store" });
+ const j = await res.json();
+ if (!res.ok) throw new Error(j.error ?? res.statusText);
+ token.current = j.token;
+ setRead(j);
+ setError(j.error ?? null);
+ } catch (err) {
+ setError(err instanceof Error ? err.message : String(err));
+ } finally {
+ setLoading(false);
+ }
+ }, [query]);
+
+ useEffect(() => {
+ if (!initial) void reload();
+ // `initial` is the server render's read; only a changed target re-reads.
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [reload]);
+
+ const write = useCallback(
+ async (op: NoteOp): Promise<Note | null> => {
+ if (!query) return null;
+ setBusy(true);
+ try {
+ const res = await fetch(`/api/notes?${query}`, {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ token: token.current, op }),
+ });
+ const j = await res.json();
+ if (res.status === 409) {
+ await reload();
+ setError(`${j.error ?? "the notes changed"} -- reloaded; check and try again`);
+ return null;
+ }
+ if (!res.ok) throw new Error(j.error ?? res.statusText);
+ token.current = j.token;
+ setRead((r) => (r ? { ...r, doc: j.doc, token: j.token, source: j.doc?.source ?? r.source } : r));
+ setError(null);
+ return j.note ?? null;
+ } catch (err) {
+ setError(err instanceof Error ? err.message : String(err));
+ return null;
+ } finally {
+ setBusy(false);
+ }
+ },
+ [query, reload],
+ );
+
+ return { doc: read?.doc ?? null, notes: read?.doc?.notes ?? [], source: read?.source ?? null, error, loading, busy, write, reload };
+}
diff --git a/umtool/lib/articles/article.ts b/umtool/lib/articles/article.ts
@@ -0,0 +1,191 @@
+import { readFile, stat } from "node:fs/promises";
+import path from "node:path";
+import {
+ buildReportPageView,
+ REPORT_PAGE_FORMAT,
+ REPORT_VIEWS_VERSION,
+ type RecordView,
+ type ReportPageView,
+} from "yt-dlp-transcript-common/lib/report/views";
+import type { Report } from "yt-dlp-transcript-common/lib/report/schema";
+import { platformMomentUrl } from "yt-dlp-transcript-common/lib/momentUrl";
+import { readAllPosts } from "yt-dlp-transcript-common/lib/posts-server";
+import type { Post } from "yt-dlp-transcript-common/lib/posts";
+import { readCues } from "@/lib/projects/report.mjs";
+import { CHANNELS_DIR } from "@/lib/paths";
+
+// A report's PAGE VIEW, built the way the export site builds it
+// (common/lib/report/views.ts buildReportPageView) but resolved against what
+// umtool can read without the LMDB index or a compose run: each cited record's
+// own files on disk. compose's resolveSiteReports is not reused: it verifies,
+// prepares and THROWS on any problem, and half of what this page is for is
+// reading drafts that have problems.
+//
+// A record resolves from its transcript.cues.json (title, date, the uploader's
+// display name, webpageUrl -- through lib/projects/report.mjs readCues, which is
+// memoised on the file's mtime), else its metadata.info.json, else the
+// citation's own label, speaker and date. A post resolves from the channel's
+// posts. Nothing here fails the page: a record that cannot be read is a card
+// with less on it.
+
+const isoDay = (d: unknown): string | undefined => {
+ const s = typeof d === "string" ? d : "";
+ if (/^\d{8}$/.test(s)) return `${s.slice(0, 4)}-${s.slice(4, 6)}-${s.slice(6, 8)}`;
+ if (/^\d{4}-\d{2}-\d{2}/.test(s)) return s.slice(0, 10);
+ return undefined;
+};
+
+export const recordDir = (channel: string, id: string) =>
+ path.join(/* turbopackIgnore: true */ CHANNELS_DIR, channel, "data", id);
+
+type RecordMeta = { title?: string; date?: string; channelTitle?: string; webpageUrl?: string };
+
+const metaMemo = new Map<string, { key: string; value: RecordMeta | null }>();
+
+/** What a record says about itself; null when its directory holds neither file. */
+export async function recordMeta(channel: string, id: string): Promise<RecordMeta | null> {
+ if (!/^[A-Za-z0-9_.@-]+$/.test(channel) || !/^[A-Za-z0-9_.@-]+$/.test(id)) return null;
+ const dir = recordDir(channel, id);
+ const cues = (await readCues(path.join(/* turbopackIgnore: true */ dir, "transcript.cues.json"))) as
+ | { title?: string; uploadDate?: string; webpageUrl?: string; channel?: string }
+ | null;
+ if (cues && (cues.title || cues.webpageUrl)) {
+ return { title: cues.title, date: isoDay(cues.uploadDate), channelTitle: cues.channel, webpageUrl: cues.webpageUrl };
+ }
+ const file = path.join(/* turbopackIgnore: true */ dir, "metadata.info.json");
+ const st = await stat(/* turbopackIgnore: true */ file).catch(() => null);
+ if (!st) return null;
+ const key = `${Math.round(st.mtimeMs)}-${st.size}`;
+ const hit = metaMemo.get(file);
+ if (hit?.key === key) return hit.value;
+ let value: RecordMeta | null = null;
+ try {
+ const m = JSON.parse(await readFile(/* turbopackIgnore: true */ file, "utf8"));
+ value = {
+ title: typeof m.title === "string" ? m.title : undefined,
+ date: isoDay(m.upload_date),
+ channelTitle: typeof m.uploader === "string" ? m.uploader : typeof m.channel === "string" ? m.channel : undefined,
+ webpageUrl: typeof m.webpage_url === "string" ? m.webpage_url : undefined,
+ };
+ } catch {
+ value = null;
+ }
+ metaMemo.set(file, { key, value });
+ return value;
+}
+
+// A channel's posts, by id. A big X archive is thousands of posts, so one read
+// per channel per minute.
+const postsMemo = new Map<string, { at: number; value: Promise<Map<string, Post>> }>();
+export function channelPosts(channel: string): Promise<Map<string, Post>> {
+ const hit = postsMemo.get(channel);
+ if (hit && Date.now() - hit.at < 60_000) return hit.value;
+ const value = readAllPosts(path.join(/* turbopackIgnore: true */ CHANNELS_DIR, channel))
+ .then((list) => new Map(list.map((p) => [p.id, p])))
+ .catch(() => new Map<string, Post>());
+ postsMemo.set(channel, { at: Date.now(), value });
+ return value;
+}
+
+/** The capture screenshot of a post, if the editor took one. */
+export async function postShotFile(channel: string, id: string): Promise<string | null> {
+ if (!/^[A-Za-z0-9_.@-]+$/.test(channel) || !/^[A-Za-z0-9_.@-]+$/.test(id)) return null;
+ const file = path.join(/* turbopackIgnore: true */ CHANNELS_DIR, channel, "posts-media", id, "shot.png");
+ return (await stat(/* turbopackIgnore: true */ file).catch(() => null))?.isFile() ? file : null;
+}
+
+export const corpusMediaUrl = (abs: string) => `/api/sites/media?corpus=${encodeURIComponent(abs)}`;
+
+type Cite = NonNullable<Report["citations"]>[string];
+
+async function recordViewOf(
+ c: Cite,
+ metaOf: (channel: string, id: string) => Promise<RecordMeta | null> = recordMeta,
+): Promise<RecordView | undefined> {
+ if (c.kind === "video" || c.kind === "audio") {
+ const m = await metaOf(c.channel, c.id);
+ return {
+ channel: c.channel,
+ id: c.id,
+ ...(m?.channelTitle || c.speaker ? { channelTitle: m?.channelTitle ?? c.speaker } : {}),
+ ...(m?.title || c.label ? { title: m?.title ?? c.label } : {}),
+ ...(m?.date || c.date ? { date: m?.date ?? c.date } : {}),
+ ...(m?.webpageUrl ? { originalUrl: platformMomentUrl(m.webpageUrl, null, c.start) ?? m.webpageUrl } : {}),
+ };
+ }
+ if (c.kind === "post") {
+ const post = (await channelPosts(c.channel)).get(c.id);
+ return {
+ channel: c.channel,
+ id: c.id,
+ ...(post?.authorName || post?.author ? { channelTitle: post.authorName ?? post.author } : {}),
+ ...(post?.createdAt ? { date: post.createdAt.slice(0, 10) } : c.date ? { date: c.date } : {}),
+ ...(post?.platform ? { platform: post.platform } : {}),
+ ...(post?.url ? { originalUrl: post.url } : {}),
+ };
+ }
+ return undefined;
+}
+
+/**
+ * The page view of a report, or -- when the report's citations cannot be built
+ * into one (a draft naming a source it does not define) -- the same view with
+ * no citations, and the reason.
+ */
+export async function articleView(report: Report): Promise<{ view: ReportPageView; error: string | null }> {
+ const records = new Map<Cite, RecordView>();
+ const posts = new Map<Cite, { author?: string; text?: string; shot?: string }>();
+ // A fact-check cites the same few records hundreds of times: read each once,
+ // all at the same time (recordMeta memoises on the file, not on a promise).
+ const metas = new Map<string, Promise<RecordMeta | null>>();
+ const metaOf = (channel: string, id: string) => {
+ const k = `${channel}/${id}`;
+ if (!metas.has(k)) metas.set(k, recordMeta(channel, id));
+ return metas.get(k)!;
+ };
+ await Promise.all(
+ Object.values(report.citations ?? {}).map(async (c) => {
+ const r = await recordViewOf(c, metaOf);
+ if (r) records.set(c, r);
+ if (c.kind === "post") {
+ const post = (await channelPosts(c.channel)).get(c.id);
+ const shot = await postShotFile(c.channel, c.id);
+ posts.set(c, {
+ ...(post ? { author: post.authorName ?? post.author, text: post.text } : {}),
+ ...(shot ? { shot: corpusMediaUrl(shot) } : {}),
+ });
+ }
+ }),
+ );
+ try {
+ const view = buildReportPageView(report, {
+ record: (c) => records.get(c as Cite) ?? { channel: c.channel, id: c.id },
+ post: (c) => posts.get(c as Cite),
+ });
+ return { view, error: null };
+ } catch (err) {
+ const view = {
+ format: REPORT_PAGE_FORMAT,
+ version: REPORT_VIEWS_VERSION,
+ id: report.id,
+ kind: report.kind,
+ ...(report.series ? { series: report.series } : {}),
+ title: report.title,
+ ...(report.subtitle ? { subtitle: report.subtitle } : {}),
+ ...(report.summary ? { summary: report.summary } : {}),
+ ...(report.method ? { method: report.method } : {}),
+ ...(report.published ? { published: report.published } : {}),
+ ...(report.updated ? { updated: report.updated } : {}),
+ sources: {},
+ verdicts: {},
+ citations: {},
+ sections: report.sections.map((s) => ({
+ id: s.id,
+ title: s.title,
+ ...(s.body ? { body: s.body } : {}),
+ claims: (s.claims ?? []).map((cl) => ({ ...cl, citations: cl.citations ?? [] })),
+ })),
+ } as unknown as ReportPageView;
+ return { view, error: err instanceof Error ? err.message : String(err) };
+ }
+}
diff --git a/umtool/lib/articles/evidence.ts b/umtool/lib/articles/evidence.ts
@@ -0,0 +1,182 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import type { Report } from "yt-dlp-transcript-common/lib/report/schema";
+import { momentKeyOf } from "yt-dlp-transcript-common/lib/citations/moments";
+import { evidenceSpan, resolveEvidenceSource } from "yt-dlp-transcript-common/lib/evidenceClip-server";
+import { reportMediaDir, reportMediaIndexFile } from "yt-dlp-transcript-common/publish/reportMedia";
+import { readCues } from "@/lib/projects/report.mjs";
+import { CHANNELS_DIR } from "@/lib/paths";
+import { channelPosts, corpusMediaUrl, postShotFile, recordDir, recordMeta } from "./article";
+import { sitesPaths } from "./sites";
+
+// What a citation's EVIDENCE panel shows: the cited seconds with the transcript
+// around them, and something to play -- found, never fetched.
+//
+// Playback, best first:
+// 1. prepared the site's own evidence clip (.export-index/sites/<site>/
+// report-media/, what the build publishes), when prepare has run;
+// 2. window a clip window the editor fetched into data/<id>/clips/;
+// 3. saved a saved video or audio file in data/<id>/ (through its media
+// tier link -- read, never written);
+// 4. otherwise nothing to play, and the line that fetches it through the
+// editor: the MCP `fetch_clip` tool. umtool's own fetch client
+// (api/report/fetch) is a manifest's, so it cannot ask for a
+// window no project names. Never yt-dlp.
+
+export const CONTEXT_CUES = 6;
+
+export type EvidenceCue = { start: number; end: number; text: string; cited: boolean };
+
+export type EvidencePlay = {
+ kind: "prepared" | "window" | "saved" | "audio";
+ url: string;
+ /** Seconds into the file where the cited span starts. */
+ offset: number;
+ audio: boolean;
+ label: string;
+};
+
+export type Evidence = {
+ cite: string;
+ kind: string;
+ quote: string;
+ speaker?: string;
+ date?: string;
+ label?: string;
+ originalUrl?: string;
+ record?: { channel: string; id: string; title?: string; channelTitle?: string };
+ start?: number;
+ end?: number;
+ cues: EvidenceCue[];
+ cuesNote?: string;
+ play: EvidencePlay | null;
+ fetchLine?: string;
+ post?: { author?: string; text?: string; url?: string; shot?: string };
+};
+
+type Cite = NonNullable<Report["citations"]>[string];
+
+/** The ±CONTEXT_CUES cues around [start, end], the overlapping ones marked. */
+export async function cueContext(channel: string, id: string, start: number, end: number) {
+ const file = path.join(/* turbopackIgnore: true */ recordDir(channel, id), "transcript.cues.json");
+ const doc = (await readCues(file)) as { cues?: { start: number; end: number; text?: string }[] } | null;
+ const cues = doc?.cues ?? [];
+ if (!cues.length) return { cues: [] as EvidenceCue[], note: doc ? "no cues" : "no transcript.cues.json" };
+ const EPS = 0.05;
+ let first = cues.findIndex((c) => c.end > start + EPS);
+ if (first < 0) first = cues.length - 1;
+ let last = first;
+ while (last + 1 < cues.length && cues[last + 1].start < end - EPS) last += 1;
+ const from = Math.max(0, first - CONTEXT_CUES);
+ const to = Math.min(cues.length - 1, last + CONTEXT_CUES);
+ return {
+ cues: cues.slice(from, to + 1).map((c, i) => ({
+ start: c.start,
+ end: c.end,
+ text: String(c.text ?? "").replace(/\s+/g, " ").trim(),
+ cited: from + i >= first && from + i <= last,
+ })),
+ };
+}
+
+async function preparedClip(siteId: string, c: Cite): Promise<EvidencePlay | null> {
+ if (c.kind !== "video" && c.kind !== "audio") return null;
+ const key = momentKeyOf(c);
+ if (!key) return null;
+ try {
+ const index = JSON.parse(await readFile(/* turbopackIgnore: true */ reportMediaIndexFile(sitesPaths(), siteId), "utf8"));
+ const entry = index?.moments?.[key];
+ if (!entry || (entry.kind !== "video" && entry.kind !== "audio") || typeof entry.file !== "string") return null;
+ const abs = path.join(/* turbopackIgnore: true */ reportMediaDir(sitesPaths(), siteId), entry.file);
+ let from = evidenceSpan(c).from;
+ try {
+ const side = JSON.parse(await readFile(/* turbopackIgnore: true */ abs.replace(/\.(mp4|m4a)$/, ".json"), "utf8"));
+ if (typeof side?.span?.from === "number") from = side.span.from;
+ } catch {
+ // no sidecar: the citation's own pad is the best guess
+ }
+ return {
+ kind: "prepared",
+ url: `/api/sites/media?site=${encodeURIComponent(siteId)}&moment=${encodeURIComponent(key)}`,
+ offset: Math.max(0, c.start - from),
+ audio: entry.kind === "audio",
+ label: "prepared evidence clip",
+ };
+ } catch {
+ return null;
+ }
+}
+
+async function corpusClip(c: Cite): Promise<EvidencePlay | null> {
+ if (c.kind !== "video" && c.kind !== "audio") return null;
+ const span = { from: c.start, to: c.end };
+ for (const audio of c.kind === "audio" ? [true] : [false, true]) {
+ const hit = await resolveEvidenceSource({ channelsDir: CHANNELS_DIR, slug: c.channel, id: c.id, span, audio }).catch(() => null);
+ if (!hit) continue;
+ const kind = hit.kind === "corpus-window" ? "window" : hit.kind === "saved-video" ? "saved" : "audio";
+ return {
+ kind,
+ url: corpusMediaUrl(hit.path),
+ offset: Math.max(0, c.start - hit.windowStart),
+ audio: hit.kind === "audio",
+ label: kind === "window" ? `clip window ${hit.name}` : kind === "saved" ? `saved ${hit.name}` : `audio ${hit.name}`,
+ };
+ }
+ return null;
+}
+
+/** The MCP line that fetches this span through the editor. */
+export function fetchClipLine(c: { channel: string; id: string; start: number; end: number }, reason: string): string {
+ const s = (n: number) => Number(n.toFixed(2));
+ return `fetch_clip ${JSON.stringify({ channel: c.channel, video: c.id, start: s(c.start), end: s(c.end), reason })}`;
+}
+
+export async function citationEvidence(siteId: string, report: Report, citeId: string): Promise<Evidence | null> {
+ const c = report.citations?.[citeId];
+ if (!c) return null;
+ const base: Evidence = {
+ cite: citeId,
+ kind: c.kind,
+ quote: c.quote,
+ ...(c.speaker ? { speaker: c.speaker } : {}),
+ ...(c.date ? { date: c.date } : {}),
+ ...(c.label ? { label: c.label } : {}),
+ cues: [],
+ play: null,
+ };
+ if (c.kind === "video" || c.kind === "audio") {
+ const meta = await recordMeta(c.channel, c.id);
+ const ctx = await cueContext(c.channel, c.id, c.start, c.end);
+ const play = (await preparedClip(siteId, c)) ?? (await corpusClip(c));
+ return {
+ ...base,
+ start: c.start,
+ end: c.end,
+ record: { channel: c.channel, id: c.id, ...(meta?.title ? { title: meta.title } : {}), ...(meta?.channelTitle ? { channelTitle: meta.channelTitle } : {}) },
+ ...(meta?.webpageUrl ? { originalUrl: meta.webpageUrl } : {}),
+ cues: ctx.cues,
+ ...(ctx.note ? { cuesNote: ctx.note } : {}),
+ play,
+ ...(play ? {} : { fetchLine: fetchClipLine(c, `${siteId}/${report.id} ${citeId}`) }),
+ };
+ }
+ if (c.kind === "post") {
+ const post = (await channelPosts(c.channel)).get(c.id);
+ const shot = await postShotFile(c.channel, c.id);
+ return {
+ ...base,
+ record: { channel: c.channel, id: c.id },
+ ...(post?.url ? { originalUrl: post.url } : {}),
+ post: {
+ ...(post ? { author: post.authorName ?? post.author, text: post.text, url: post.url } : {}),
+ ...(shot ? { shot: corpusMediaUrl(shot) } : {}),
+ },
+ };
+ }
+ if (c.kind === "page") return { ...base, originalUrl: c.url };
+ if (c.kind === "source") {
+ const s = report.sources?.[c.source];
+ return { ...base, ...(s?.url ? { originalUrl: s.url } : {}), ...(s?.title ? { label: c.label ?? s.title } : {}) };
+ }
+ return base;
+}
diff --git a/umtool/lib/articles/files.ts b/umtool/lib/articles/files.ts
@@ -0,0 +1,77 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { REPORTS_ROOT } from "@/lib/paths";
+import { listTakes, readVerdicts } from "@/lib/report/takes.mjs";
+import { siteWorkspaces, tildify } from "./sources.mjs";
+import { untildify } from "./links.mjs";
+import { workspaceFile, workspaceFiles } from "./workspace.mjs";
+
+// The page side of lib/articles/workspace.mjs and the video projects' takes.
+
+export type WorkspaceListing = {
+ /** The workspace's name under REPORTS_ROOT (what /api/sites/workspace takes). */
+ name: string;
+ dir: string;
+ files: { rel: string; kind: "draft" | "out" | "doc"; bytes: number; mtimeMs: number }[];
+};
+
+export async function listingOf(dir: string): Promise<WorkspaceListing> {
+ const files = (await workspaceFiles(dir)).map(({ rel, kind, bytes, mtimeMs }: { rel: string; kind: WorkspaceListing["files"][number]["kind"]; bytes: number; mtimeMs: number }) => ({ rel, kind, bytes, mtimeMs }));
+ return { name: path.basename(dir), dir: tildify(dir), files };
+}
+
+/** Every workspace a site's articles were written in. */
+export async function siteWorkspaceListings(siteId: string, reportIds: string[]): Promise<WorkspaceListing[]> {
+ const dirs: string[] = await siteWorkspaces(siteId, reportIds, { reportsRoot: REPORTS_ROOT });
+ return Promise.all(dirs.map(listingOf));
+}
+
+/** One article's workspace (its source's), or null. */
+export async function articleWorkspaceListing(workspace: string | undefined | null): Promise<WorkspaceListing | null> {
+ if (!workspace) return null;
+ const abs = untildify(workspace);
+ if (path.dirname(abs) !== path.resolve(REPORTS_ROOT)) return null;
+ return listingOf(abs);
+}
+
+export type OpenedFile =
+ | { ws: string; rel: string; kind: "md"; text: string }
+ | { ws: string; rel: string; kind: "json"; value: unknown; text: string }
+ | { ws: string; rel: string; kind: "html"; url: string }
+ | { ws: string; rel: string; kind: "error"; message: string };
+
+const MAX = 2 * 1024 * 1024;
+
+/** A workspace file opened for the page, or an error saying why not. */
+export async function openWorkspaceFile(ws: string, rel: string): Promise<OpenedFile> {
+ const f = await workspaceFile(ws, rel);
+ if (!f) return { ws, rel, kind: "error", message: "not a listed workspace file" };
+ if (rel.endsWith(".html")) {
+ return { ws, rel, kind: "html", url: `/api/sites/workspace?ws=${encodeURIComponent(ws)}&rel=${encodeURIComponent(rel)}` };
+ }
+ if (f.bytes > MAX) return { ws, rel, kind: "error", message: `${f.bytes} bytes; too big to show` };
+ const text = await readFile(/* turbopackIgnore: true */ f.real, "utf8");
+ if (rel.endsWith(".json")) {
+ try {
+ return { ws, rel, kind: "json", value: JSON.parse(text), text };
+ } catch (err) {
+ return { ws, rel, kind: "error", message: `not JSON: ${(err as Error).message}` };
+ }
+ }
+ return { ws, rel, kind: "md", text };
+}
+
+export type TakeTally = { takes: number; like: number; maybe: number; no: number; skipped: number };
+
+/** How many takes a video project has, and how they were judged. */
+export async function takeTally(dir: string): Promise<TakeTally> {
+ const [t, v] = await Promise.all([listTakes(dir), readVerdicts(dir)]);
+ const rows = Object.values(v) as { verdict: string | null }[];
+ return {
+ takes: t.takes.length,
+ like: rows.filter((r) => r.verdict === "like").length,
+ maybe: rows.filter((r) => r.verdict === "maybe").length,
+ no: rows.filter((r) => r.verdict === "no").length,
+ skipped: t.skipped.length,
+ };
+}
diff --git a/umtool/lib/articles/links.mjs b/umtool/lib/articles/links.mjs
@@ -0,0 +1,100 @@
+// Which umtool report-video project is an article's video.
+//
+// Two ways, in order:
+//
+// 1. The manifest says so: a top-level `"article": "<site>/<report>"` in
+// video.manifest.json. build-video.mjs never reads top-level keys it does
+// not know (it reads `generatedBy` no more than this), so the key costs
+// the render nothing. A manifest that names an article is linked to it
+// and to nothing else.
+// 2. The slug matches: the manifest's `slug` (else the project directory's
+// name) is the report id, `polemic-<id>`, or the id without `polemic-` --
+// and the project lives in the same workspace as the article's draft
+// (lib/articles/sources.mjs). A UNIQUE match is linked; two or more are
+// only "possible", and the page says so rather than picking one.
+//
+// Plain ESM, so `umtool notes` can name an article's video too.
+import { readFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { REPORTS_ROOT } from "../paths.mjs";
+import { projectRefs } from "../projects/core.mjs";
+import { kindLinksArticles } from "../projects/kinds.mjs";
+import { idKeys } from "./sources.mjs";
+
+const CACHE_MS = 30_000;
+/** @type {Map<string, { at: number, value: Promise<any[]> }>} */
+const cache = new Map();
+
+/** `~/x` back to an absolute path. */
+export function untildify(p) {
+ if (typeof p !== "string") return p;
+ return p === "~" || p.startsWith("~/") ? path.join(/* turbopackIgnore: true */ os.homedir(), p.slice(2)) : p;
+}
+
+/**
+ * Every report-video project with what linking needs from its manifest:
+ * `{ id, dir, name, slug, article, generatedBy, title }`.
+ *
+ * @param {string} [reportsRoot]
+ */
+export function videoProjects(reportsRoot = REPORTS_ROOT) {
+ const hit = cache.get(reportsRoot);
+ if (hit && Date.now() - hit.at < CACHE_MS) return hit.value;
+ const value = (async () => {
+ const out = [];
+ for (const p of await projectRefs(reportsRoot)) {
+ if (!kindLinksArticles(p.kind)) continue;
+ let m = {};
+ try {
+ m = JSON.parse(await readFile(/* turbopackIgnore: true */ path.join(/* turbopackIgnore: true */ p.dir, "video.manifest.json"), "utf8"));
+ } catch {
+ // a project with no readable manifest still links by its directory name
+ }
+ out.push({
+ id: p.id,
+ dir: p.dir,
+ name: p.name,
+ slug: typeof m.slug === "string" && m.slug ? m.slug : p.name,
+ article: typeof m.article === "string" ? m.article : null,
+ generatedBy: typeof m.generatedBy === "string" ? m.generatedBy : null,
+ title: typeof m.title === "string" ? m.title : p.name,
+ });
+ }
+ return out.sort((a, b) => a.id.localeCompare(b.id));
+ })();
+ cache.set(reportsRoot, { at: Date.now(), value });
+ value.catch(() => cache.delete(reportsRoot));
+ return value;
+}
+
+export function clearLinksCache() {
+ cache.clear();
+}
+
+const inside = (root, p) => p === root || p.startsWith(root + path.sep);
+
+/**
+ * The projects linked to one article: `{ linked, possible, how }`.
+ *
+ * @param {string} siteId
+ * @param {string} reportId
+ * @param {{ reportsRoot?: string, workspace?: string | null, projects?: any[] }} [opts]
+ * `workspace` is the article's workspace dir (sourceFor's, `~` allowed); without
+ * one, slug matches are only ever "possible".
+ */
+export async function linkedProjects(siteId, reportId, { reportsRoot = REPORTS_ROOT, workspace = null, projects } = {}) {
+ const all = projects ?? (await videoProjects(reportsRoot));
+ const key = `${siteId}/${reportId}`;
+ const declared = all.filter((p) => p.article === key);
+ if (declared.length) return { linked: declared, possible: [], how: "manifest names the article" };
+
+ const keys = idKeys(reportId);
+ // A manifest that names a DIFFERENT article is never a slug match.
+ const bySlug = all.filter((p) => !p.article && keys.has(p.slug));
+ const ws = workspace ? untildify(workspace) : null;
+ const inWs = ws ? bySlug.filter((p) => inside(ws, p.dir)) : [];
+ if (inWs.length === 1) return { linked: inWs, possible: [], how: `slug ${inWs[0].slug} in the article's workspace` };
+ const possible = inWs.length > 1 ? inWs : bySlug;
+ return { linked: [], possible, how: possible.length ? `${possible.length} project(s) share the slug` : "no project" };
+}
diff --git a/umtool/lib/articles/sites.ts b/umtool/lib/articles/sites.ts
@@ -0,0 +1,177 @@
+import { readFile, stat } from "node:fs/promises";
+import path from "node:path";
+import { getPaths, type Paths } from "yt-dlp-transcript-common/lib/paths";
+import { getSite, isListedSite, isPrivateSite, listSites, type Site } from "yt-dlp-transcript-common/lib/site";
+import { listReportDirs, siteReportDir } from "yt-dlp-transcript-common/publish/reportMedia";
+import { parseReport } from "yt-dlp-transcript-common/lib/report/validate";
+import type { Report } from "yt-dlp-transcript-common/lib/report/schema";
+import { CHANNELS_DIR, REPORTS_ROOT, SITES_DIR } from "@/lib/paths";
+import { readNotes } from "@/lib/annotations/store.mjs";
+import { corpusNotesFile } from "@/lib/paths";
+import type { NotesDoc } from "@/lib/annotations/types";
+import { sourceFor } from "./sources.mjs";
+import { linkedProjects, videoProjects } from "./links.mjs";
+
+// Every site's articles, as umtool reads them: the site list from common
+// (site.json through getSite, so a default is the editor's default), each
+// report directory through common's listReportDirs (the editor's report tab
+// uses the same enumerator), each report.json through the report document's
+// own validator, and published-or-draft from the site's `reports` order. Plus
+// what only umtool knows: its notes, its source draft, its video project.
+//
+// READ-ONLY. The one thing umtool writes under SITES_DIR is a notes.json, and
+// that goes through lib/annotations, never here.
+
+/** common's Paths, with the two roots umtool resolves itself (and e2e confines). */
+export function sitesPaths(): Paths {
+ return { ...getPaths(), sitesDir: SITES_DIR, channelsDir: CHANNELS_DIR };
+}
+
+export type ArticleStatus = "published" | "draft";
+
+export type ProjectLinkRow = { id: string; title: string; slug: string };
+
+export type ArticleRow = {
+ site: string;
+ id: string;
+ title: string;
+ series: string | null;
+ kind: string | null;
+ status: ArticleStatus;
+ published: string | null;
+ updated: string | null;
+ /** report.json's mtime, for "updated" when the report names no date. */
+ mtimeMs: number | null;
+ citations: number;
+ notes: number;
+ openNotes: number;
+ hasVideo: boolean;
+ hasPoster: boolean;
+ /** report.json missing, unparseable, or invalid: the first problem, else null. */
+ problem: string | null;
+ problems: number;
+ source: { draft?: string; generator?: string; how?: string; workspace?: string } | null;
+ projects: { linked: ProjectLinkRow[]; possible: ProjectLinkRow[] };
+};
+
+export type SiteRow = {
+ siteId: string;
+ title: string;
+ private: boolean;
+ listed: boolean;
+ search: boolean;
+ published: number;
+ drafts: number;
+ openNotes: number;
+ articles: ArticleRow[];
+};
+
+const exists = (p: string) => stat(/* turbopackIgnore: true */ p).then((s) => s.isFile(), () => false);
+
+export type ArticleRead = {
+ report: Report | null;
+ problems: { path?: string; message: string }[];
+ mtimeMs: number | null;
+};
+
+/** One report.json, read and validated; never throws. */
+export async function readReportFile(siteId: string, reportId: string): Promise<ArticleRead> {
+ const file = path.join(/* turbopackIgnore: true */ siteReportDir(sitesPaths(), siteId, reportId), "report.json");
+ let text: string;
+ let mtimeMs: number | null = null;
+ try {
+ const [t, st] = await Promise.all([readFile(/* turbopackIgnore: true */ file, "utf8"), stat(/* turbopackIgnore: true */ file)]);
+ text = t;
+ mtimeMs = Math.round(st.mtimeMs);
+ } catch {
+ return { report: null, problems: [{ message: "no report.json" }], mtimeMs: null };
+ }
+ let raw: unknown;
+ try {
+ raw = JSON.parse(text);
+ } catch (err) {
+ return { report: null, problems: [{ message: `report.json is not JSON: ${(err as Error).message}` }], mtimeMs };
+ }
+ const parsed = parseReport(raw, { id: reportId });
+ if (!parsed.ok) return { report: null, problems: parsed.problems, mtimeMs };
+ return { report: parsed.value, problems: parsed.problems, mtimeMs };
+}
+
+export async function readArticleNotes(siteId: string, reportId: string): Promise<{ doc: NotesDoc | null; token: string; error?: string }> {
+ const file = corpusNotesFile(siteId, reportId);
+ if (!file) return { doc: null, token: "absent" };
+ return readNotes(file);
+}
+
+async function articleRow(site: Site, id: string, published: Set<string>, projects: Awaited<ReturnType<typeof videoProjects>>): Promise<ArticleRow> {
+ const [read, notes, source] = await Promise.all([
+ readReportFile(site.siteId, id),
+ readArticleNotes(site.siteId, id),
+ sourceFor(site.siteId, id, { reportsRoot: REPORTS_ROOT }),
+ ]);
+ const dir = siteReportDir(sitesPaths(), site.siteId, id);
+ const r = read.report;
+ const links = await linkedProjects(site.siteId, id, { workspace: source?.workspace ?? null, projects });
+ const row = (p: { id: string; title: string; slug: string }) => ({ id: p.id, title: p.title, slug: p.slug });
+ return {
+ site: site.siteId,
+ id,
+ title: r?.title ?? id,
+ series: r?.series ?? null,
+ kind: r?.kind ?? null,
+ status: published.has(id) ? "published" : "draft",
+ published: r?.published ?? null,
+ updated: r?.updated ?? r?.published ?? null,
+ mtimeMs: read.mtimeMs,
+ citations: Object.keys(r?.citations ?? {}).length,
+ notes: notes.doc?.notes.length ?? 0,
+ openNotes: notes.doc?.notes.filter((n) => n.status === "open").length ?? 0,
+ hasVideo: !!r?.video?.src && (await exists(path.join(/* turbopackIgnore: true */ dir, r.video.src))),
+ hasPoster: !!r?.video?.poster && (await exists(path.join(/* turbopackIgnore: true */ dir, r.video.poster))),
+ problem: read.problems[0]?.message ?? null,
+ problems: read.problems.length,
+ source,
+ projects: { linked: links.linked.map(row), possible: links.possible.map(row) },
+ };
+}
+
+function siteFlags(site: Site) {
+ return { private: isPrivateSite(site), listed: isListedSite(site), search: site.search !== false };
+}
+
+/** One site with every article, published (in the site's order) then drafts (by id). */
+export async function readSiteRow(site: Site, projects?: Awaited<ReturnType<typeof videoProjects>>): Promise<SiteRow> {
+ const all = projects ?? (await videoProjects(REPORTS_ROOT));
+ const published = new Set(site.reports ?? []);
+ const dirs = await listReportDirs(sitesPaths(), site.siteId);
+ const ids = [...(site.reports ?? []), ...dirs.filter((d) => !published.has(d))];
+ const articles = await Promise.all(ids.map((id) => articleRow(site, id, published, all)));
+ return {
+ siteId: site.siteId,
+ title: site.siteTitle || site.siteId,
+ ...siteFlags(site),
+ published: articles.filter((a) => a.status === "published").length,
+ drafts: articles.filter((a) => a.status === "draft").length,
+ openNotes: articles.reduce((n, a) => n + a.openNotes, 0),
+ articles,
+ };
+}
+
+/** Every site, private first, then by id. */
+export async function listSiteRows(): Promise<SiteRow[]> {
+ const projects = await videoProjects(REPORTS_ROOT);
+ const sites = listSites(sitesPaths());
+ const rows = await Promise.all(sites.map((s) => readSiteRow(s, projects)));
+ return rows.sort((a, b) => Number(b.private) - Number(a.private) || a.siteId.localeCompare(b.siteId));
+}
+
+/** A site by id, or null (a bad id or no site.json). */
+export function siteById(siteId: string): Site | null {
+ try {
+ const paths = sitesPaths();
+ if (!listSites(paths).some((s) => s.siteId === siteId)) return null;
+ return getSite(siteId, paths);
+ } catch {
+ return null;
+ }
+}
diff --git a/umtool/lib/articles/sources.mjs b/umtool/lib/articles/sources.mjs
@@ -0,0 +1,219 @@
+// Which file an agent should EDIT to change an article.
+//
+// A report.json under transcripts/sites/ is generated: a workspace under
+// ~/reports keeps the draft (`<ws>/polemics/drafts/<slug>.json`, the source of
+// truth) and a generator script that writes report.json from it
+// (`<ws>/polemics/make-site.py`, `<ws>/site/polemics.py`, ...). A note that
+// says "fix this sentence" is useless to an agent that edits report.json -- the
+// next generator run puts the old sentence back. So every notes.json carries a
+// `source` block naming the draft and the generator, found here.
+//
+// The match is a heuristic, written down with its reason (`how`), and the
+// agent may correct it (`umtool notes source`):
+//
+// draft a drafts/*.json whose `id`, or file name, is the report id --
+// allowing for a `polemic-` prefix on either side (candalyzer's
+// polemic-israel is drafts/israel.json with id polemic-israel;
+// jeralyzer-private's `blame` is drafts/blame.json with id
+// polemic-blame). Several matches: the one whose workspace has a
+// generator naming the site wins; still several, none is chosen.
+// generator a *.py / *.mts under <ws>/polemics or <ws>/site that names the
+// site (or the report id), preferring one that names the report
+// id itself, then one that reads the drafts. Backups
+// (`make-report.pre-2026-10-05.py`: a second dot) are skipped.
+//
+// Cheap: one readdir per workspace and one read per generator, cached for 30 s
+// like the project walk.
+import { readdir, readFile, stat } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { REPORTS_ROOT } from "../paths.mjs";
+
+const CACHE_MS = 30_000;
+const GEN_DIRS = ["polemics", "site"];
+const GEN_EXT = /^[^.]+\.(py|mts|mjs|sh)$/;
+const MAX_GEN_BYTES = 2 * 1024 * 1024;
+
+/** `~/…` for a path under the home directory: what a note shows an agent. */
+export function tildify(abs) {
+ const home = os.homedir();
+ return abs === home || abs.startsWith(home + path.sep) ? `~${abs.slice(home.length)}` : abs;
+}
+
+/** The ids a report or draft may go by: itself, without `polemic-`, with it. */
+export function idKeys(id) {
+ const bare = id.replace(/^polemic-/, "");
+ return new Set([id, bare, `polemic-${bare}`]);
+}
+
+/** Does `text` name `id` as a whole token (so `jasolyzer` is not `jasolyzer-private`)? */
+export function names(text, id) {
+ const esc = id.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+ return new RegExp(`(?<![A-Za-z0-9_-])${esc}(?![A-Za-z0-9_-])`).test(text);
+}
+
+const isDir = (p) => stat(/* turbopackIgnore: true */ p).then((s) => s.isDirectory(), () => false);
+
+/** @type {Map<string, { at: number, value: Promise<any[]> }>} */
+const cache = new Map();
+
+/**
+ * Every workspace under `reportsRoot` that has drafts or a generator dir:
+ * `{ dir, name, drafts: [{ file, slug, id }], generators: [{ file, text }] }`.
+ *
+ * @param {string} [reportsRoot]
+ */
+export function scanWorkspaces(reportsRoot = REPORTS_ROOT) {
+ const hit = cache.get(reportsRoot);
+ if (hit && Date.now() - hit.at < CACHE_MS) return hit.value;
+ const value = scan(reportsRoot);
+ cache.set(reportsRoot, { at: Date.now(), value });
+ value.catch(() => cache.delete(reportsRoot));
+ return value;
+}
+
+/** Forget the scan (a test that writes a fixture, then reads it). */
+export function clearSourcesCache() {
+ cache.clear();
+}
+
+async function scan(reportsRoot) {
+ const entries = await readdir(/* turbopackIgnore: true */ reportsRoot, { withFileTypes: true }).catch(() => []);
+ const out = [];
+ for (const e of entries) {
+ if (e.name.startsWith(".") || e.name === "data") continue;
+ const dir = path.join(/* turbopackIgnore: true */ reportsRoot, e.name);
+ if (!(e.isDirectory() || (e.isSymbolicLink() && (await isDir(dir))))) continue;
+ const drafts = [];
+ const draftsDir = path.join(/* turbopackIgnore: true */ dir, "polemics", "drafts");
+ for (const f of await readdir(/* turbopackIgnore: true */ draftsDir).catch(() => [])) {
+ if (!f.endsWith(".json")) continue;
+ const file = path.join(/* turbopackIgnore: true */ draftsDir, f);
+ let id = null;
+ try {
+ const j = JSON.parse(await readFile(/* turbopackIgnore: true */ file, "utf8"));
+ if (j && typeof j.id === "string") id = j.id;
+ } catch {
+ // an unreadable draft still matches by its file name
+ }
+ drafts.push({ file, slug: f.slice(0, -5), id });
+ }
+ const generators = [];
+ for (const g of GEN_DIRS) {
+ const gdir = path.join(/* turbopackIgnore: true */ dir, g);
+ for (const f of await readdir(/* turbopackIgnore: true */ gdir).catch(() => [])) {
+ if (!GEN_EXT.test(f)) continue;
+ const file = path.join(/* turbopackIgnore: true */ gdir, f);
+ const st = await stat(/* turbopackIgnore: true */ file).catch(() => null);
+ if (!st?.isFile() || st.size > MAX_GEN_BYTES) continue;
+ generators.push({ file, text: await readFile(/* turbopackIgnore: true */ file, "utf8").catch(() => "") });
+ }
+ }
+ if (drafts.length || generators.length) {
+ drafts.sort((a, b) => a.file.localeCompare(b.file));
+ generators.sort((a, b) => a.file.localeCompare(b.file));
+ out.push({ dir, name: e.name, drafts, generators });
+ }
+ }
+ return out.sort((a, b) => a.name.localeCompare(b.name));
+}
+
+/** Is `draft` this report's? */
+function draftMatches(draft, keys) {
+ return (draft.id !== null && keys.has(draft.id)) || keys.has(draft.slug) || keys.has(`polemic-${draft.slug}`);
+}
+
+/**
+ * The source of truth for one report, or null when no workspace claims it.
+ *
+ * @param {string} siteId
+ * @param {string} reportId
+ * @param {{ reportsRoot?: string }} [opts]
+ * @returns {Promise<{ draft?: string, generator?: string, how: string, workspace?: string } | null>}
+ */
+export async function sourceFor(siteId, reportId, { reportsRoot = REPORTS_ROOT } = {}) {
+ const workspaces = await scanWorkspaces(reportsRoot);
+ const keys = idKeys(reportId);
+ const namesSite = (ws) => ws.generators.some((g) => names(g.text, siteId));
+
+ const cands = [];
+ for (const ws of workspaces) {
+ for (const d of ws.drafts) {
+ if (!draftMatches(d, keys)) continue;
+ const score = (namesSite(ws) ? 4 : 0) + (d.id === reportId ? 2 : 0) + (d.slug === reportId.replace(/^polemic-/, "") ? 1 : 0);
+ cands.push({ ws, d, score });
+ }
+ }
+ cands.sort((a, b) => b.score - a.score);
+ const top = cands[0];
+ const unique = top && (cands.length === 1 || cands[1].score < top.score);
+ const draft = unique ? top : null;
+
+ const pool = draft ? [draft.ws] : workspaces;
+ let gen = null;
+ let genScore = 0;
+ for (const ws of pool) {
+ for (const g of ws.generators) {
+ const site = names(g.text, siteId);
+ const report = names(g.text, reportId);
+ if (!site && !report) continue;
+ const base = path.basename(g.file);
+ const score =
+ (report ? 3 : 0) + (site ? 2 : 0) + (draft && /\bdrafts\b/.test(g.text) ? 1 : 0) + (/^(make-site|polemics)\./.test(base) ? 0.5 : 0);
+ if (score > genScore) {
+ gen = { ws, g };
+ genScore = score;
+ }
+ }
+ }
+ // Without a draft, a generator that only names the SITE is every report's
+ // generator and says nothing about this one; keep it only if it names the id.
+ // A bare id (`deleted`, `poker`) is also an English word, so naming it is
+ // only evidence when the generator names the site too.
+ if (!draft && gen && !(names(gen.g.text, reportId) && (reportId.includes("-") || names(gen.g.text, siteId)))) gen = null;
+ if (!draft && !gen) {
+ if (cands.length > 1) {
+ return { how: `several drafts match ${reportId}: ${cands.map((c) => tildify(c.d.file)).join(", ")}; none chosen` };
+ }
+ return null;
+ }
+
+ const how = [];
+ if (draft) {
+ const by = draft.d.id === reportId ? `id ${reportId}` : draft.d.id && keys.has(draft.d.id) ? `id ${draft.d.id}` : `file name ${draft.d.slug}`;
+ how.push(`draft matched by ${by}`);
+ if (cands.length > 1) how.push(`preferred over ${cands.length - 1} other`);
+ }
+ if (gen) {
+ const named = [siteId, reportId].filter((id) => names(gen.g.text, id));
+ how.push(`generator names ${named.join(" and ")}`);
+ }
+ const out = { how: how.join("; ") };
+ if (draft) out.draft = tildify(draft.d.file);
+ if (gen) out.generator = tildify(gen.g.file);
+ out.workspace = tildify((draft?.ws ?? gen?.ws).dir);
+ return out;
+}
+
+/**
+ * The workspace directories a site's articles come from: every workspace that
+ * holds a matched draft for one of `reportIds`, or a generator that names the
+ * site. Absolute paths, sorted.
+ *
+ * @param {string} siteId
+ * @param {string[]} reportIds
+ * @param {{ reportsRoot?: string }} [opts]
+ */
+export async function siteWorkspaces(siteId, reportIds, { reportsRoot = REPORTS_ROOT } = {}) {
+ const workspaces = await scanWorkspaces(reportsRoot);
+ const dirs = new Set();
+ for (const ws of workspaces) {
+ if (ws.generators.some((g) => names(g.text, siteId))) dirs.add(ws.dir);
+ }
+ for (const id of reportIds) {
+ const s = await sourceFor(siteId, id, { reportsRoot });
+ const ws = s?.draft ? workspaces.find((w) => w.drafts.some((d) => tildify(d.file) === s.draft)) : null;
+ if (ws) dirs.add(ws.dir);
+ }
+ return [...dirs].sort();
+}
diff --git a/umtool/lib/articles/sources.test.mjs b/umtool/lib/articles/sources.test.mjs
@@ -0,0 +1,69 @@
+// Finding an article's draft and generator, on the three workspace layouts
+// the live private sites use.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { clearSourcesCache, names, siteWorkspaces, sourceFor } from "./sources.mjs";
+
+async function put(file, text) {
+ await mkdir(path.dirname(file), { recursive: true });
+ await writeFile(file, text);
+}
+
+async function fixture() {
+ const root = await mkdtemp(path.join(tmpdir(), "umtool-sources-"));
+ // candalyzer: drafts/<bare>.json with id polemic-<bare>; generator site/polemics.py;
+ // a hand-written fact-check whose generator names it; a backup generator.
+ await put(path.join(root, "candace/polemics/drafts/israel.json"), JSON.stringify({ id: "polemic-israel" }));
+ await put(path.join(root, "candace/site/polemics.py"), 'OUT = ".../sites/candalyzer/reports"\nfor f in DRAFTS.glob("drafts/*.json"): rid = f"polemic-{slug}"\n');
+ await put(path.join(root, "candace/site/make-report.py"), 'SITE = "candalyzer"\nREPORT = "deconstruction-fact-check"\n');
+ await put(path.join(root, "candace/site/make-report.pre-2026-10-05.py"), 'REPORT = "polemic-israel" # candalyzer\n');
+ // jasolyzer-private: drafts/<bare>.json, generator polemics/make-site.py
+ await put(path.join(root, "pirate/polemics/drafts/skg.json"), JSON.stringify({ id: "polemic-skg" }));
+ await put(path.join(root, "pirate/polemics/make-site.py"), 'SITE = "jasolyzer-private"\nrid = f"polemic-{slug}" # from drafts\n');
+ // jeralyzer-private: the report id is BARE (`blame`), the draft id is polemic-blame
+ await put(path.join(root, "quartering/polemics/drafts/blame.json"), JSON.stringify({ id: "polemic-blame" }));
+ await put(path.join(root, "quartering/polemics/make-site.py"), 'SITE = "jeralyzer-private"\nrid = slug # drafts\n');
+ // a decoy: a public site's workspace with a same-named draft
+ await put(path.join(root, "decoy/polemics/drafts/blame.json"), JSON.stringify({ id: "blame" }));
+ await put(path.join(root, "decoy/polemics/make-site.py"), 'SITE = "jeralyzer"\n');
+ clearSourcesCache();
+ return root;
+}
+
+test("each layout finds its draft and generator", async () => {
+ const root = await fixture();
+ const o = { reportsRoot: root };
+ const isr = await sourceFor("candalyzer", "polemic-israel", o);
+ assert.equal(isr.draft, path.join(root, "candace/polemics/drafts/israel.json"));
+ assert.equal(isr.generator, path.join(root, "candace/site/polemics.py"));
+ assert.match(isr.how, /draft matched by id polemic-israel/);
+
+ const fc = await sourceFor("candalyzer", "deconstruction-fact-check", o);
+ assert.equal(fc.draft, undefined);
+ assert.equal(fc.generator, path.join(root, "candace/site/make-report.py"));
+
+ const skg = await sourceFor("jasolyzer-private", "polemic-skg", o);
+ assert.equal(skg.draft, path.join(root, "pirate/polemics/drafts/skg.json"));
+ assert.equal(skg.generator, path.join(root, "pirate/polemics/make-site.py"));
+
+ // The decoy's draft id is exactly `blame`, but its workspace never names the site.
+ const blame = await sourceFor("jeralyzer-private", "blame", o);
+ assert.equal(blame.draft, path.join(root, "quartering/polemics/drafts/blame.json"));
+ assert.equal(blame.generator, path.join(root, "quartering/polemics/make-site.py"));
+ assert.match(blame.how, /preferred over 1 other/);
+
+ assert.equal(await sourceFor("candalyzer", "no-such-report", o), null);
+ assert.deepEqual(await siteWorkspaces("jasolyzer-private", ["polemic-skg"], o), [path.join(root, "pirate")]);
+ await rm(root, { recursive: true });
+});
+
+test("names() matches whole ids only", () => {
+ assert.equal(names('"jasolyzer-private"', "jasolyzer"), false);
+ assert.equal(names("sites/jasolyzer/reports", "jasolyzer"), true);
+ assert.equal(names("polemic-blame", "blame"), false);
+});
diff --git a/umtool/lib/articles/workspace.mjs b/umtool/lib/articles/workspace.mjs
@@ -0,0 +1,65 @@
+// The files an article was WRITTEN from, for reading beside it: a workspace's
+// drafts, the rendered drafts and briefs under polemics/, and the notes a
+// workspace keeps at its top level. Read-only -- this lists and serves, it
+// never writes a workspace.
+//
+// polemics/drafts/*.json the drafts (source of truth)
+// polemics/out/*.{md,html} what the drafts render to
+// polemics/*.md BRIEF.md, NOTES.md, PRIVACY-SWEEP.md, …
+// <ws>/{NOTES,BRIEF,LEADS,PLAN,PRIVACY-SWEEP}.md
+// <ws>/site/PLAN.md, <ws>/site/MERGE-PLAN.md
+import { readdir, realpath, stat } from "node:fs/promises";
+import path from "node:path";
+import { REPORTS_ROOT } from "../paths.mjs";
+
+export const TOP_LEVEL = ["NOTES.md", "BRIEF.md", "LEADS.md", "PLAN.md", "PRIVACY-SWEEP.md"];
+const SITE_LEVEL = ["PLAN.md", "MERGE-PLAN.md"];
+export const WORKSPACE_EXT = /\.(md|html|json)$/;
+
+const statFile = (p) => stat(/* turbopackIgnore: true */ p).then((s) => (s.isFile() ? s : null), () => null);
+
+/**
+ * Every listed file of one workspace: `{ rel, abs, kind, bytes, mtimeMs }`,
+ * `kind` one of "draft" | "out" | "doc". Sorted: drafts, docs, outputs; by name.
+ *
+ * @param {string} wsDir
+ */
+export async function workspaceFiles(wsDir) {
+ const out = [];
+ const add = async (rel, kind) => {
+ const abs = path.join(/* turbopackIgnore: true */ wsDir, rel);
+ const st = await statFile(abs);
+ if (st) out.push({ rel, abs, kind, bytes: st.size, mtimeMs: Math.round(st.mtimeMs) });
+ };
+ const ls = (rel) => readdir(/* turbopackIgnore: true */ path.join(/* turbopackIgnore: true */ wsDir, rel)).catch(() => []);
+ for (const f of await ls("polemics/drafts")) if (f.endsWith(".json")) await add(`polemics/drafts/${f}`, "draft");
+ for (const f of await ls("polemics/out")) if (/\.(md|html)$/.test(f)) await add(`polemics/out/${f}`, "out");
+ for (const f of await ls("polemics")) if (f.endsWith(".md")) await add(`polemics/${f}`, "doc");
+ for (const f of TOP_LEVEL) await add(f, "doc");
+ for (const f of SITE_LEVEL) await add(`site/${f}`, "doc");
+ const rank = { draft: 0, doc: 1, out: 2 };
+ return out.sort((a, b) => rank[a.kind] - rank[b.kind] || a.rel.localeCompare(b.rel));
+}
+
+/**
+ * A workspace file a client named, or null: `ws` must be a directory directly
+ * under REPORTS_ROOT and `rel` one of the files workspaceFiles lists for it,
+ * and its REAL path must stay inside the workspace.
+ *
+ * @param {string} ws the workspace's name under REPORTS_ROOT
+ * @param {string} rel
+ * @param {{ reportsRoot?: string }} [opts]
+ */
+export async function workspaceFile(ws, rel, { reportsRoot = REPORTS_ROOT } = {}) {
+ if (typeof ws !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(ws) || ws === "..") return null;
+ if (typeof rel !== "string" || rel.includes("\0") || rel.split("/").some((s) => s === ".." || s === "" || s === ".")) return null;
+ const dir = path.join(/* turbopackIgnore: true */ reportsRoot, ws);
+ const listed = (await workspaceFiles(dir)).find((f) => f.rel === rel);
+ if (!listed) return null;
+ const [realDir, realAbs] = await Promise.all([
+ realpath(/* turbopackIgnore: true */ dir).catch(() => null),
+ realpath(/* turbopackIgnore: true */ listed.abs).catch(() => null),
+ ]);
+ if (!realDir || !realAbs || !realAbs.startsWith(realDir + path.sep)) return null;
+ return { ...listed, real: realAbs };
+}
diff --git a/umtool/lib/decisions.ts b/umtool/lib/decisions.ts
@@ -4,6 +4,7 @@ import { DEFAULT_TARGET, loudnessVerdict } from "./loudness-types";
import { buildStatus, readManifest } from "./manifest";
import { readSpec, validateSpec } from "./spec";
import { readNotes, type NoteMap } from "./notes";
+import { listNotesFiles } from "./annotations/targets.mjs";
import { acceptedFor, readThumbAccepted, readThumbManifest, thumbNamesFor } from "./thumbs";
// ---------------------------------------------------------------------------
@@ -298,3 +299,79 @@ export function decisionsMarkdown(items: Decision[]): string[] {
}
return out;
}
+
+// ---------------------------------------------------------------------------
+// Open NOTES -- on an article (sites/<site>/reports/<id>/notes.json) or on a
+// report-video project (<project>/notes.json) -- are open decisions: somebody
+// asked for a change and nobody has answered it. One row per open note, kind
+// `open-note`, linked to the note on its page. A notes.json that does not
+// parse is blocking: nothing can write to it until somebody fixes it by hand.
+//
+// An ARTICLE is not a project, so its row's `project` is its page's path
+// (`sites/<site>/<report>`), which is also where its href points.
+// ---------------------------------------------------------------------------
+
+const firstLine = (s: string, max = 140) => {
+ const line = s.split("\n").find((l) => l.trim()) ?? "";
+ return line.length > max ? `${line.slice(0, max - 1)}…` : line;
+};
+
+function anchorLabel(a: { kind: string; [k: string]: unknown }): string {
+ switch (a.kind) {
+ case "text":
+ return `“${firstLine(String(a.quote ?? ""), 48)}”`;
+ case "cite":
+ return `cite ${a.cite}`;
+ case "section":
+ return `section ${a.section}`;
+ case "moment":
+ return `${a.file} @ ${Number(a.t).toFixed(1)}s`;
+ case "entry":
+ return `entry ${a.entry}`;
+ case "take":
+ return `take ${a.take}`;
+ case "edit":
+ return `edit ${a.field}${a.entry ? ` on ${a.entry}` : ""}`;
+ default:
+ return "whole";
+ }
+}
+
+export async function noteDecisions(): Promise<Decision[]> {
+ const files = await listNotesFiles().catch(() => []);
+ const out: Decision[] = [];
+ for (const f of files) {
+ const article = f.kind === "article";
+ const project = article ? `sites/${f.id}` : f.id;
+ const page = article ? `/sites/${f.id}` : `/browse/${f.id}`;
+ if (f.error || !f.doc) {
+ out.push({
+ kind: "unreadable-notes",
+ project,
+ projectKind: article ? "article" : ((f as { projectKind?: string }).projectKind ?? "project"),
+ target: "notes.json",
+ why: `notes.json does not parse (${f.error ?? "unknown"}); nothing will write to it until it is fixed`,
+ href: page,
+ severity: "blocking",
+ at: Date.now(),
+ });
+ continue;
+ }
+ for (const n of f.doc.notes) {
+ if (n.status !== "open") continue;
+ const replies = n.replies.length ? ` · ${n.replies.length} repl${n.replies.length === 1 ? "y" : "ies"}` : "";
+ const at = Date.parse(n.updatedAt);
+ out.push({
+ kind: "open-note",
+ project,
+ projectKind: article ? "article" : ((f as { projectKind?: string }).projectKind ?? "project"),
+ target: anchorLabel(n.anchor as { kind: string }),
+ why: `${firstLine(n.text)}${replies}`,
+ href: `${page}?note=${encodeURIComponent(n.id)}`,
+ severity: "open",
+ at: Number.isFinite(at) ? at : 0,
+ });
+ }
+ }
+ return out;
+}
diff --git a/umtool/lib/paths.mjs b/umtool/lib/paths.mjs
@@ -5,6 +5,7 @@
// `umtool ls` and the page it is supposed to describe. lib/paths.ts re-exports
// everything here with types; nothing computes a root twice.
import { existsSync } from "node:fs";
+import { lstat, realpath } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { SONG_DATA, SONG_REPORTS } from "../song/paths.mjs";
@@ -208,12 +209,74 @@ export const CHANNELS_DIR = path.resolve(
: path.join(/* turbopackIgnore: true */ REPO_ROOT, "transcripts", "channels")),
);
+// The SITES -- `transcripts/sites/<site>/`, each a site.json and its reports
+// (`reports/<id>/report.json`, `video.mp4`, `poster.jpg`). Resolved the way
+// CHANNELS_DIR is, and the way common/lib/paths.ts resolves its sitesDir, so
+// one `SITES_DIR` confines both. Readable only; the ONE file in it umtool may
+// write is a report's notes.json, and only through isCorpusNotesFile below.
+export const SITES_DIR = path.resolve(
+ /* turbopackIgnore: true */
+ process.env.SITES_DIR ??
+ (process.env.TRANSCRIPTS_DIR
+ ? path.join(/* turbopackIgnore: true */ process.env.TRANSCRIPTS_DIR, "sites")
+ : path.join(/* turbopackIgnore: true */ REPO_ROOT, "transcripts", "sites")),
+);
+
export const READ_ROOTS = dedupe(
process.env.MIX_ROOTS
? process.env.MIX_ROOTS.split(":").filter(Boolean)
- : [SONG_REPORTS, REPORTS_ROOT, SONG_DATA, SONG_SCRATCH, CHANNELS_DIR, MEDIA_ROOT],
+ : [SONG_REPORTS, REPORTS_ROOT, SONG_DATA, SONG_SCRATCH, CHANNELS_DIR, MEDIA_ROOT, SITES_DIR],
);
+/** A site id or a report id: one lowercase url-safe segment (common/lib/report/schema.ts REPORT_ID_RE). */
+export const SEGMENT_RE = /^[a-z0-9][a-z0-9-]{0,63}$/;
+export const NOTES_FILENAME = "notes.json";
+
+/**
+ * The ONE corpus path umtool may write: `SITES_DIR/<site>/reports/<id>/notes.json`.
+ *
+ * Lexically: exactly four segments under SITES_DIR, the second `reports`, the
+ * site and report ids in the report-id grammar, the file named notes.json, and
+ * no `..` or doubled separator anywhere. Then on disk: the report directory's
+ * REAL path must be the lexical one under the real SITES_DIR -- a report dir
+ * that is a symlink out of its site (or a site dir that is one) is refused --
+ * and it must already exist: a note never creates a report directory. A
+ * notes.json that exists and is not a plain file (a symlink) is refused too.
+ *
+ * WRITE_ROOTS is untouched: nothing else in the corpus becomes writable.
+ *
+ * @param {string} abs
+ * @param {{ sitesDir?: string }} [opts]
+ * @returns {Promise<boolean>}
+ */
+export async function isCorpusNotesFile(abs, { sitesDir = SITES_DIR } = {}) {
+ if (typeof abs !== "string" || !path.isAbsolute(abs) || abs.includes("\0")) return false;
+ if (path.resolve(/* turbopackIgnore: true */ abs) !== abs) return false;
+ const root = path.resolve(/* turbopackIgnore: true */ sitesDir);
+ const rel = path.relative(/* turbopackIgnore: true */ root, abs);
+ if (!rel || rel.startsWith("..") || path.isAbsolute(rel)) return false;
+ const parts = rel.split(path.sep);
+ if (parts.length !== 4) return false;
+ const [site, reports, report, name] = parts;
+ if (!SEGMENT_RE.test(site) || reports !== "reports" || !SEGMENT_RE.test(report) || name !== NOTES_FILENAME) {
+ return false;
+ }
+ const [realRoot, realDir] = await Promise.all([
+ realpath(/* turbopackIgnore: true */ root).catch(() => null),
+ realpath(/* turbopackIgnore: true */ path.dirname(/* turbopackIgnore: true */ abs)).catch(() => null),
+ ]);
+ if (!realRoot || !realDir) return false;
+ if (realDir !== path.join(/* turbopackIgnore: true */ realRoot, site, "reports", report)) return false;
+ const st = await lstat(/* turbopackIgnore: true */ abs).catch(() => null);
+ return !st || st.isFile();
+}
+
+/** `SITES_DIR/<site>/reports/<report>/notes.json`, or null for a bad id. */
+export function corpusNotesFile(site, report, { sitesDir = SITES_DIR } = {}) {
+ if (!SEGMENT_RE.test(String(site)) || !SEGMENT_RE.test(String(report))) return null;
+ return path.join(/* turbopackIgnore: true */ path.resolve(/* turbopackIgnore: true */ sitesDir), site, "reports", report, NOTES_FILENAME);
+}
+
export const WRITE_ROOTS = dedupe(
process.env.MIX_WRITE_ROOTS
? process.env.MIX_WRITE_ROOTS.split(":").filter(Boolean)
diff --git a/umtool/lib/paths.ts b/umtool/lib/paths.ts
@@ -20,7 +20,9 @@ import path from "node:path";
// ---------------------------------------------------------------------------
export {
CACHE_DIR,
+ CHANNELS_DIR,
cacheFile,
+ corpusNotesFile,
INDEX_DIR,
MEDIA_ROOT,
MEDIA_ROOTS,
@@ -31,8 +33,10 @@ export {
SONG_DATA,
SONG_REPORTS,
SONG_SCRATCH,
+ SITES_DIR,
WRITE_ROOTS,
inside,
+ isCorpusNotesFile,
labelFor,
mediaMirror,
resolveInRoots,
diff --git a/umtool/lib/projects.ts b/umtool/lib/projects.ts
@@ -5,7 +5,7 @@ import { collapseFolders, foldersFor, walkProjects } from "./projects/walk.mjs";
import { SONG_KIND } from "./projects/song.mjs";
import { openIndex, signRecord } from "./projects/index-db.mjs";
import { BROWSE_ROOT } from "./browse";
-import { decisionsForSong } from "./decisions";
+import { decisionsForSong, noteDecisions } from "./decisions";
import { listMedia, listMediaUnder, type MediaRow } from "./media";
import type { Decision } from "./decisions";
import type {
@@ -302,8 +302,10 @@ const RANK: Record<string, number> = { blocking: 0, open: 1, info: 2 };
export async function openDecisions(): Promise<Decision[]> {
const refs = await projectRefs();
const per = await Promise.all(refs.map(decisionsForProject));
- return per
- .flat()
+ // Open notes on articles and report videos (lib/decisions.ts noteDecisions):
+ // an article is not a project, so they are added here, not per project.
+ const notes = await noteDecisions();
+ return [...per.flat(), ...notes]
.sort((a, b) => RANK[a.severity] - RANK[b.severity] || b.at - a.at);
}
diff --git a/umtool/lib/projects/kinds.mjs b/umtool/lib/projects/kinds.mjs
@@ -104,6 +104,12 @@ export const PROJECT_KINDS = [
// `brand` offers report-to-video's presets (render.brand); none is the
// default and writes the manifest it always did.
scaffold: { fields: ["from", "siteOrigin", "seed", "brand"], brands: BRAND_CHOICES },
+ // Takes a notes.json beside its manifest (lib/annotations/targets.mjs):
+ // timed notes on its cuts, row notes, take notes, edit notes.
+ notes: true,
+ // Its manifest can be an ARTICLE's video (lib/articles/links.mjs): a
+ // top-level `article`, else its slug against the report id.
+ linksArticles: true,
},
{
id: "song",
@@ -182,6 +188,12 @@ if (process.env.E2E_UMTOOL_EXTRA_KINDS) {
export const kindById = (id) => PROJECT_KINDS.find((k) => k.id === id) ?? null;
+/** Does a project of this kind keep a notes.json (lib/annotations)? Declared on the kind, never branched on its id. */
+export const kindTakesNotes = (id) => kindById(id)?.notes === true;
+
+/** Can a project of this kind be an article's video (lib/articles/links.mjs)? Declared on the kind. */
+export const kindLinksArticles = (id) => kindById(id)?.linksArticles === true;
+
/** What a client component needs, with none of what it must not have. */
export const kindMeta = (k) => ({
id: k.id,
diff --git a/umtool/lib/report/edit-notes.mjs b/umtool/lib/report/edit-notes.mjs
@@ -0,0 +1,174 @@
+// An edit made in umtool to a GENERATED manifest, written down for the agent
+// that generates it.
+//
+// A manifest with `generatedBy` (polemics/video/make-videos.py, …) is rebuilt
+// from the generator's inputs, and the rebuild overwrites whatever was edited
+// here. So edits are still allowed -- the operator is watching the cut and the
+// fix belongs there -- and each one becomes an `edit` note in the project's
+// notes.json (lib/annotations/): which entry, which field, from what, to what.
+// The agent ports it into the generator's inputs (BEATS, drafts) and resolves
+// the note, and the next rebuild keeps it.
+//
+// ONE wrapper does this for every manifest writer (lib/report/guard.ts), by
+// diffing the manifest before and after the write -- so a writer added later
+// is covered without knowing this exists.
+//
+// Repeated saves of one field COALESCE: an open edit note on the same entry and
+// field keeps its original `from` and takes the new `to`, and an edit that
+// returns the field to its `from` deletes the note. Dragging a window five
+// times is one note, and dragging it back is none.
+import { readNotes } from "../annotations/store.mjs";
+import { writeNote } from "../annotations/targets.mjs";
+
+const MAX_VALUE = 3000;
+
+/** A value small enough to keep in a note; a large one is summarised. */
+function keep(v) {
+ if (v === undefined) return null;
+ const s = JSON.stringify(v);
+ if (s.length <= MAX_VALUE) return v;
+ return `(${Array.isArray(v) ? `${v.length} items` : "object"}, ${s.length} characters)`;
+}
+
+const ID_LISTS = ["posts", "ledger"];
+
+const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+
+/** Key the timeline by `id` (and its variant, since twins share an id). */
+function entryKey(e) {
+ return e?.variant ? `${e.id}@${e.variant}` : String(e?.id);
+}
+
+/**
+ * Every change between two manifests, as `{ entry?, field, from, to }`.
+ *
+ * timeline entry field { entry: id, field: "<key>" }
+ * an entry added { entry: id, field: "timeline+", from: null, to: <entry> }
+ * an entry removed { entry: id, field: "timeline-", from: <entry>, to: null }
+ * the order { field: "timeline.order", from: [ids], to: [ids] } (survivors only)
+ * a post, a ledger claim { entry: id, field: "posts.<key>" | "posts+" | "posts-" } (ledger alike)
+ * anything else { field: "<top>.<key>" } one level down (render.chrome, provenance.siteOrigin)
+ *
+ * @param {Record<string, any>} before
+ * @param {Record<string, any>} after
+ */
+export function editsBetween(before, after) {
+ const out = [];
+ const a = (before?.timeline ?? []).filter((e) => e && e.id != null);
+ const b = (after?.timeline ?? []).filter((e) => e && e.id != null);
+ const byA = new Map(a.map((e) => [entryKey(e), e]));
+ const byB = new Map(b.map((e) => [entryKey(e), e]));
+ for (const [k, e] of byA) if (!byB.has(k)) out.push({ entry: String(e.id), field: "timeline-", from: keep(e), to: null });
+ for (const [k, e] of byB) if (!byA.has(k)) out.push({ entry: String(e.id), field: "timeline+", from: null, to: keep(e) });
+ const sa = a.map(entryKey).filter((k) => byB.has(k));
+ const sb = b.map(entryKey).filter((k) => byA.has(k));
+ if (!same(sa, sb)) out.push({ field: "timeline.order", from: keep(sa), to: keep(sb) });
+ for (const [k, x] of byA) {
+ const y = byB.get(k);
+ if (!y) continue;
+ for (const f of new Set([...Object.keys(x), ...Object.keys(y)])) {
+ // sectionEnter follows the order; the order change already says it.
+ if (f === "sectionEnter" || same(x[f], y[f])) continue;
+ out.push({ entry: String(x.id), field: f, from: keep(x[f]), to: keep(y[f]) });
+ }
+ }
+
+ // Lists of things with ids -- the posts, the ledger's claims -- by id.
+ for (const list of ID_LISTS) {
+ const pa = new Map((Array.isArray(before?.[list]) ? before[list] : []).map((p) => [String(p?.id), p]));
+ const pb = new Map((Array.isArray(after?.[list]) ? after[list] : []).map((p) => [String(p?.id), p]));
+ for (const [id, p] of pa) if (!pb.has(id)) out.push({ entry: id, field: `${list}-`, from: keep(p), to: null });
+ for (const [id, p] of pb) {
+ const q = pa.get(id);
+ if (!q) {
+ out.push({ entry: id, field: `${list}+`, from: null, to: keep(p) });
+ continue;
+ }
+ for (const f of new Set([...Object.keys(q), ...Object.keys(p)])) {
+ if (!same(q[f], p[f])) out.push({ entry: id, field: `${list}.${f}`, from: keep(q[f]), to: keep(p[f]) });
+ }
+ }
+ }
+
+ for (const top of new Set([...Object.keys(before ?? {}), ...Object.keys(after ?? {})])) {
+ if (top === "timeline" || ID_LISTS.includes(top)) continue;
+ const x = before?.[top];
+ const y = after?.[top];
+ if (same(x, y)) continue;
+ const isObj = (v) => v && typeof v === "object" && !Array.isArray(v);
+ if (isObj(x) && isObj(y)) {
+ for (const f of new Set([...Object.keys(x), ...Object.keys(y)])) {
+ if (!same(x[f], y[f])) out.push({ field: `${top}.${f}`, from: keep(x[f]), to: keep(y[f]) });
+ }
+ } else {
+ out.push({ field: top, from: keep(x), to: keep(y) });
+ }
+ }
+ return out;
+}
+
+/** The sentence an edit note carries; the anchor carries the values. */
+export function editText(edit, generatedBy) {
+ const where = edit.entry ? `${edit.entry} ` : "";
+ const what =
+ edit.field === "timeline+"
+ ? "added to the timeline"
+ : edit.field === "timeline-"
+ ? "removed from the timeline"
+ : edit.field === "timeline.order"
+ ? "the timeline was re-ordered"
+ : edit.field.endsWith("+")
+ ? `${edit.field.slice(0, -1)} entry added`
+ : edit.field.endsWith("-")
+ ? `${edit.field.slice(0, -1)} entry removed`
+ : `${edit.field} changed`;
+ return `${where}${what} in umtool. Port it into the inputs of ${generatedBy}; a rebuild of manifests overwrites it.`;
+}
+
+/**
+ * Write `edits` into a project's notes as `edit` notes, coalescing with open
+ * ones on the same entry and field (see the top of this file). Returns how many
+ * notes were added, updated and deleted. Errors are returned, not thrown: the
+ * manifest write already happened, and a notes file that will not take a note
+ * must not turn a saved edit into a reported failure.
+ *
+ * @param {{ file: string, subject: Record<string, string>, source: () => Promise<any> }} target
+ * @param {Array<{ entry?: string, field: string, from: unknown, to: unknown }>} edits
+ * @param {string} generatedBy
+ */
+export async function recordEdits(target, edits, generatedBy) {
+ const counts = { added: 0, updated: 0, deleted: 0, errors: /** @type {string[]} */ ([]) };
+ for (const edit of edits) {
+ try {
+ const { doc } = await readNotes(target.file);
+ const open = (doc?.notes ?? []).find(
+ (n) =>
+ n.status === "open" &&
+ n.author === "operator" &&
+ n.anchor.kind === "edit" &&
+ n.anchor.field === edit.field &&
+ (n.anchor.entry ?? null) === (edit.entry ?? null),
+ );
+ if (open) {
+ const from = /** @type {any} */ (open.anchor).from;
+ // Put back as it was: the note says nothing -- unless somebody has
+ // already replied to it, and then it stays for them to resolve.
+ if (same(from, edit.to) && !open.replies.length) {
+ await writeNote(target, { op: "delete", id: open.id }, { by: "operator" });
+ counts.deleted += 1;
+ } else {
+ const anchor = { kind: "edit", field: edit.field, from, to: edit.to, ...(edit.entry ? { entry: edit.entry } : {}) };
+ await writeNote(target, { op: "edit", id: open.id, anchor }, { by: "operator" });
+ counts.updated += 1;
+ }
+ continue;
+ }
+ const anchor = { kind: "edit", field: edit.field, from: edit.from, to: edit.to, ...(edit.entry ? { entry: edit.entry } : {}) };
+ await writeNote(target, { op: "add", text: editText(edit, generatedBy), anchor }, { by: "operator" });
+ counts.added += 1;
+ } catch (e) {
+ counts.errors.push(e instanceof Error ? e.message : String(e));
+ }
+ }
+ return counts;
+}
diff --git a/umtool/lib/report/edit-notes.test.mjs b/umtool/lib/report/edit-notes.test.mjs
@@ -0,0 +1,67 @@
+// Edits to a generated manifest, as notes: the diff, and the coalescing.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { readNotes } from "../annotations/store.mjs";
+import { editsBetween, editText, recordEdits } from "./edit-notes.mjs";
+
+const m = () => ({
+ generatedBy: "polemics/video/make-videos.py",
+ render: { fps: 30, chrome: { engine: "hyperframes" } },
+ timeline: [
+ { type: "clip", id: "a", start: 1, end: 5, quote: "q" },
+ { type: "clip", id: "b", start: 10, end: 15 },
+ { type: "card", id: "k" },
+ ],
+ posts: [{ id: "p1", text: "t", attachTo: "a" }],
+ ledger: [{ id: "c1", scope: "x" }],
+});
+
+test("editsBetween names each change by entry and field", () => {
+ const before = m();
+ const after = m();
+ after.timeline[0].quote = "new";
+ after.timeline[0].start = 2;
+ after.timeline = [after.timeline[1], after.timeline[0], { type: "card", id: "k2" }];
+ after.posts[0].attachTo = "b";
+ after.ledger[0].scope = "y";
+ after.render.chrome.layout = "deck";
+ const e = editsBetween(before, after);
+ const keyOf = (x) => `${x.entry ?? ""}|${x.field}`;
+ assert.deepEqual(
+ e.map(keyOf).sort(),
+ ["a|quote", "a|start", "c1|ledger.scope", "k2|timeline+", "k|timeline-", "p1|posts.attachTo", "|render.chrome", "|timeline.order"].sort(),
+ );
+ const quote = e.find((x) => x.field === "quote");
+ assert.deepEqual([quote.from, quote.to], ["q", "new"]);
+ assert.deepEqual(editsBetween(before, m()), []);
+ assert.match(editText({ entry: "a", field: "quote" }, "gen.py"), /^a quote changed in umtool\. Port it into the inputs of gen\.py/);
+});
+
+test("recordEdits adds, coalesces, and deletes a note an edit put back", async () => {
+ const dir = await mkdtemp(path.join(tmpdir(), "umtool-editnotes-"));
+ const target = {
+ file: path.join(dir, "notes.json"),
+ subject: { kind: "video-project", project: "x/y" },
+ source: async () => ({ manifest: "~/x/y/video.manifest.json" }),
+ };
+ try {
+ let c = await recordEdits(target, [{ entry: "a", field: "start", from: 1, to: 2 }], "gen.py");
+ assert.deepEqual([c.added, c.updated, c.deleted], [1, 0, 0]);
+ c = await recordEdits(target, [{ entry: "a", field: "start", from: 2, to: 3 }], "gen.py");
+ assert.deepEqual([c.added, c.updated], [0, 1]);
+ let doc = (await readNotes(target.file)).doc;
+ assert.equal(doc.notes.length, 1);
+ assert.deepEqual([doc.notes[0].anchor.from, doc.notes[0].anchor.to], [1, 3]);
+ assert.equal(doc.source.manifest, "~/x/y/video.manifest.json");
+ c = await recordEdits(target, [{ entry: "a", field: "start", from: 3, to: 1 }], "gen.py");
+ assert.equal(c.deleted, 1);
+ assert.equal((await readNotes(target.file)).doc, null);
+ } finally {
+ await rm(dir, { recursive: true });
+ }
+});
diff --git a/umtool/lib/report/guard.ts b/umtool/lib/report/guard.ts
@@ -0,0 +1,38 @@
+import { projectTargetFor } from "@/lib/annotations/targets.mjs";
+import { readManifest } from "@/lib/projects/report.mjs";
+import { editsBetween, recordEdits } from "./edit-notes.mjs";
+
+// THE one wrapper every manifest writer's route goes through.
+//
+// A manifest with `generatedBy` is rebuilt by its generator, and the rebuild
+// overwrites edits made here (the banner on the project page, the bench and
+// the On-screen section says so). The edit is still made; what this adds is a
+// record of it: the manifest is read before and after the write, and every
+// change becomes an `edit` note in the project's notes.json for the agent to
+// port into the generator's inputs (lib/report/edit-notes.mjs). A hand-edited
+// manifest (no `generatedBy`) is written exactly as before.
+//
+// The notes are written AFTER the manifest and never fail the request: the
+// edit is saved either way, and `editNotes.errors` says when its note is not.
+
+export type EditNotes = { generatedBy: string; added: number; updated: number; deleted: number; errors: string[] };
+
+export async function withEditNotes<T>(
+ project: { id: string; dir: string; kind: string },
+ write: () => Promise<T>,
+): Promise<{ result: T; editNotes: EditNotes | null }> {
+ const before = await readManifest(project.dir);
+ const result = await write();
+ const generatedBy = typeof before?.generatedBy === "string" ? before.generatedBy.trim() : "";
+ if (!generatedBy) return { result, editNotes: null };
+ const after = await readManifest(project.dir);
+ const edits = editsBetween(before, after);
+ if (!edits.length) return { result, editNotes: { generatedBy, added: 0, updated: 0, deleted: 0, errors: [] } };
+ try {
+ const target = await projectTargetFor(project);
+ const counts = await recordEdits(target, edits, generatedBy);
+ return { result, editNotes: { generatedBy, ...counts } };
+ } catch (e) {
+ return { result, editNotes: { generatedBy, added: 0, updated: 0, deleted: 0, errors: [e instanceof Error ? e.message : String(e)] } };
+ }
+}
diff --git a/umtool/lib/report/manifest.mjs b/umtool/lib/report/manifest.mjs
@@ -30,10 +30,12 @@ import {
rolesGaps,
} from "umtool-report-to-video/ledger-totals";
import { isCalendarDate } from "umtool-report-to-video/attribution";
-import { normalizeOnscreen, validateChrome, validatePosts } from "umtool-report-to-video/deck";
+import { normalizeOnscreen, validateChrome, validatePosts, validateTeaser, validateTeasers } from "umtool-report-to-video/deck";
+import { applySectionEnter } from "./sections.mjs";
import { normalizeClaim, validateClaims } from "umtool-report-to-video/factcheck";
import { parseMuteFrom } from "./playback.mjs";
import { DELIVERABLES_MODES } from "./storage.mjs";
+import { createSnapshot, listSnapshots } from "./snapshots.mjs";
// Its own write queue, not lib/state.ts's.
//
@@ -787,3 +789,381 @@ export async function updateStorage(dir, { deliverables } = {}, { token = null }
return { storage: manifest.storage, token: nextToken, changed: true };
});
}
+
+
+// ---------------------------------------------------------------------------
+// STRUCTURE: re-ordering the cut, and the edits that add or remove a thing.
+//
+// Every writer above changes the fields of something already in the manifest.
+// These change WHAT IS IN IT -- the order of the timeline, which entries it
+// holds, a teaser's lines, the posts, the fact-check's labels -- so they are
+// the edits a person wants to take back. Each one snapshots the manifest into
+// revisions/ first (`auto-before-<op>`, at most one per op every two minutes,
+// so a burst of drags is one step back), and `undoStructural` restores the
+// newest of those.
+//
+// Same four rules as the rest of the file: the mtime token, tmp + rename under
+// the lock, 2 dp, and validation by the BUILD's own checks (deck.mjs,
+// factcheck.mjs) before anything is written, so a manifest these accept is one
+// the build accepts.
+//
+// An entry is named by its id. A timeline may repeat an id across variants
+// (`variant: "sourced"` / `"full"` twins); then the caller passes `at`, the
+// index it means, and a stale `at` (the id is not there any more) refuses.
+// ---------------------------------------------------------------------------
+
+export const AUTO_SNAPSHOT_PREFIX = "auto-before-";
+const AUTO_SNAPSHOT_EVERY_MS = 2 * 60 * 1000;
+const ENTRY_ID_RE = /^[A-Za-z0-9_-]{1,64}$/;
+
+/**
+ * Copy the manifest into revisions/ as `auto-before-<op>`, unless one for the
+ * same op was taken in the last two minutes. Never fatal: an undo point that
+ * cannot be taken is reported, and the edit still lands.
+ */
+export async function autoSnapshot(dir, op, { now = Date.now() } = {}) {
+ const label = `${AUTO_SNAPSHOT_PREFIX}${op}`;
+ try {
+ const recent = (await listSnapshots(dir)).find((s) => !s.legacy && s.label === label);
+ if (recent && now - recent.mtimeMs < AUTO_SNAPSHOT_EVERY_MS) return { skipped: true, rel: recent.rel };
+ return { skipped: false, ...(await createSnapshot(dir, { label })) };
+ } catch (e) {
+ return { skipped: true, error: e instanceof Error ? e.message : String(e) };
+ }
+}
+
+/** The index of entry `id`: `at` when it names it, else the only entry with that id. */
+export function entryIndex(timeline, id, at = null) {
+ if (at !== null && at !== undefined) {
+ const i = Number(at);
+ if (!Number.isInteger(i) || timeline[i]?.id !== id) {
+ throw new Error(`timeline[${at}] is not ${id} any more — reload`);
+ }
+ return i;
+ }
+ const hits = [];
+ timeline.forEach((e, i) => {
+ if (e?.id === id) hits.push(i);
+ });
+ if (!hits.length) throw new Error(`no timeline entry with id ${id}`);
+ if (hits.length > 1) throw new Error(`${id} is in the timeline ${hits.length} times — say which (at)`);
+ return hits[0];
+}
+
+/** An id not yet in the timeline: `base`, else `base-2`, `base-3`, … */
+export function freshEntryId(timeline, base) {
+ const clean = String(base).replace(/[^A-Za-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 56) || "entry";
+ const ids = new Set(timeline.map((e) => e?.id));
+ if (!ids.has(clean)) return clean;
+ for (let n = 2; ; n += 1) if (!ids.has(`${clean}-${n}`)) return `${clean}-${n}`;
+}
+
+/** Every reason the build would refuse the manifest's structure, after an edit. */
+function structureErrors(manifest) {
+ return [
+ ...validatePosts(manifest.posts, manifest.timeline ?? [], manifest.render),
+ ...validateClaims(manifest),
+ ...validateTeasers(manifest),
+ ];
+}
+
+/** The build's sentences, thrown whole so a route can return each one. */
+export class StructureRefused extends Error {
+ /** @param {string[]} errors */
+ constructor(errors) {
+ super(errors.join("; "));
+ this.name = "StructureRefused";
+ this.errors = errors;
+ }
+}
+
+/**
+ * One structural write: token, read, `mutate` (which throws to refuse), the
+ * build's checks, the auto snapshot of the file as it still is, then the write.
+ */
+async function structural(dir, op, token, mutate) {
+ return withManifestLock(async () => {
+ const current = await manifestToken(dir);
+ if (token !== null && current !== token) throw new StaleToken(token, current);
+ const manifest = JSON.parse(await readFile(manifestFile(dir), "utf8"));
+ if (!Array.isArray(manifest.timeline)) manifest.timeline = [];
+ // Only what THIS edit breaks refuses it: a manifest that already carries a
+ // problem the build would name (somebody's hand edit) can still be
+ // re-ordered, and the problem is still the build's to report.
+ const already = new Set(structureErrors(manifest));
+ const result = mutate(manifest);
+ const errors = structureErrors(manifest).filter((e) => !already.has(e));
+ if (errors.length) throw new StructureRefused(errors);
+ applySectionEnter(manifest.timeline);
+ const snapshot = await autoSnapshot(dir, op);
+ const nextToken = await writeManifestAtomic(dir, manifest);
+ return { ...result, snapshot, token: nextToken };
+ });
+}
+
+/**
+ * Move one entry to `toIndex` (its index in the timeline AFTER the move).
+ * `sectionEnter` is recomputed by report-to-video's own rule (sections.mjs),
+ * and only on a manifest that already uses it.
+ *
+ * @param {string} dir
+ * @param {string} id
+ * @param {number} toIndex
+ * @param {{ token?: string | null, at?: number | null }} [opts]
+ */
+export async function moveEntry(dir, id, toIndex, { token = null, at = null } = {}) {
+ return structural(dir, "move", token, (m) => {
+ const from = entryIndex(m.timeline, id, at);
+ const to = Number(toIndex);
+ if (!Number.isInteger(to) || to < 0 || to >= m.timeline.length) {
+ throw new Error(`toIndex must be 0–${m.timeline.length - 1}`);
+ }
+ if (to === from) throw new Error(`${id} is already at ${to}`);
+ const [e] = m.timeline.splice(from, 1);
+ m.timeline.splice(to, 0, e);
+ return { id, from, to };
+ });
+}
+
+/**
+ * Remove one entry. Refused when something still points at it -- a post
+ * attached to a clip, a claim -- in the build's own words.
+ *
+ * @param {string} dir
+ * @param {string} id
+ * @param {{ token?: string | null, at?: number | null }} [opts]
+ */
+export async function removeEntry(dir, id, { token = null, at = null } = {}) {
+ return structural(dir, "remove", token, (m) => {
+ const i = entryIndex(m.timeline, id, at);
+ const [removed] = m.timeline.splice(i, 1);
+ return { id, at: i, removed };
+ });
+}
+
+/** Copy one entry to just after itself, under a fresh id. A copied claim is dropped (one claim, one entry). *
+ * @param {string} dir
+ * @param {string} id
+ * @param {{ token?: string | null, at?: number | null }} [opts]
+ */
+export async function duplicateEntry(dir, id, { token = null, at = null } = {}) {
+ return structural(dir, "duplicate", token, (m) => {
+ const i = entryIndex(m.timeline, id, at);
+ const copy = JSON.parse(JSON.stringify(m.timeline[i]));
+ copy.id = freshEntryId(m.timeline, `${id}-copy`);
+ delete copy.claim;
+ m.timeline.splice(i + 1, 0, copy);
+ return { id: copy.id, at: i + 1, entry: copy };
+ });
+}
+
+/** `<channel>/<video>@<start>-<end>`, in seconds. */
+const CLIP_SPEC_RE = /^([A-Za-z0-9._-]+)\/([^@\s/]+)@(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)$/;
+
+/**
+ * What `insertEntry` will put in the timeline, checked: a clip spec string
+ * (`<channel>/<video>@<start>-<end>`) becomes a bare clip; an object is an
+ * entry as written, given a fresh id when it has none (or one already taken).
+ *
+ * @param {Array<Record<string, unknown>>} timeline
+ * @param {unknown} spec
+ */
+export function entryFromSpec(timeline, spec) {
+ if (typeof spec === "string") {
+ const mm = spec.trim().match(CLIP_SPEC_RE);
+ if (!mm) throw new Error("a clip is <channel>/<video>@<start>-<end> in seconds");
+ const [, channel, video, s, e] = mm;
+ return clipEntry(timeline, { type: "clip", channel, video, start: Number(s), end: Number(e) });
+ }
+ if (!spec || typeof spec !== "object" || Array.isArray(spec)) throw new Error("an entry is an object, or a clip spec string");
+ const entry = JSON.parse(JSON.stringify(spec));
+ if (JSON.stringify(entry).length > 20000) throw new Error("that entry is too large");
+ if (typeof entry.type !== "string" || !entry.type) throw new Error("an entry needs a type");
+ if (entry.type === "clip") return clipEntry(timeline, entry);
+ const base = typeof entry.id === "string" && ENTRY_ID_RE.test(entry.id) ? entry.id : entry.type;
+ entry.id = freshEntryId(timeline, base);
+ return entry;
+}
+
+function clipEntry(timeline, e) {
+ if (typeof e.video !== "string" || !e.video.trim()) throw new Error("a clip needs its video id");
+ for (const k of ["start", "end"]) {
+ const v = Number(e[k]);
+ if (!Number.isFinite(v) || v < 0) throw new Error(`a clip's ${k} must be a number ≥ 0`);
+ e[k] = round2(v);
+ }
+ if (e.end - e.start < 0.5) throw new Error(`a clip must be at least half a second (${e.start}–${e.end})`);
+ if (e.channel !== undefined && (typeof e.channel !== "string" || !e.channel)) delete e.channel;
+ const base = typeof e.id === "string" && ENTRY_ID_RE.test(e.id) ? e.id : `${e.video}-${Math.floor(e.start)}`;
+ // `type` and `id` first: the manifests are read by humans.
+ const { type: _t, id: _i, ...rest } = e;
+ return { type: "clip", id: freshEntryId(timeline, base), ...rest };
+}
+
+/**
+ * Insert an entry after `afterId` (null or "": at the start). `spec` is a clip
+ * spec string or an entry object (entryFromSpec).
+ *
+ * @param {string} dir
+ * @param {string | null} afterId
+ * @param {unknown} spec
+ * @param {{ token?: string | null, at?: number | null }} [opts] `at` is afterId's index
+ */
+export async function insertEntry(dir, afterId, spec, { token = null, at = null } = {}) {
+ return structural(dir, "insert", token, (m) => {
+ const i = afterId ? entryIndex(m.timeline, afterId, at) + 1 : 0;
+ const entry = entryFromSpec(m.timeline, spec);
+ m.timeline.splice(i, 0, entry);
+ return { id: entry.id, at: i, entry };
+ });
+}
+
+const TEASER_PATCH_KEYS = ["lines", "beat", "dip", "tail", "tailWait"];
+
+/**
+ * Patch one teaser: its lines, beat, dip, tail and tail wait. A key given as
+ * null (or "") is removed; a key not given is kept. Checked with the build's
+ * validateTeaser (which checks the dip too) against the patched entry.
+ *
+ * @param {string} dir
+ * @param {string} id
+ * @param {Record<string, unknown>} patch
+ * @param {{ token?: string | null, at?: number | null }} [opts]
+ */
+export async function updateTeaser(dir, id, patch, { token = null, at = null } = {}) {
+ if (!patch || typeof patch !== "object" || Array.isArray(patch)) throw new Error("a teaser patch is an object");
+ const keys = Object.keys(patch);
+ const bad = keys.filter((k) => !TEASER_PATCH_KEYS.includes(k));
+ if (bad.length) throw new Error(`${bad.join(", ")}: not something this writer changes (${TEASER_PATCH_KEYS.join(", ")})`);
+ if (!keys.length) throw new Error("nothing to change");
+ return structural(dir, "teaser", token, (m) => {
+ const e = m.timeline[entryIndex(m.timeline, id, at)];
+ if (e.type !== "teaser") throw new Error(`${id} is a ${e.type ?? "non-teaser"} entry, not a teaser`);
+ for (const k of keys) {
+ const v = patch[k];
+ if (v === null || v === "" || v === undefined) delete e[k];
+ else if (k === "beat" || k === "tailWait") e[k] = round2(Number(v));
+ else if (k === "dip") {
+ e.dip = typeof v === "object" && v ? { fade: round2(Number(v.fade)), black: round2(Number(v.black)) } : v;
+ } else if (k === "tail") e.tail = String(v);
+ else e[k] = v;
+ }
+ const errors = validateTeaser(e);
+ if (errors.length) throw new StructureRefused(errors);
+ return { id, entry: e };
+ });
+}
+
+const POST_TEXT_KEYS = ["platform", "author", "handle", "date", "text", "url", "shot", "flag", "accent", "logo", "siteChannel", "siteUrl", "postId", "variant"];
+
+/**
+ * Add a post, or replace the one with its id. The value is the post as it
+ * should be stored: an empty optional string is dropped rather than written.
+ * `attachTo` and `hide` are kept from the stored post unless the value names
+ * them. Checked with the build's validatePosts.
+ *
+ * @param {string} dir
+ * @param {Record<string, any>} post
+ * @param {{ token?: string | null }} [opts]
+ */
+export async function upsertPost(dir, post, { token = null } = {}) {
+ if (!post || typeof post !== "object" || Array.isArray(post)) throw new Error("a post is an object");
+ if (typeof post.id !== "string" || !ENTRY_ID_RE.test(post.id)) throw new Error("a post needs an id: letters, digits, dashes, underscores");
+ return structural(dir, "post", token, (m) => {
+ if (!Array.isArray(m.posts)) m.posts = [];
+ const i = m.posts.findIndex((p) => p?.id === post.id);
+ const prev = i >= 0 ? m.posts[i] : {};
+ const next = { id: post.id };
+ for (const k of POST_TEXT_KEYS) {
+ const v = k in post ? post[k] : prev[k];
+ if (v === undefined || v === null || (typeof v === "string" && !v.trim())) continue;
+ next[k] = typeof v === "string" ? v.trim() : v;
+ }
+ for (const k of ["attachTo", "hide"]) {
+ const v = k in post ? post[k] : prev[k];
+ if (v === undefined || v === null || v === "" || v === false) continue;
+ next[k] = v;
+ }
+ if (i >= 0) m.posts[i] = next;
+ else m.posts.push(next);
+ return { post: next, created: i < 0 };
+ });
+}
+
+/** Remove one post by id. *
+ * @param {string} dir
+ * @param {string} id
+ * @param {{ token?: string | null }} [opts]
+ */
+export async function removePost(dir, id, { token = null } = {}) {
+ return structural(dir, "post", token, (m) => {
+ const i = (m.posts ?? []).findIndex((p) => p?.id === id);
+ if (i < 0) throw new Error(`no post with id ${id}`);
+ const [removed] = m.posts.splice(i, 1);
+ if (!m.posts.length) delete m.posts;
+ return { removed };
+ });
+}
+
+/**
+ * Set or remove `render.chrome.factcheck`: the stamp, the tally, and each
+ * verdict's label and colour. Only on a manifest whose deck is on (the
+ * fact-check is drawn by the deck); null removes it. Checked by the build's
+ * validateChrome against the rest of the render block.
+ *
+ * @param {string} dir
+ * @param {Record<string, unknown> | null} factcheck
+ * @param {{ token?: string | null }} [opts]
+ */
+export async function updateFactcheck(dir, factcheck, { token = null } = {}) {
+ if (factcheck === undefined) throw new Error("factcheck must be an object, or null to remove it");
+ return structural(dir, "factcheck", token, (m) => {
+ const render = m.render ?? {};
+ if (!render.chrome || typeof render.chrome !== "object") {
+ throw new Error("the fact-check is drawn by the on-screen deck, and this manifest has none — turn the deck on first");
+ }
+ const chrome = { ...render.chrome };
+ if (factcheck === null) delete chrome.factcheck;
+ else chrome.factcheck = factcheck;
+ const { chrome: _old, ...rest } = render;
+ const already = new Set(validateChrome(render.chrome, rest));
+ const errors = validateChrome(chrome, rest).filter((e) => !already.has(e));
+ if (errors.length) throw new StructureRefused(errors);
+ m.render = { ...render, chrome };
+ return { factcheck: chrome.factcheck ?? null };
+ });
+}
+
+/**
+ * Take back the newest structural edit: restore the newest `auto-before-<op>`
+ * snapshot, byte for byte. The manifest as it is now is kept first
+ * (`undo-saved`), and the restored snapshot is renamed `undone-<op>` so the
+ * next undo goes one step further back rather than round in a circle.
+ *
+ * @param {string} dir
+ * @param {{ token?: string | null }} [opts]
+ */
+export async function undoStructural(dir, { token = null } = {}) {
+ return withManifestLock(async () => {
+ const current = await manifestToken(dir);
+ if (token !== null && current !== token) throw new StaleToken(token, current);
+ const target = (await listSnapshots(dir)).find((s) => !s.legacy && s.label?.startsWith(AUTO_SNAPSHOT_PREFIX));
+ if (!target) throw new Error("nothing to undo: no automatic snapshot in revisions/");
+ const op = target.label.slice(AUTO_SNAPSHOT_PREFIX.length);
+ const text = await readFile(path.join(dir, target.rel), "utf8");
+ JSON.parse(text); // a snapshot that does not parse is not restored over a manifest that does
+ let saved = null;
+ try {
+ saved = (await createSnapshot(dir, { label: "undo-saved" })).rel;
+ } catch {
+ // within the same second as another snapshot: the state is already kept
+ }
+ const file = manifestFile(dir);
+ const tmp = `${file}.tmp-${process.pid}-${Math.random().toString(36).slice(2, 8)}`;
+ await writeFile(tmp, text, "utf8");
+ await rename(tmp, file);
+ const undone = target.rel.replace(`-${target.label}.manifest.json`, `-undone-${op}.manifest.json`);
+ await rename(path.join(dir, target.rel), path.join(dir, undone)).catch(() => {});
+ return { restored: target.rel, op, saved, token: await manifestToken(dir) };
+ });
+}
diff --git a/umtool/lib/report/moments.mjs b/umtool/lib/report/moments.mjs
@@ -0,0 +1,155 @@
+// A moment on a rendered cut -> the entry playing there.
+//
+// A timed note is a second on a FILE (`out/sourced/<slug>.mp4`, a take's
+// `takes/<id>/preview.mp4`). What makes it worth an agent's time is what that
+// second resolves to: the entry on screen, its onscreen title and quote, and
+// the source second with a link into the archive. That join is the build's
+// `schedule.json` (build-video.mjs writeChromeSchedule: `out/<variant>/`, or a
+// take's own `takes/<id>/out/<variant>/`), which says where each entry starts
+// in the cut.
+//
+// THE SCHEDULE MATCHES THE BUILD'S OUTPUT, and a take's preview.mp4 is that
+// output copied: measured on every take of candace/polemic-israel, the
+// preview's duration equals the schedule's `total` to the millisecond. So a
+// mark is exact when the file it was made on is as long as the schedule says,
+// and APPROXIMATE (`approx: true`) when it is not -- a different preset, a
+// trimmed preview -- or when the schedule is an estimate, or the second falls
+// in a held frame past the clip's own source.
+//
+// The resolution is written INTO the note at write time (`anchor.resolved`):
+// a later rebuild moves entries around, and the agent reading the note must
+// see what was on screen when the operator pressed the key.
+import { readdir, readFile, stat } from "node:fs/promises";
+import path from "node:path";
+import { DEFAULT_VARIANT } from "umtool-report-to-video/build-video";
+import { channelFor } from "../projects/report.mjs";
+
+const TAKE_RE = /^[a-z0-9][a-z0-9-]{0,63}$/;
+const APPROX_TOLERANCE = 0.25;
+
+/**
+ * Where a moment's file sits: in a take (`takes/<id>/…`) and/or a variant's
+ * output dir (`…out/<variant>/…`). Pure.
+ *
+ * @param {string} rel project-relative, `/`-separated
+ */
+export function momentFileInfo(rel) {
+ const parts = String(rel ?? "").split("/");
+ let take = null;
+ let i = 0;
+ if (parts[0] === "takes" && TAKE_RE.test(parts[1] ?? "")) {
+ take = parts[1];
+ i = 2;
+ }
+ const variant = parts[i] === "out" && parts.length > i + 2 && TAKE_RE.test(parts[i + 1]) ? parts[i + 1] : null;
+ return { take, variant };
+}
+
+async function readJson(file) {
+ try {
+ return JSON.parse(await readFile(/* turbopackIgnore: true */ file, "utf8"));
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * The schedule and manifest a moment on `rel` resolves against, or null when
+ * there is no schedule. A take's own manifest wins over the project's.
+ *
+ * @param {string} projectDir
+ * @param {string} rel
+ */
+export async function scheduleForFile(projectDir, rel) {
+ const { take, variant } = momentFileInfo(rel);
+ const base = take ? path.join(/* turbopackIgnore: true */ projectDir, "takes", take) : projectDir;
+ let variants = [];
+ if (variant) variants = [variant];
+ else {
+ const dirs = await readdir(/* turbopackIgnore: true */ path.join(/* turbopackIgnore: true */ base, "out"), { withFileTypes: true }).catch(() => []);
+ variants = dirs.filter((d) => d.isDirectory() || d.isSymbolicLink()).map((d) => d.name);
+ // A deliverable names its cut (`<slug>-full.mp4`; the default cut is
+ // `<slug>.mp4`): that variant's schedule first, then the default's.
+ const named = (v) => path.basename(String(rel)).endsWith(`-${v}.mp4`);
+ const rank = (v) => (named(v) ? 0 : v === DEFAULT_VARIANT ? 1 : 2);
+ variants.sort((a, b) => rank(a) - rank(b) || a.localeCompare(b));
+ }
+ for (const v of variants) {
+ const file = path.join(/* turbopackIgnore: true */ base, "out", v, "schedule.json");
+ const schedule = await readJson(file);
+ if (!schedule || !Array.isArray(schedule.segments)) continue;
+ const manifest =
+ (take ? await readJson(path.join(/* turbopackIgnore: true */ base, "video.manifest.json")) : null) ??
+ (await readJson(path.join(/* turbopackIgnore: true */ projectDir, "video.manifest.json")));
+ const st = await stat(/* turbopackIgnore: true */ file).catch(() => null);
+ return {
+ schedule,
+ manifest,
+ variant: v,
+ take,
+ scheduleRel: path.relative(/* turbopackIgnore: true */ projectDir, file).split(path.sep).join("/"),
+ scheduleMtimeMs: st ? Math.round(st.mtimeMs) : null,
+ };
+ }
+ return null;
+}
+
+/**
+ * The entry playing at `t` seconds of a cut. Pure.
+ *
+ * During a crossfade both segments are on screen; the incoming one is taken
+ * from half-way through it. `duration` is the file's own (the player knows it):
+ * when it differs from the schedule's total the result is `approx`.
+ *
+ * @param {{ schedule: any, manifest: any, variant?: string | null, t: number, duration?: number | null }} args
+ * @returns {{ entry: string | null, title?: string, quote?: string, channel?: string, video?: string, sourceT?: number, url?: string, approx?: boolean }}
+ */
+export function resolveMoment({ schedule, manifest, variant = null, t, duration = null }) {
+ const segs = Array.isArray(schedule?.segments) ? schedule.segments : [];
+ if (!segs.length || !Number.isFinite(t)) return { entry: null };
+ const D = Number(schedule.transition) || 0;
+ let i = 0;
+ for (let k = 0; k < segs.length; k += 1) if (Number(segs[k].start) <= t - D / 2) i = k;
+ const seg = segs[i];
+ const out = { entry: String(seg.id) };
+ let approx = schedule.estimated === true;
+ if (Number.isFinite(duration) && Number.isFinite(Number(schedule.total)) && Math.abs(duration - Number(schedule.total)) > APPROX_TOLERANCE) {
+ approx = true;
+ }
+ const e = (manifest?.timeline ?? []).find((x) => x?.id === seg.id && (!x.variant || !variant || x.variant === variant)) ?? null;
+ const title = seg.title ?? e?.onscreen?.title ?? e?.title ?? e?.heading ?? null;
+ if (title) out.title = String(title);
+ if (e?.quote) out.quote = String(e.quote);
+ if (e?.type === "clip") {
+ const from = Number(e.cutStart ?? e.start);
+ const to = Number(e.cutEnd ?? e.end);
+ const into = Math.max(0, t - Number(seg.start));
+ if (Number.isFinite(from) && Number.isFinite(to)) {
+ if (from + into > to + 0.05) approx = true; // a held frame past the clip's own source
+ out.sourceT = Number(Math.min(to, from + into).toFixed(2));
+ const channel = channelFor(manifest, e);
+ if (channel) out.channel = channel;
+ out.video = String(e.video);
+ const origin = manifest?.provenance?.siteOrigin;
+ if (origin && channel) {
+ out.url = `${origin}/?v=${encodeURIComponent(`${channel}/${e.video}`)}&t=${Math.floor(out.sourceT)}`;
+ }
+ }
+ }
+ if (approx) out.approx = true;
+ return out;
+}
+
+/**
+ * What a moment on `rel` at `t` resolves to, or `{ entry: null }` with no schedule.
+ *
+ * @param {string} projectDir
+ * @param {string} rel
+ * @param {number} t
+ * @param {number | null} [duration]
+ */
+export async function resolveMomentOnFile(projectDir, rel, t, duration = null) {
+ const s = await scheduleForFile(projectDir, rel);
+ if (!s) return { entry: null, schedule: null };
+ return { ...resolveMoment({ ...s, t, duration }), schedule: s.scheduleRel };
+}
diff --git a/umtool/lib/report/moments.test.mjs b/umtool/lib/report/moments.test.mjs
@@ -0,0 +1,81 @@
+// A second on a rendered cut -> the entry, title, quote and source second.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { momentFileInfo, resolveMoment, resolveMomentOnFile } from "./moments.mjs";
+
+const manifest = {
+ provenance: { siteOrigin: "https://arch.test", channelSlug: "chan" },
+ timeline: [
+ { type: "teaser", id: "tz", lines: ["X"] },
+ { type: "clip", id: "c1", video: "v1", start: 100, end: 110, cutStart: 102, cutEnd: 108, quote: "the words", onscreen: { title: "Own title" } },
+ { type: "clip", id: "c2", video: "v2", channel: "other", start: 50, end: 60 },
+ ],
+};
+const schedule = {
+ transition: 0.5,
+ total: 20.5,
+ segments: [
+ { id: "tz", start: 0, end: 4.5, title: "Teaser" },
+ { id: "c1", start: 4, end: 10.5 },
+ { id: "c2", start: 10, end: 20.5, title: "Schedule title" },
+ ],
+};
+
+test("momentFileInfo reads the take and the variant from the path", () => {
+ assert.deepEqual(momentFileInfo("takes/deck/preview.mp4"), { take: "deck", variant: null });
+ assert.deepEqual(momentFileInfo("takes/deck/out/full/x.mp4"), { take: "deck", variant: "full" });
+ assert.deepEqual(momentFileInfo("out/sourced/slug.mp4"), { take: null, variant: "sourced" });
+ assert.deepEqual(momentFileInfo("video.mp4"), { take: null, variant: null });
+});
+
+test("resolveMoment: the entry on screen, the incoming one past half a crossfade", () => {
+ assert.equal(resolveMoment({ schedule, manifest, t: 1 }).entry, "tz");
+ assert.equal(resolveMoment({ schedule, manifest, t: 4.1 }).entry, "tz", "still the outgoing one");
+ const c1 = resolveMoment({ schedule, manifest, t: 6, duration: 20.5 });
+ assert.deepEqual(c1, {
+ entry: "c1",
+ title: "Own title",
+ quote: "the words",
+ sourceT: 104,
+ channel: "chan",
+ video: "v1",
+ url: "https://arch.test/?v=chan%2Fv1&t=104",
+ });
+ const c2 = resolveMoment({ schedule, manifest, t: 12 });
+ assert.equal(c2.title, "Schedule title");
+ assert.equal(c2.channel, "other");
+ assert.equal(c2.sourceT, 52);
+});
+
+test("approx: a file of another length, an estimated schedule, a held frame", () => {
+ assert.equal(resolveMoment({ schedule, manifest, t: 6, duration: 30 }).approx, true);
+ assert.equal(resolveMoment({ schedule: { ...schedule, estimated: true }, manifest, t: 6 }).approx, true);
+ // c1's cut is 6 s; 7.5 s in is a held frame
+ const held = resolveMoment({ schedule, manifest, t: 4 + 7.5 - 0.01 + 0, duration: 20.5 });
+ assert.equal(held.entry, "c2");
+ const c1held = resolveMoment({ schedule: { ...schedule, segments: [schedule.segments[0], { id: "c1", start: 4 }] }, manifest, t: 12, duration: 20.5 });
+ assert.equal(c1held.sourceT, 108);
+ assert.equal(c1held.approx, true);
+});
+
+test("resolveMomentOnFile finds a take's schedule, preferring the default variant", async () => {
+ const dir = await mkdtemp(path.join(tmpdir(), "umtool-moments-"));
+ try {
+ await writeFile(path.join(dir, "video.manifest.json"), JSON.stringify(manifest));
+ await mkdir(path.join(dir, "takes", "t1", "out", "full"), { recursive: true });
+ await mkdir(path.join(dir, "takes", "t1", "out", "sourced"), { recursive: true });
+ await writeFile(path.join(dir, "takes", "t1", "out", "sourced", "schedule.json"), JSON.stringify(schedule));
+ await writeFile(path.join(dir, "takes", "t1", "out", "full", "schedule.json"), JSON.stringify({ ...schedule, segments: [{ id: "c2", start: 0 }] }));
+ const r = await resolveMomentOnFile(dir, "takes/t1/preview.mp4", 6, 20.5);
+ assert.equal(r.entry, "c1");
+ assert.equal(r.schedule, "takes/t1/out/sourced/schedule.json");
+ assert.deepEqual(await resolveMomentOnFile(dir, "takes/none/preview.mp4", 6), { entry: null, schedule: null });
+ } finally {
+ await rm(dir, { recursive: true });
+ }
+});
diff --git a/umtool/lib/report/sections.mjs b/umtool/lib/report/sections.mjs
@@ -0,0 +1,64 @@
+// `sectionEnter`: the flag on the first clip of each section, which is what
+// makes the legacy footer's marker slide from one node to the next
+// (report-to-video/build-video.mjs reads it; README "The marker slides").
+//
+// report-to-video only ever READS it; the generators author it. This writes
+// the rule down for umtool's one writer that re-orders a timeline
+// (lib/report/manifest.mjs moveEntry), derived from what the builder reads and
+// checked against every real manifest: in the one that carries the flag
+// (quartering-employee-count, seven of nineteen clips) a clip carries it
+// exactly when its `section` differs from the previous CLIP's -- the first
+// clip counts, a card between two clips does not break a section, and a clip
+// with no `section` never enters one. Re-applying it to every real manifest
+// under ~/reports changes nothing (lib/report/sections.test.mjs has the shape).
+//
+// `section` itself is the author's: it says which chapter an entry belongs
+// to, and a move does not change that. Only the flag follows the order.
+//
+// A manifest that carries no `sectionEnter` key at all (every deck-era cut:
+// the deck draws no footer) is left exactly as it is: `applySectionEnter`
+// changes nothing unless some entry already carries the key.
+
+/**
+ * The flag each entry should carry, by index: true on a clip whose `section`
+ * is set and differs from the previous clip's; false everywhere else.
+ *
+ * @param {Array<Record<string, unknown>>} timeline
+ * @returns {boolean[]}
+ */
+export function sectionEnterFlags(timeline) {
+ let prev;
+ return (timeline ?? []).map((e) => {
+ if (e?.type !== "clip") return false;
+ const s = e.section;
+ const enters = s !== undefined && s !== null && s !== prev;
+ prev = s;
+ return enters;
+ });
+}
+
+/** Does this timeline use the flag at all? */
+export const usesSectionEnter = (timeline) => (timeline ?? []).some((e) => e && Object.hasOwn(e, "sectionEnter"));
+
+/**
+ * Recompute `sectionEnter` in place after a re-order. Only on a timeline that
+ * already uses it; written as `true` or removed (an absent key reads as false,
+ * and `"sectionEnter": false` is noise a human reads as a decision). Returns
+ * the ids whose flag changed.
+ *
+ * @param {Array<Record<string, unknown>>} timeline
+ * @returns {string[]}
+ */
+export function applySectionEnter(timeline) {
+ if (!usesSectionEnter(timeline)) return [];
+ const flags = sectionEnterFlags(timeline);
+ const changed = [];
+ timeline.forEach((e, i) => {
+ const was = e.sectionEnter === true;
+ if (flags[i] === was) return;
+ if (flags[i]) e.sectionEnter = true;
+ else delete e.sectionEnter;
+ changed.push(String(e.id));
+ });
+ return changed;
+}
diff --git a/umtool/lib/report/sections.test.mjs b/umtool/lib/report/sections.test.mjs
@@ -0,0 +1,42 @@
+// The sectionEnter rule, against the shape of the one manifest that uses it.
+import assert from "node:assert/strict";
+import test from "node:test";
+import { applySectionEnter, sectionEnterFlags, usesSectionEnter } from "./sections.mjs";
+
+const clip = (id, section, extra = {}) => ({ type: "clip", id, section, ...extra });
+
+test("a clip enters a section when its section differs from the previous clip's", () => {
+ const tl = [
+ { type: "card", id: "t0" },
+ clip("a", 1),
+ clip("b", 1),
+ { type: "card", id: "mid" },
+ clip("c", 1),
+ clip("d", 2),
+ clip("e"),
+ clip("f", 2),
+ { type: "scroll", id: "s" },
+ ];
+ assert.deepEqual(sectionEnterFlags(tl), [false, true, false, false, false, true, false, true, false]);
+});
+
+test("applySectionEnter leaves a timeline that never used the flag alone", () => {
+ const tl = [clip("a", 1), clip("b", 2)];
+ assert.deepEqual(applySectionEnter(tl), []);
+ assert.equal(usesSectionEnter(tl), false);
+ assert.equal("sectionEnter" in tl[0], false);
+});
+
+test("applySectionEnter is the identity on a timeline already in order, and fixes a move", () => {
+ const tl = [clip("a", 1, { sectionEnter: true }), clip("b", 1), clip("c", 2, { sectionEnter: true }), clip("d", 2)];
+ assert.deepEqual(applySectionEnter(tl), []);
+ // d moved to the front: d enters 2, a enters 1, c no longer enters (b was 1, c is 2 -> still enters)
+ const moved = [tl[3], tl[0], tl[1], tl[2]];
+ assert.deepEqual(applySectionEnter(moved).sort(), ["d"]);
+ assert.equal(moved[0].sectionEnter, true);
+ assert.equal(moved[3].sectionEnter, true);
+ // and moving it back removes the flag again (deleted, never written false)
+ const back = [moved[1], moved[2], moved[3], moved[0]];
+ assert.deepEqual(applySectionEnter(back), ["d"]);
+ assert.equal("sectionEnter" in back[3], false);
+});
diff --git a/umtool/lib/report/structure.test.mjs b/umtool/lib/report/structure.test.mjs
@@ -0,0 +1,235 @@
+// The structural writers: move, remove, duplicate, insert, the teaser, the
+// posts, the fact-check, and undo. Each goes through the token, the lock, the
+// build's own checks and an automatic snapshot, and each refusal leaves the
+// file byte-for-byte as it was.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import { mkdtemp, readFile, readdir, rm, utimes, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import {
+ MANIFEST_NAME,
+ StaleToken,
+ StructureRefused,
+ duplicateEntry,
+ entryFromSpec,
+ insertEntry,
+ manifestToken,
+ moveEntry,
+ removeEntry,
+ removePost,
+ undoStructural,
+ updateFactcheck,
+ updateTeaser,
+ upsertPost,
+} from "./manifest.mjs";
+
+const base = () => ({
+ slug: "t",
+ provenance: { siteOrigin: "https://example.test", channelSlug: "chan" },
+ render: { width: 1920, height: 1080, fps: 30, transition: 0.5 },
+ timeline: [
+ { type: "teaser", id: "tz", lines: ["THE PROMISE"] },
+ { type: "clip", id: "c01", video: "v1", start: 10, end: 20, section: 1, sectionEnter: true },
+ { type: "clip", id: "c02", video: "v2", start: 30, end: 41, section: 1 },
+ { type: "clip", id: "c03", video: "v3", start: 50, end: 55, section: 2, sectionEnter: true },
+ ],
+ posts: [
+ { id: "p1", platform: "x", date: "2024-01-02", text: "hello", url: "https://x.com/a/status/1", attachTo: "c02" },
+ ],
+});
+
+async function project(manifest = base()) {
+ const dir = await mkdtemp(path.join(tmpdir(), "umtool-structure-"));
+ await writeFile(path.join(dir, MANIFEST_NAME), JSON.stringify(manifest, null, 2) + "\n");
+ return dir;
+}
+const readRaw = (dir) => readFile(path.join(dir, MANIFEST_NAME), "utf8");
+const read = async (dir) => JSON.parse(await readRaw(dir));
+const ids = (m) => m.timeline.map((e) => e.id);
+const revisions = async (dir) => (await readdir(path.join(dir, "revisions")).catch(() => [])).sort();
+
+test("moveEntry: re-orders, recomputes sectionEnter, snapshots once per burst", async () => {
+ const dir = await project();
+ try {
+ const r = await moveEntry(dir, "c03", 1, { token: await manifestToken(dir) });
+ assert.deepEqual([r.from, r.to], [3, 1]);
+ const m = await read(dir);
+ assert.deepEqual(ids(m), ["tz", "c03", "c01", "c02"]);
+ // c03 (section 2) now first: it enters; c01 (section 1, after a 2) enters; c02 does not
+ assert.equal(m.timeline[1].sectionEnter, true);
+ assert.equal(m.timeline[2].sectionEnter, true);
+ assert.equal("sectionEnter" in m.timeline[3], false);
+ // The file is still the CLI's formatting.
+ assert.equal(await readRaw(dir), JSON.stringify(m, null, 2) + "\n");
+ assert.equal((await revisions(dir)).filter((n) => n.includes("auto-before-move")).length, 1);
+ await moveEntry(dir, "c03", 3);
+ assert.equal((await revisions(dir)).filter((n) => n.includes("auto-before-move")).length, 1, "throttled");
+ await assert.rejects(moveEntry(dir, "c03", 3), /already at 3/);
+ await assert.rejects(moveEntry(dir, "c03", 9), /toIndex/);
+ await assert.rejects(moveEntry(dir, "nope", 0), /no timeline entry/);
+ } finally {
+ await rm(dir, { recursive: true });
+ }
+});
+
+test("a stale token refuses every structural write and leaves the file alone", async () => {
+ const dir = await project();
+ try {
+ const before = await readRaw(dir);
+ for (const call of [
+ () => moveEntry(dir, "c03", 1, { token: "1" }),
+ () => removeEntry(dir, "c03", { token: "1" }),
+ () => duplicateEntry(dir, "c03", { token: "1" }),
+ () => insertEntry(dir, "c03", "chan/vx@1-5", { token: "1" }),
+ () => updateTeaser(dir, "tz", { beat: 1 }, { token: "1" }),
+ () => upsertPost(dir, { id: "p2" }, { token: "1" }),
+ () => removePost(dir, "p1", { token: "1" }),
+ () => undoStructural(dir, { token: "1" }),
+ ]) {
+ await assert.rejects(call(), StaleToken);
+ }
+ assert.equal(await readRaw(dir), before);
+ } finally {
+ await rm(dir, { recursive: true });
+ }
+});
+
+test("removeEntry: refused while a post rides on the clip; fine once it does not", async () => {
+ const dir = await project();
+ try {
+ const before = await readRaw(dir);
+ await assert.rejects(removeEntry(dir, "c02"), (e) => e instanceof StructureRefused && /attachTo/.test(e.message));
+ assert.equal(await readRaw(dir), before);
+ const r = await removeEntry(dir, "c03");
+ assert.equal(r.removed.id, "c03");
+ assert.deepEqual(ids(await read(dir)), ["tz", "c01", "c02"]);
+ } finally {
+ await rm(dir, { recursive: true });
+ }
+});
+
+test("duplicateEntry and insertEntry: fresh ids, the clip spec, 2 dp, at the right place", async () => {
+ const dir = await project();
+ try {
+ const d = await duplicateEntry(dir, "c01");
+ assert.equal(d.id, "c01-copy");
+ const again = await duplicateEntry(dir, "c01");
+ assert.equal(again.id, "c01-copy-2");
+ const ins = await insertEntry(dir, "c02", "mychan/abc123@12.3456-20.1");
+ assert.deepEqual(ins.entry, { type: "clip", id: "abc123-12", channel: "mychan", video: "abc123", start: 12.35, end: 20.1 });
+ const first = await insertEntry(dir, null, { type: "card", heading: "Start" });
+ assert.equal(first.at, 0);
+ assert.equal(first.id, "card");
+ const m = await read(dir);
+ assert.deepEqual(ids(m), ["card", "tz", "c01", "c01-copy-2", "c01-copy", "c02", "abc123-12", "c03"]);
+ await assert.rejects(insertEntry(dir, "c02", "not a spec"), /channel/);
+ await assert.rejects(insertEntry(dir, "c02", "chan/v@5-5.2"), /half a second/);
+ // A teaser that the build would refuse is refused here.
+ await assert.rejects(insertEntry(dir, "c02", { type: "teaser", lines: [] }), StructureRefused);
+ } finally {
+ await rm(dir, { recursive: true });
+ }
+});
+
+test("entryFromSpec never reuses an id", () => {
+ const tl = [{ id: "a" }, { id: "v-1" }];
+ assert.equal(entryFromSpec(tl, { type: "card", id: "a" }).id, "a-2");
+ assert.equal(entryFromSpec(tl, "c/v@1-3").id, "v-1-2");
+});
+
+test("updateTeaser: lines, beat, tail, dip — validated by the build's own check", async () => {
+ const dir = await project();
+ try {
+ const r = await updateTeaser(dir, "tz", { lines: ["THE PROMISE", "AND WHAT HAPPENED"], beat: 1.2345, tail: "?", tailWait: 1 });
+ assert.equal(r.entry.beat, 1.23);
+ await assert.rejects(updateTeaser(dir, "tz", { lines: [] }), StructureRefused);
+ await assert.rejects(updateTeaser(dir, "tz", { tail: null }), /tailWait/);
+ await updateTeaser(dir, "tz", { tail: null, tailWait: null });
+ const m = await read(dir);
+ assert.equal("tail" in m.timeline[0], false);
+ await assert.rejects(updateTeaser(dir, "c01", { beat: 1 }), /not a teaser/);
+ await assert.rejects(updateTeaser(dir, "tz", { seconds: 4 }), /not something/);
+ } finally {
+ await rm(dir, { recursive: true });
+ }
+});
+
+test("upsertPost / removePost: add, replace keeping attachTo, refuse what validatePosts refuses", async () => {
+ const dir = await project();
+ try {
+ const add = await upsertPost(dir, { id: "p2", platform: "bluesky", date: "2024-03-04", text: " words ", url: "https://bsky.app/x", author: "" });
+ assert.equal(add.created, true);
+ assert.deepEqual(add.post, { id: "p2", platform: "bluesky", date: "2024-03-04", text: "words", url: "https://bsky.app/x" });
+ const rep = await upsertPost(dir, { id: "p1", platform: "x", date: "2024-01-02", text: "edited", url: "https://x.com/a/status/1" });
+ assert.equal(rep.post.attachTo, "c02");
+ await assert.rejects(upsertPost(dir, { id: "p3", platform: "myspace", date: "2024", text: "x", url: "http://no" }), StructureRefused);
+ await removePost(dir, "p2");
+ await removePost(dir, "p1");
+ assert.equal("posts" in (await read(dir)), false);
+ await assert.rejects(removePost(dir, "p1"), /no post/);
+ } finally {
+ await rm(dir, { recursive: true });
+ }
+});
+
+test("updateFactcheck: needs the deck; labels and colours checked by validateChrome", async () => {
+ const dir = await project();
+ try {
+ await assert.rejects(updateFactcheck(dir, { stamp: { seconds: 3 } }), /deck on first/);
+ const m = await read(dir);
+ m.render.chrome = { engine: "hyperframes", layout: "deck" };
+ await writeFile(path.join(dir, MANIFEST_NAME), JSON.stringify(m, null, 2) + "\n");
+ const r = await updateFactcheck(dir, { verdicts: { CONTRADICTED: { label: "NOPE", color: "#ff0000" } }, stamp: { seconds: 4 } });
+ assert.equal(r.factcheck.stamp.seconds, 4);
+ await assert.rejects(updateFactcheck(dir, { stamp: { seconds: 99 } }), StructureRefused);
+ await updateFactcheck(dir, null);
+ assert.equal("factcheck" in (await read(dir)).render.chrome, false);
+ } finally {
+ await rm(dir, { recursive: true });
+ }
+});
+
+test("undoStructural restores the newest automatic snapshot, keeps the current state, and walks back", async () => {
+ const dir = await project();
+ try {
+ const original = await readRaw(dir);
+ await assert.rejects(undoStructural(dir), /nothing to undo/);
+ await moveEntry(dir, "c03", 1);
+ // Age the move's snapshot so the next op is not throttled into it.
+ const rev = path.join(dir, "revisions");
+ for (const n of await readdir(rev)) await utimes(path.join(rev, n), new Date(Date.now() - 600_000), new Date(Date.now() - 600_000));
+ await new Promise((r) => setTimeout(r, 1100));
+ await removeEntry(dir, "c01");
+ const afterRemove = ids(await read(dir));
+ assert.deepEqual(afterRemove, ["tz", "c03", "c02"]);
+
+ const u1 = await undoStructural(dir);
+ assert.equal(u1.op, "remove");
+ assert.deepEqual(ids(await read(dir)), ["tz", "c03", "c01", "c02"]);
+ await new Promise((r) => setTimeout(r, 1100));
+ const u2 = await undoStructural(dir);
+ assert.equal(u2.op, "move");
+ assert.equal(await readRaw(dir), original, "byte for byte");
+ const names = await revisions(dir);
+ assert.ok(names.some((n) => n.includes("undone-move")));
+ assert.ok(names.some((n) => n.includes("undo-saved")));
+ } finally {
+ await rm(dir, { recursive: true });
+ }
+});
+
+test("a manifest that already has a build problem can still be re-ordered", async () => {
+ const m = base();
+ m.posts[0].attachTo = "gone"; // already broken before the edit
+ const dir = await project(m);
+ try {
+ await moveEntry(dir, "c03", 1);
+ assert.deepEqual(ids(await read(dir)), ["tz", "c03", "c01", "c02"]);
+ } finally {
+ await rm(dir, { recursive: true });
+ }
+});
diff --git a/umtool/lib/report/takes.mjs b/umtool/lib/report/takes.mjs
@@ -290,3 +290,33 @@ export async function setTakeVerdict(projectDir, takeId, patch) {
return { entry: map[takeId] ?? null, verdicts: map };
});
}
+
+/**
+ * Every take with its verdict, for an agent reading the conversation back
+ * (`umtool notes`, the notes digest): label, group, summary and changes from
+ * take.json, and the operator's verdict and note from verdicts.json. A verdict
+ * on a take that no longer exists is kept, with `missing: true`, rather than
+ * dropped -- the agent may have deleted the take it was about.
+ *
+ * @param {string} projectDir
+ * @returns {Promise<Array<{ id: string, group: string | null, label: string | null, summary: string, changes: string[], verdict: "like" | "maybe" | "no" | null, note: string, at: string, missing?: true }>>}
+ */
+export async function takesWithVerdicts(projectDir) {
+ const [listing, verdicts] = await Promise.all([listTakes(projectDir), readVerdicts(projectDir)]);
+ const out = listing.takes.map((t) => ({
+ id: t.id,
+ group: t.group,
+ label: t.label,
+ summary: t.summary,
+ changes: t.changes,
+ verdict: verdicts[t.id]?.verdict ?? null,
+ note: verdicts[t.id]?.note ?? "",
+ at: verdicts[t.id]?.at ?? "",
+ }));
+ const known = new Set(out.map((t) => t.id));
+ for (const [id, v] of Object.entries(verdicts)) {
+ if (known.has(id)) continue;
+ out.push({ id, group: null, label: null, summary: "", changes: [], verdict: v.verdict, note: v.note, at: v.at, missing: true });
+ }
+ return out;
+}
diff --git a/umtool/lib/report/takes.test.mjs b/umtool/lib/report/takes.test.mjs
@@ -237,3 +237,26 @@ test("concurrent verdicts on different takes are all kept", async () => {
await rm(dir, { recursive: true, force: true });
}
});
+
+test("takesWithVerdicts joins take.json and verdicts.json, keeping a verdict on a vanished take", async () => {
+ const { takesWithVerdicts } = await import("./takes.mjs");
+ const dir = await mkdtemp(path.join(tmpdir(), "umtool-takes-digest-"));
+ try {
+ await mkdir(path.join(dir, "takes", "a"), { recursive: true });
+ await writeFile(
+ path.join(dir, "takes", "a", "take.json"),
+ JSON.stringify({ id: "a", group: "g", order: 1, label: "A", kind: "reference", preview: "p.mp4", summary: "s" }),
+ );
+ await writeFile(
+ path.join(dir, "takes", "verdicts.json"),
+ JSON.stringify({ a: { verdict: "like", note: "yes", at: "x" }, gone: { verdict: "no", note: "", at: "y" } }),
+ );
+ const rows = await takesWithVerdicts(dir);
+ assert.deepEqual(rows.map((r) => [r.id, r.label, r.verdict, r.note, r.missing ?? false]), [
+ ["a", "A", "like", "yes", false],
+ ["gone", null, "no", "", true],
+ ]);
+ } finally {
+ await rm(dir, { recursive: true });
+ }
+});
diff --git a/umtool/playwright.config.ts b/umtool/playwright.config.ts
@@ -76,6 +76,10 @@ export default defineConfig({
// var. CHANNELS_DIR has to be said explicitly: it is where a report
// video's cue files live, and its default is the real 3 GB corpus.
`CHANNELS_DIR=${FIXTURE}/channels ` +
+ // The sites (/sites, article notes): the fixture's own, never the real
+ // transcripts/sites -- the one corpus file umtool writes is a report's
+ // notes.json, and the suite writes them.
+ `SITES_DIR=${FIXTURE}/sites ` +
// The cache (the project index, posters, analyses) is no longer under
// SONG_DIR (release 17): its default is the user's ~/.cache, which a
// suite must never write. The fixture's own, rebuilt with it every run.
