commit bda861cacd0e4c14dea093356315bc0ddcd9b814
parent 2d6b38b445f177b227864e22aa772452e406efb0
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 2 Oct 2026 10:21:23 -0400
report-to-video: post links after review -- one failure memo (url -> {err, at, refresh}) read by both resolver passes, expiring after refreshAfterMs (never in a build, ten minutes in umtool's server) and cleared by a success, so an archive that stalls costs one timeout per URL, not per post; resolvePostLinks skips hidden posts and cuts without the deck, takes a deadline (the preview's is 10 s), and its no-origin note says to set provenance.siteOrigin; null is unset for siteChannel/siteUrl/postId; createJsonCache shares an in-flight fetch; cues.mjs notes the shared, refreshed corpus.json; the preview links the posts as the draft leaves them
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
6 files changed, 238 insertions(+), 42 deletions(-)
diff --git a/umtool/lib/report/onscreen.mjs b/umtool/lib/report/onscreen.mjs
@@ -331,24 +331,41 @@ async function readBuiltSchedule(dir, variant) {
export async function scheduleForPreview(project, manifest, variant, draft = new Map(), postsDraft = {}, { resolver = previewPostResolver() } = {}) {
const selected = selectVariant(manifest, variant);
const entries = selected.timeline ?? [];
+ const posts = Array.isArray(selected.posts) ? selected.posts : [];
const [built, metas, linked] = await Promise.all([
readBuiltSchedule(project.dir, variant),
deckMetas(project.dir, manifest, entries),
- resolvePostLinks({ posts: selected.posts ?? [], provenance: selected.provenance ?? {}, render: selected.render ?? {}, resolver }),
+ // The posts as the draft leaves them, so a post the draft un-hides is
+ // linked too (a hidden one is not looked up).
+ resolvePostLinks({
+ posts: applyPostsDraft(posts, postsDraft),
+ provenance: selected.provenance ?? {},
+ render: selected.render ?? {},
+ resolver,
+ deadlineMs: PREVIEW_LINK_DEADLINE_MS,
+ }),
]);
- // The posts with the archive channel each is kept in, found as the build
- // finds it (post-links.mjs, the same cache on disk), so the preview's QRs
- // are the build's. Never written to the manifest.
- const variantManifest = linked.posts === selected.posts ? selected : { ...selected, posts: linked.posts };
+ // The archive channel each post is kept in, found as the build finds it
+ // (post-links.mjs, the same cache on disk), so the preview's QRs are the
+ // build's. Set on this request's copy of the posts only, never written.
+ const found = new Map(linked.posts.filter((p) => p.siteChannel).map((p) => [p.id, p.siteChannel]));
+ const variantManifest = found.size
+ ? { ...selected, posts: posts.map((p) => (!p.siteChannel && found.has(p.id) ? { ...p, siteChannel: found.get(p.id) } : p)) }
+ : selected;
return { variantManifest, metas, schedule: previewSchedule({ variantManifest, built, draft, metas, postsDraft }) };
}
+/** The most a preview request waits on the archive for its posts' links, all of them together. */
+const PREVIEW_LINK_DEADLINE_MS = 10000;
+
/**
* The preview's post-link resolver: one for the server's life, so its memory
- * cache spans requests. A post missing from the cached archive is looked for
- * in a fresh copy at most every ten minutes, and a request waits at most eight
- * seconds on an archive that does not answer (then the QR links the original,
- * as the build's would).
+ * cache spans requests. Each archive request may take eight seconds; a URL
+ * that failed is not asked again for ten minutes, and a post missing from the
+ * cached archive is looked for in a fresh copy at most that often. A request
+ * waits at most PREVIEW_LINK_DEADLINE_MS for all its posts; one not found by
+ * then links the original, as the build's would when the archive does not
+ * answer, and its lookup finishes in the background for the next request.
*/
let previewResolver = null;
function previewPostResolver() {
diff --git a/umtool/lib/report/onscreen.test.mjs b/umtool/lib/report/onscreen.test.mjs
@@ -412,3 +412,21 @@ test("scheduleForPreview: the posts are linked by the resolver the build uses, i
await rm(dir, { recursive: true, force: true });
}
});
+
+test("scheduleForPreview: a post the draft un-hides is linked; a hidden one is not asked about", async () => {
+ const dir = await mkdtemp(path.join(tmpdir(), "onscreen-links-"));
+ try {
+ const manifest = withPosts([{ ...POSTS[1], hide: true }]);
+ const asked = [];
+ const resolver = { find: async (origin, post) => (asked.push(post.id), "b-x") };
+ const hidden = await scheduleForPreview({ dir }, manifest, "sourced", new Map(), {}, { resolver });
+ assert.deepEqual(asked, []);
+ assert.ok(!("posts" in hidden.schedule));
+ const shown = await scheduleForPreview({ dir }, manifest, "sourced", new Map(), { p2: { hide: false } }, { resolver });
+ assert.deepEqual(asked, ["p2"]);
+ assert.equal(shown.schedule.posts[0].qrUrl, "https://example.test/?v=b-x%2F2&vm=post");
+ assert.equal(shown.variantManifest.posts[0].hide, true, "the draft is the preview's, not the returned manifest's");
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+});
diff --git a/umtool/report-to-video/cues.mjs b/umtool/report-to-video/cues.mjs
@@ -47,6 +47,14 @@
// reader following the citation will actually see, and the only
// option that is reproducible on a machine with no corpus.
// Whatever answers, the returned record carries `from` so a caller can record it.
+//
+// THE ON-DISK CACHE IS SHARED, AND ONE READER REFRESHES IT. post-links.mjs (a
+// post's archive channel, for its QR) reads `/corpus.json` and the posts
+// manifests through createJsonCache below, the same files on disk, and on a
+// post it cannot find it fetches them again and rewrites them. The cue walk
+// never refreshes, but it can read a corpus.json post-links rewrote: newer, so
+// a channel the stale copy lacked is found. Manifest and shard URLs carry no
+// version, so a cue window does not move because of it.
import { readFile, writeFile, mkdir, lstat, readlink } from "node:fs/promises";
import path from "node:path";
@@ -137,6 +145,8 @@ export function createJsonCache({
const mem = new Map();
/** URL -> when this cache last fetched it from the network (ms). */
const fetchedAt = new Map();
+ /** URL -> the network fetch of it in progress. */
+ const inflight = new Map();
async function getJson(url, { refresh = false } = {}) {
const recent = fetchedAt.has(url) && Date.now() - fetchedAt.get(url) < refreshAfterMs;
@@ -152,21 +162,32 @@ export function createJsonCache({
/* cold cache */
}
}
- log(`fetch ${url}`);
- const res = await fetchImpl(url);
- if (!res.ok) throw new Error(`GET ${url} -> ${res.status}`);
- const json = await res.json();
- mem.set(url, json);
- fetchedAt.set(url, Date.now());
- if (disk) {
- try {
- await mkdir(path.dirname(disk), { recursive: true });
- await writeFile(disk, JSON.stringify(json));
- } catch {
- // A cache we cannot write is a slow run, not a failed one.
+ // Callers that ask for the same URL while it is on its way share the one
+ // fetch (umtool's preview routes run side by side).
+ if (inflight.has(url)) return inflight.get(url);
+ const pending = (async () => {
+ log(`fetch ${url}`);
+ const res = await fetchImpl(url);
+ if (!res.ok) throw new Error(`GET ${url} -> ${res.status}`);
+ const json = await res.json();
+ mem.set(url, json);
+ fetchedAt.set(url, Date.now());
+ if (disk) {
+ try {
+ await mkdir(path.dirname(disk), { recursive: true });
+ await writeFile(disk, JSON.stringify(json));
+ } catch {
+ // A cache we cannot write is a slow run, not a failed one.
+ }
}
+ return json;
+ })();
+ inflight.set(url, pending);
+ try {
+ return await pending;
+ } finally {
+ inflight.delete(url);
}
- return json;
}
return { getJson };
diff --git a/umtool/report-to-video/deck.mjs b/umtool/report-to-video/deck.mjs
@@ -719,13 +719,14 @@ export function validatePosts(posts, timeline = [], render = null) {
// Where the archive keeps the post: its own channel's slug (a post channel
// is not the video channel -- `piratesoftware-bsky`, not `piratesoftware`),
// a page to link instead, or the post's id when its url does not carry one.
- if (p.siteChannel !== undefined && (typeof p.siteChannel !== "string" || !SLUG_RE.test(p.siteChannel))) {
+ // `null` is unset, as `attachTo: null` is.
+ if (p.siteChannel != null && (typeof p.siteChannel !== "string" || !SLUG_RE.test(p.siteChannel))) {
errors.push(`${w}.siteChannel must be the archive's channel slug (letters, digits, dots, dashes, underscores)`);
}
- if (p.siteUrl !== undefined && (typeof p.siteUrl !== "string" || !/^https?:\/\/\S+$/.test(p.siteUrl))) {
+ if (p.siteUrl != null && (typeof p.siteUrl !== "string" || !/^https?:\/\/\S+$/.test(p.siteUrl))) {
errors.push(`${w}.siteUrl must be an http(s) link`);
}
- if (p.postId !== undefined && (typeof p.postId !== "string" || !POST_ID_RE.test(p.postId))) {
+ if (p.postId != null && (typeof p.postId !== "string" || !POST_ID_RE.test(p.postId))) {
errors.push(`${w}.postId must be the post's id on its platform (letters, digits, dashes, underscores)`);
}
});
diff --git a/umtool/report-to-video/post-links.mjs b/umtool/report-to-video/post-links.mjs
@@ -23,9 +23,10 @@
// cue walk reads -- with one refresh: a post missing from a CACHED corpus or
// manifest is looked for again in a fresh copy (once per process for a build,
// once per `refreshAfterMs` for a server), so a post published since the cache
-// was filled is found.
+// was filled is found. A URL that failed is not asked again for as long (the
+// failure memo in createPostChannelResolver).
import { createJsonCache } from "./cues.mjs";
-import { postNativeId, resolveDeck } from "./deck.mjs";
+import { deckOn, postNativeId, resolveDeck } from "./deck.mjs";
/** How long one archive request may take before the post links its original. */
export const POST_LINK_TIMEOUT_MS = 15000;
@@ -61,20 +62,34 @@ export function createPostChannelResolver({
log = () => {},
timeoutMs = POST_LINK_TIMEOUT_MS,
refreshAfterMs = Infinity,
+ now = Date.now,
} = {}) {
const timed = (url) => fetchImpl(url, { signal: AbortSignal.timeout(timeoutMs) });
const get = getJson ?? createJsonCache({ ...(cacheDir !== undefined ? { cacheDir } : {}), fetchImpl: timed, log, refreshAfterMs }).getJson;
- // One failed read of an archive is remembered for the resolver's life, so a
- // cut of twenty posts against an archive that does not answer waits once.
+ // ONE memo of failed reads, `url -> { err, at, refresh }`, consulted by both
+ // passes (the cached read and the fresh one): a URL that failed is not asked
+ // again until `refreshAfterMs` has passed -- never again in a build
+ // (Infinity), after ten minutes in umtool's server -- and a read of it that
+ // succeeds clears it. So each URL of an archive that does not answer costs
+ // one timeout per build, not one per post.
+ //
+ // A cached read that failed went to the network (nothing was cached), so it
+ // bars both passes. A fresh read that failed bars only fresh reads: the copy
+ // already in hand still answers the cached pass, and a cached read that
+ // succeeds does not clear it -- it proves nothing about the network.
const failed = new Map();
async function read(url, refresh) {
- if (failed.has(url) && !refresh) throw failed.get(url);
+ const f = failed.get(url);
+ const live = f && now() - f.at < refreshAfterMs;
+ if (live && (refresh || !f.refresh)) throw f.err;
try {
- return await get(url, { refresh });
+ const doc = await get(url, { refresh });
+ if (f && (refresh || !f.refresh || !live)) failed.delete(url);
+ return doc;
} catch (e) {
const err = e instanceof Error ? e : new Error(String(e));
- failed.set(url, err);
+ failed.set(url, { err, at: now(), refresh });
throw err;
}
}
@@ -111,28 +126,47 @@ export function createPostChannelResolver({
/**
* The posts with their archive channel found, and a note per post saying
- * where its QR goes. Only under `posts.links: "archive"` with an archive named
- * (`provenance.siteOrigin`) and posts to draw; otherwise the posts come back
- * as they were, with no notes.
+ * where its QR goes. Only under the deck, with `posts.links: "archive"`, an
+ * archive named (`provenance.siteOrigin`) and posts to draw; otherwise the
+ * posts come back as they were, with no notes. A hidden post is not drawn, so
+ * it is not looked up either.
*
* Never throws for the archive: a post it cannot place keeps its own link.
+ * `deadlineMs` bounds the whole call (umtool's preview passes one): a post
+ * still unresolved when it passes keeps its own link, and its lookup goes on
+ * in the background, filling the cache for the next call.
*
* @param {{ posts?: Array<Record<string, any>>, provenance?: Record<string, any>,
- * render?: Record<string, any>, resolver: ReturnType<typeof createPostChannelResolver> }} args
+ * render?: Record<string, any>, resolver: ReturnType<typeof createPostChannelResolver>,
+ * deadlineMs?: number }} args
* @returns {Promise<{ posts: Array<Record<string, any>>, notes: string[] }>}
*/
-export async function resolvePostLinks({ posts = [], provenance = {}, render = {}, resolver }) {
+export async function resolvePostLinks({ posts = [], provenance = {}, render = {}, resolver, deadlineMs = Infinity }) {
const settings = resolveDeck(render).posts;
const base = origin(provenance);
- if (!Array.isArray(posts) || !posts.length || !settings.show || settings.links !== "archive") {
+ if (!deckOn(render) || !Array.isArray(posts) || !posts.length || !settings.show || settings.links !== "archive") {
return { posts, notes: [] };
}
if (!base) {
- return { posts, notes: ["posts: no provenance.siteOrigin names an archive -- every post's QR links the original"] };
+ return {
+ posts,
+ notes: ["posts: no provenance.siteOrigin names an archive -- every post's QR links the original (set provenance.siteOrigin to the site that keeps them)"],
+ };
}
const notes = [];
const out = [];
+ const until = Date.now() + deadlineMs;
+ const timeUp = Symbol("time up");
+ const inTime = (promise) => {
+ const left = until - Date.now();
+ if (!Number.isFinite(left)) return promise;
+ let timer;
+ const t = new Promise((res) => { timer = setTimeout(() => res(timeUp), left); });
+ promise.catch(() => {});
+ return Promise.race([promise, t]).finally(() => clearTimeout(timer));
+ };
for (const p of posts) {
+ if (p.hide) { out.push(p); continue; }
if (p.siteUrl) { notes.push(`${p.id}: QR links ${p.siteUrl} (siteUrl)`); out.push(p); continue; }
if (p.siteChannel) { notes.push(`${p.id}: archive link via ${p.siteChannel} (pinned)`); out.push(p); continue; }
if (!postNativeId(p)) {
@@ -140,9 +174,17 @@ export async function resolvePostLinks({ posts = [], provenance = {}, render = {
out.push(p);
continue;
}
+ if (Date.now() >= until) {
+ notes.push(`${p.id}: no time left to ask the archive at ${base} -- QR links the original`);
+ out.push(p);
+ continue;
+ }
try {
- const slug = await resolver.find(base, p);
- if (slug) {
+ const slug = await inTime(resolver.find(base, p));
+ if (slug === timeUp) {
+ notes.push(`${p.id}: the archive at ${base} was still being asked when time ran out -- QR links the original`);
+ out.push(p);
+ } else if (slug) {
notes.push(`${p.id}: archive link via ${slug}`);
out.push({ ...p, siteChannel: slug });
} else {
diff --git a/umtool/report-to-video/post-links.test.mjs b/umtool/report-to-video/post-links.test.mjs
@@ -199,7 +199,8 @@ test("resolvePostLinks: pinned posts, links \"original\", no siteOrigin, no id",
// No archive named: one note for the lot.
const none = await resolvePostLinks({ posts: [BSKY], provenance: {}, render: DECK, resolver });
assert.equal(none.posts[0], BSKY);
- assert.deepEqual(none.notes, ["posts: no provenance.siteOrigin names an archive -- every post's QR links the original"]);
+ assert.equal(none.notes.length, 1);
+ assert.match(none.notes[0], /^posts: no provenance\.siteOrigin names an archive -- every post's QR links the original \(set provenance\.siteOrigin/);
// No id in the url.
const noId = await resolvePostLinks({ posts: [{ ...BSKY, url: "https://bsky.app/p/1" }], provenance: PROV, render: DECK, resolver });
assert.deepEqual(noId.notes, [`${BSKY.id}: no post id in its url -- QR links the original`]);
@@ -228,3 +229,99 @@ test("createJsonCache: a refresh refetches once per process, then reads what it
await rm(dir, { recursive: true, force: true });
}
});
+
+// ---- the failure memo, the deadline, what is not looked up -------------------
+
+test("failure memo: an archive that hangs costs each URL one try per build, across both passes and every post", async () => {
+ // corpus.json is cached (the cached pass answers); every fresh read and the
+ // uncached posts manifest fail, as an archive that connects and stalls would.
+ const a = archive([{ slug: "piratesoftware-bsky", name: "piratesoftware.live (BlueSky)", ids: ["other"] }]);
+ const manifest = `${ORIGIN}/posts/piratesoftware-bsky/manifest.json`;
+ const asked = [];
+ const getJson = async (url, { refresh = false } = {}) => {
+ asked.push({ url, refresh });
+ if (url.endsWith("/corpus.json") && !refresh) return a.docs.get(url);
+ throw new Error(`GET ${url} -> timed out`);
+ };
+ const resolver = createPostChannelResolver({ getJson });
+ const posts = [BSKY, { ...BSKY, id: "b2", url: "https://bsky.app/profile/piratesoftware.live/post/rk2" }, { ...BSKY, id: "b3", url: "https://bsky.app/profile/piratesoftware.live/post/rk3" }];
+ const r = await resolvePostLinks({ posts, provenance: PROV, render: DECK, resolver });
+ assert.ok(r.posts.every((p) => p.siteChannel === undefined));
+ assert.ok(r.notes.every((n) => /did not answer .* -- QR links the original/.test(n)), r.notes.join("\n"));
+ // The manifest: asked once, by the first post's cached pass, never again.
+ assert.deepEqual(asked.filter((x) => x.url === manifest), [{ url: manifest, refresh: false }]);
+ // corpus.json fresh: asked once, by the first post's second pass.
+ assert.equal(asked.filter((x) => x.url.endsWith("/corpus.json") && x.refresh).length, 1);
+});
+
+test("failure memo: a server's resolver asks again once refreshAfterMs has passed, and a success re-arms it", async () => {
+ let t = 0;
+ let down = true;
+ const a = archive([{ slug: "piratesoftware-bsky", name: "piratesoftware.live (BlueSky)", ids: ["3l6uxj6esfr2z"] }]);
+ let asks = 0;
+ const getJson = async (url, opts) => {
+ asks += 1;
+ if (down) throw new Error(`GET ${url} -> 503`);
+ return a.getJson(url, opts);
+ };
+ const resolver = createPostChannelResolver({ getJson, refreshAfterMs: 600_000, now: () => t });
+ await assert.rejects(resolver.find(ORIGIN, BSKY), /503/);
+ assert.equal(asks, 1);
+ // Within the window: refused from the memo, nothing asked, even after the archive is back.
+ down = false;
+ t = 599_999;
+ await assert.rejects(resolver.find(ORIGIN, BSKY), /503/);
+ assert.equal(asks, 1);
+ // Past it: asked, found.
+ t = 600_000;
+ assert.equal(await resolver.find(ORIGIN, BSKY), "piratesoftware-bsky");
+ // The success cleared the memo; a new failure is remembered from its own time.
+ down = true;
+ const before = asks;
+ await assert.rejects(resolver.find(ORIGIN, { ...BSKY, url: "https://bsky.app/profile/piratesoftware.live/post/zz" }), /503/);
+ t = 600_001;
+ await assert.rejects(resolver.find(ORIGIN, { ...BSKY, url: "https://bsky.app/profile/piratesoftware.live/post/zz" }), /503/);
+ assert.equal(asks, before + 1, "one failed ask, then the memo");
+});
+
+test("resolvePostLinks: a deadline bounds the whole call; the rest keep their own links", async () => {
+ const resolver = { find: () => new Promise(() => {}) }; // an archive that never answers
+ const posts = [BSKY, { ...BSKY, id: "b2" }];
+ const started = Date.now();
+ const r = await resolvePostLinks({ posts, provenance: PROV, render: DECK, resolver, deadlineMs: 50 });
+ assert.ok(Date.now() - started < 1000);
+ assert.ok(r.posts.every((p) => p.siteChannel === undefined));
+ assert.match(r.notes[0], /still being asked when time ran out -- QR links the original/);
+ assert.match(r.notes[1], /no time left to ask the archive/);
+});
+
+test("resolvePostLinks: hidden posts and a cut without the deck are not looked up", async () => {
+ const asked = [];
+ const resolver = { find: async (o, p) => (asked.push(p.id), "piratesoftware-bsky") };
+ const r = await resolvePostLinks({ posts: [{ ...BSKY, hide: true }, X], provenance: PROV, render: DECK, resolver });
+ assert.deepEqual(asked, [X.id]);
+ assert.equal(r.posts[0].siteChannel, undefined);
+ assert.equal(r.notes.length, 1);
+ const off = await resolvePostLinks({ posts: [BSKY], provenance: PROV, render: {}, resolver });
+ assert.deepEqual(off, { posts: [BSKY], notes: [] });
+ assert.deepEqual(asked, [X.id]);
+});
+
+test("validatePosts: null is unset for siteChannel, siteUrl and postId, as the README's example writes them", () => {
+ assert.deepEqual(validatePosts([{ ...BSKY, attachTo: null, hide: false, siteChannel: null, siteUrl: null, postId: null }]), []);
+ assert.equal(postQrUrl({ ...BSKY, siteChannel: null, siteUrl: null, postId: null }, PROV), BSKY.url);
+});
+
+test("createJsonCache: callers asking for one URL at once share one fetch", async () => {
+ let n = 0;
+ let release;
+ const gate = new Promise((r) => { release = r; });
+ const fetchImpl = async () => { n += 1; await gate; return { ok: true, status: 200, json: async () => ({ n }) }; };
+ const c = createJsonCache({ cacheDir: null, fetchImpl });
+ const both = Promise.all([c.getJson("https://a.example/x"), c.getJson("https://a.example/x", { refresh: true })]);
+ release();
+ assert.deepEqual(await both, [{ n: 1 }, { n: 1 }]);
+ assert.equal(n, 1);
+ // Once it has landed, a refresh is a new fetch only when allowed (once per process here: not again).
+ assert.deepEqual(await c.getJson("https://a.example/x", { refresh: true }), { n: 1 });
+});
