commit c1c69a59fece812f57a089c6a04c5cde549b5672
parent 6b097ff87b1cc826a1fb5abc758fbc4fb19115fe
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 21 Sep 2026 13:21:57 -0400
tags: curated tags need no SCHEMA_VERSION bump, and the site layer holds no rules
SCHEMA_VERSION goes back to 13. Every prior bump existed because something
would otherwise never be derived at all — v13 populates a new sub-DB and
nothing re-derives it lazily. Curated tags re-derive lazily by design:
curatedRulesHash is absent on an index built before them, so it can never equal
the current hash, and the first build re-derives every record on its own out of
LMDB (~80 s across 30k videos). On an untagged corpus that pass writes nothing
at all — an empty derivation leaves each summary byte-identical — so the
shared pages are never rewritten and compose copies nothing. A bump would have
wiped the cache and re-read 76k video directories to reach the same state. The
comment at the constant says all of that, so the next person does not re-add it.
Two tests pin the cold start: an index with no stored hashes examines every
record, and an untagged corpus cold-starts to zero changes with the summaries
left byte-identical.
Also written down where it belongs (store header + FACTS): a site-layer RULE
never tags a record. mergeTagDefs still appends one — harmless on a published
def — but rules are evaluated once at index time over records SHARED by every
site carrying the channel, so there is no per-site record for a per-site hit to
live in. Rules bind only from transcripts/tags.json.
Co-Authored-By: Claude Opus <noreply@anthropic.com>
Diffstat:
4 files changed, 74 insertions(+), 13 deletions(-)
diff --git a/common/controller/buildIndex.ts b/common/controller/buildIndex.ts
@@ -149,12 +149,19 @@ import {
// (effectiveDigest of the machine sidecar + the human override file) plus a
// shared /digests/<slug>/ page tree. Bumped so existing indexes populate the
// new sub-DB — nothing re-derives it lazily.
-// v14: curated per-video tags. Summaries now carry `curatedTags` (derived from
-// transcripts/tags.json: rule hits folded with the operator's pins and
-// suppressions). Bumped so every existing record is re-derived once; after
-// that the two `meta` hashes (curatedRulesHash / curatedAssignHash) carry
-// invalidation, and no new sub-DB was added — see curatedTagsIndex.ts.
-const SCHEMA_VERSION = 14;
+// NOT a v14 — curated tags (`curatedTags` on every summary, derived from
+// transcripts/tags.json) deliberately did NOT bump this, and the reason is
+// worth keeping: every prior bump existed because something would otherwise
+// never be derived at all (v13 populates a new sub-DB; nothing re-derives it
+// lazily). Curated tags DO re-derive lazily, by design. `curatedRulesHash` is
+// absent on an index built before them, so it can never equal the current hash
+// and `reapplyCuratedTags` re-derives every record on the first build after
+// this ships — measured at ~80 s across 30k videos, straight out of LMDB, and
+// zero page writes on an untagged corpus because an empty derivation leaves
+// each summary byte-identical. A bump would instead wipe the cache and re-read
+// 76k video directories to reach exactly the same state. No sub-DB was added,
+// so the clearAsync() enumeration above is unchanged too.
+const SCHEMA_VERSION = 13;
// Per-channel post stats, persisted so per-site aggregates survive a no-op
// rebuild that doesn't re-encode the post pages. Mirrors ChannelSubsStat.
diff --git a/common/controller/curatedTagsIndex.test.ts b/common/controller/curatedTagsIndex.test.ts
@@ -208,6 +208,40 @@ test("unchanged hashes do no work at all", () => {
});
});
+// THE COLD START, and the reason curated tags need no SCHEMA_VERSION bump: an
+// index built before they existed carries no hash at all, which can never equal
+// the current one, so the first build re-derives every record by itself.
+test("an index with no stored hashes re-derives every record", () => {
+ const f = fixture(cfg([COLLAB_RULE]));
+ f.put(summary("lm", "v1", "20260101", { title: "Elfpire collab" }));
+ f.put(summary("lm", "v2", "20260102", { title: "ordinary" }));
+ assert.equal(f.meta.get(META_RULES_HASH), undefined);
+ const res = reapplyCuratedTags({ ...f });
+ assert.equal(res.rulesChanged, true);
+ assert.equal(res.examined, 2, "every record is a candidate on a cold index");
+ assert.deepEqual(res.changed.map((k) => k[2]), ["v1"]);
+});
+
+test("an UNTAGGED corpus cold-starts to zero changes — no page rewrites", () => {
+ // The state of every existing install the day this ships: no vocabulary, no
+ // assignments. The pass still runs (the hashes have to be recorded), but
+ // every record derives [] and comes out byte-identical, so nothing is written
+ // and the caller never flips sharedNeedsBuild. This is what makes the absent
+ // SCHEMA_VERSION bump free rather than merely cheap.
+ const f = fixture(cfg([]));
+ for (let i = 0; i < 10; i++) f.put(summary("lm", `v${i}`, `2026010${i}`));
+ const before = JSON.stringify(Array.from(f.sums.map.values()));
+ const res = reapplyCuratedTags({ ...f });
+ assert.deepEqual(res.changed, []);
+ assert.equal(res.examined, 10);
+ assert.equal(JSON.stringify(Array.from(f.sums.map.values())), before);
+ for (const { value } of f.sums.map.values()) {
+ assert.equal("curatedTags" in value, false);
+ }
+ // And the hashes are now recorded, so every later build is a true no-op.
+ assert.equal(reapplyCuratedTags({ ...f }).examined, 0);
+});
+
test("a rule change re-derives the whole corpus from LMDB and reports the movers", () => {
const f = fixture(cfg([COLLAB_RULE]));
f.put(summary("lm", "v1", "20260101", { title: "Elfpire collab" }));
diff --git a/common/lib/curatedTagsStore.ts b/common/lib/curatedTagsStore.ts
@@ -3,9 +3,16 @@
// - per-site: sites/<siteId>/tags.json (siteTagsFile)
// The global file is AUTHORITATIVE for both the vocabulary and the assignments
// (an assignment is a fact about a video, not a presentation choice). The
-// per-site file is presentation — relabel/recolour/reorder/hide — plus
-// site-only rules and site-only tags. See common/lib/curatedTags.ts for the
-// pure model, the coercion and mergeTagDefs.
+// per-site file is PRESENTATION — relabel/recolour/reorder/hide — plus
+// site-only tags. See common/lib/curatedTags.ts for the pure model, the
+// coercion and mergeTagDefs.
+//
+// **A site-layer RULE never tags a record.** mergeTagDefs will append one (it
+// is harmless on a published /tags.json def), but rules are evaluated once at
+// index time over records SHARED by every site that carries the channel —
+// there is no per-site record for a per-site hit to live in. Rules bind only
+// from transcripts/tags.json; promote one there to make it fire. This is why
+// the editor offers no site-side rule editor.
//
// Unlike aliasesStore there are NO seeded defaults: a fresh install has no
// tags, and an absent global file reads as an empty config.
diff --git a/plans/FACTS.md b/plans/FACTS.md
@@ -201,7 +201,7 @@ Exposed to the settings form via `listTranscriptionApps()` (`:272`), consumed at
| Constant | Where | Value |
| --- | --- | --- |
-| `SCHEMA_VERSION` | `buildIndex.ts:157` | **14** (13 → 14 for curated tags, 2026-09-21) |
+| `SCHEMA_VERSION` | `buildIndex.ts:164` | **13** — and curated tags deliberately did NOT bump it (2026-09-21); see below |
| `CUES_FILE_VERSION` | `normalizeTranscript.ts:33` | 2 |
| `CORPUS_SPEC_VERSION` | `common/lib/archive/contract.ts:35` (re-exported `corpus.ts:59`) | **4** (3 → 4 for `/tags.json`, 2026-09-21) |
| `MANIFEST_VERSION` (site summaries) | `common/lib/manifest.ts:36` | 3 |
@@ -246,9 +246,22 @@ then leaves the untouched pages untouched. The per-site fingerprint (`buildIndex
includes both hashes, or a tag-only edit would leave a site reporting "up to date" with
yesterday's counts. **No new sub-DB**, so the `clearAsync()` enumeration is unchanged.
-Only the CORPUS vocabulary is evaluated: a site-only rule contributes its definition to that
-site's `/tags.json` but does **not** tag records, because records are shared by every site
-carrying the channel. Promote a rule to `transcripts/tags.json` to make it bind.
+**And no `SCHEMA_VERSION` bump — on purpose.** Every prior bump existed because something
+would otherwise never be derived at all (v13 populates a new sub-DB; nothing re-derives it
+lazily). Curated tags re-derive lazily *by design*: `curatedRulesHash` is absent on an index
+built before them, so it can never equal the current hash and the first build re-derives
+every record on its own — ~80 s across 30k videos, out of LMDB, and **zero page writes on an
+untagged corpus** because an empty derivation leaves each summary byte-identical (tested:
+"an UNTAGGED corpus cold-starts to zero changes"). A bump would instead wipe the cache and
+re-read 76k video directories to reach exactly the same state.
+
+**The site layer is presentation-only.** `sites/<id>/tags.json` relabels, recolours,
+reorders, hides and may add site-only tags; it can never delete a corpus rule or an
+assignment. A site-layer RULE is accepted by `mergeTagDefs` (harmless on a published def)
+but **never tags a record**: rules are evaluated once at index time over records SHARED by
+every site carrying the channel, so there is no per-site record for a per-site hit to live
+in. Rules bind only from `transcripts/tags.json` — promote one there to make it fire. The
+editor therefore offers no site-side rule editor.
**A chat cue's author is a string prefix, not a field.** `common/lib/liveChat.ts:100-105`
emits every live-chat cue as ``text: author ? `${author}: ${text}` : text`` — `Cue` is
