commit d4436ad75f04d63a0e797f9662e1bf4b3c0dc7f0
parent 9c60e59f976f6b3fd878349008f48427d1f804fa
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 7 Sep 2026 16:03:12 -0400
common: one pageFileName and one CONTRACT
`page-${n padded to 4}.json` existed SIX times: three identical copies in
lib/manifest.ts (pageFileName / transcriptPageFileName / subsPageFileName) and
one each in lib/stats.ts, lib/posts.ts, lib/digests.ts. `page-0.json` is a 404
on every published archive, so a copy that lost its padding would fail only
against a real site and pass every unit test.
lib/manifest.ts now exports ONE `pageFileName`. The two aliases are gone and
their 12 call sites use it (controller/buildIndex.ts, components/
{subsCache,transcriptCache}.ts, export/app/lib/offlineCache.ts,
mcp/src/{source,protocol.test}.ts). The three per-layer names stay exported —
statsPageFileName, postsPageFileName, digestPageFileName are now aliases of the
one implementation, so no caller changed. The fourth copy, in
components/searchIndex.worker.ts, stays deliberately: it is a bundled web
worker whose only import today is its sibling protocol module.
`lib/corpus.ts` gains `CONTRACT`, one object holding every version constant
and layer name of the published contract: corpusSpec 3, siteDescriptor 1,
manifest 3, transcriptsManifest 1, subsManifest 4, subsChannelManifest 1,
summariesPageSize 1000, pagePad 4, and the five layer names. They lived in
three files. Each existing named export (CORPUS_SPEC_VERSION,
SITE_DESCRIPTOR_VERSION, MANIFEST_VERSION, TRANSCRIPTS_MANIFEST_VERSION,
SUBS_MANIFEST_VERSION, SUBS_CHANNEL_MANIFEST_VERSION, SUMMARIES_PAGE_SIZE) is
now a re-export of a CONTRACT field, so nothing outside common/lib changed.
Direction is corpus.ts -> {manifest,siteDescriptor}.ts; corpus.ts imports
siteDescriptor type-only, so there is no runtime cycle. Phase 2's
common/lib/archive/contract.ts re-exports this object.
THE WIRE IS PROVED UNCHANGED. A fixture corpus (editor/e2e/fixtures/
test-transcripts/one-youtube-channel-with-data plus a one-channel site.json)
was run through `bin/build-index.ts` + `bin/compose-site.ts` into two scratch
dirs, before this commit and after:
public/ 16 files vs 16 — identical once `generatedAt` is normalized
index/ 7 files vs 7 — identical once `generatedAt` is normalized
The ONLY differing bytes in either tree were `generatedAt` (in corpus.json,
site.json, the subs/posts/digests/summaries/transcripts manifests and the
archive's channel.json) and the zip's entry mtimes; the zip's CONTENTS diff
clean. Values on the wire are the ones CONTRACT now owns: corpus spec 3,
site.json contract 1, summaries manifest version 3 / pageSize 1000, subs
manifest 4, transcripts manifest 1, shards named page-0000.json.
Verified: tsc --noEmit clean in all six packages; common 876/876, mcp 205/205.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
12 files changed, 64 insertions(+), 46 deletions(-)
diff --git a/common/components/subsCache.ts b/common/components/subsCache.ts
@@ -3,7 +3,7 @@
import { useQueries, useQuery } from "@tanstack/react-query";
import type { SubsDetail } from "../lib/subs";
import type { ChannelSubsManifest, SubsManifest } from "../lib/manifest";
-import { subsPageFileName } from "../lib/manifest";
+import { pageFileName } from "../lib/manifest";
import { makeId, splitId, idBaseUrl } from "./originId";
async function fetchJson<T>(url: string): Promise<T> {
@@ -63,7 +63,7 @@ function fetchPage(
let p = pagePromises.get(key);
if (!p) {
p = fetchJson<SubsDetail[]>(
- `${idBaseUrl(origin)}/subs/${channelSlug}/${subsPageFileName(pageIndex)}`,
+ `${idBaseUrl(origin)}/subs/${channelSlug}/${pageFileName(pageIndex)}`,
);
p.catch(() => pagePromises.delete(key));
pagePromises.set(key, p);
diff --git a/common/components/transcriptCache.ts b/common/components/transcriptCache.ts
@@ -2,7 +2,7 @@
import type { TranscriptDetail } from "../lib/transcripts";
import type { ChannelTranscriptsManifest } from "../lib/manifest";
-import { transcriptPageFileName } from "../lib/manifest";
+import { pageFileName } from "../lib/manifest";
import { idbGet, idbPutBatch } from "./transcriptStore";
import { makeId, splitId, idBaseUrl } from "./originId";
@@ -64,7 +64,7 @@ function fetchPage(
let p = pagePromises.get(key);
if (!p) {
p = fetch(
- `${idBaseUrl(origin)}/transcripts/${channelSlug}/${transcriptPageFileName(pageIndex)}`,
+ `${idBaseUrl(origin)}/transcripts/${channelSlug}/${pageFileName(pageIndex)}`,
).then((r) => {
if (!r.ok)
throw new Error(
diff --git a/common/controller/buildIndex.ts b/common/controller/buildIndex.ts
@@ -61,8 +61,6 @@ import {
SUBS_CHANNEL_MANIFEST_VERSION,
SUBS_MANIFEST_VERSION,
pageFileName,
- transcriptPageFileName,
- subsPageFileName,
} from "../lib/manifest";
import { getSettings } from "../lib/settings";
import {
@@ -987,7 +985,7 @@ export async function buildIndex({
const writer = createPageWriter({
outDir: channelDir,
- fileName: transcriptPageFileName,
+ fileName: pageFileName,
maxPageBytes: maxTranscriptPageBytes,
ensureDir: false,
getPrevHash: (idx) => pageHashes.get([channelSlug, idx])?.hash,
@@ -1018,7 +1016,7 @@ export async function buildIndex({
pagesSkipped += chPagesSkipped;
const keep = new Set<string>(["manifest.json"]);
- for (let i = 0; i < pageCount; i++) keep.add(transcriptPageFileName(i));
+ for (let i = 0; i < pageCount; i++) keep.add(pageFileName(i));
const existing = await readdir(channelDir).catch(() => [] as string[]);
for (const name of existing) {
if (keep.has(name)) continue;
@@ -1153,7 +1151,7 @@ export async function buildIndex({
const subWriter = createPageWriter({
outDir: subsChannelDir,
- fileName: subsPageFileName,
+ fileName: pageFileName,
maxPageBytes: maxTranscriptPageBytes,
ensureDir: true,
getPrevHash: (idx) => subPageHashes.get([channelSlug, idx])?.hash,
@@ -1209,7 +1207,7 @@ export async function buildIndex({
}
const subKeep = new Set<string>(["manifest.json"]);
- for (let i = 0; i < subPageCount; i++) subKeep.add(subsPageFileName(i));
+ for (let i = 0; i < subPageCount; i++) subKeep.add(pageFileName(i));
const subExisting = await readdir(subsChannelDir).catch(
() => [] as string[],
);
diff --git a/common/lib/corpus.ts b/common/lib/corpus.ts
@@ -24,7 +24,33 @@ import { PROJECT_GENERATOR } from "./project";
// changes by not knowing about it, and the only in-repo consumer
// (mcp/src/source.ts) casts the parsed corpus loosely. Bumping the spec would
// force every client to re-evaluate compatibility for a credit line.
-export const CORPUS_SPEC_VERSION = 3;
+export const CONTRACT = {
+ // /corpus.json's own `spec`. See the note above for why `generator` did not
+ // bump it.
+ corpusSpec: 3,
+ // /site.json's `contract` (siteDescriptor.ts).
+ siteDescriptor: 1,
+ // The four manifest versions (manifest.ts). Each is the version field of one
+ // served document; they move independently and always have.
+ manifest: 3,
+ transcriptsManifest: 1,
+ subsManifest: 4,
+ subsChannelManifest: 1,
+ // Records per /summaries/page-NNNN.json.
+ summariesPageSize: 1000,
+ // The zero-padding on every page shard's file name. `page-0.json` is a 404 —
+ // this is the whole reason the constant exists in one place.
+ pagePad: 4,
+ // The served trees that follow the manifest -> slugToPage -> page-NNNN walk.
+ // "summaries" is flat (one manifest, pages, no per-channel level).
+ layers: ["transcripts", "subs", "posts", "digests", "summaries"],
+} as const;
+
+export type ContractLayer = (typeof CONTRACT.layers)[number];
+
+// Every version constant below is a field of CONTRACT, re-exported under the
+// name its callers already use. Nothing on the wire changes by moving one.
+export const CORPUS_SPEC_VERSION = CONTRACT.corpusSpec;
// How to resolve a single transcript from the paginated shards, described once
// and embedded in every corpus.json so any HTTP client can navigate without
diff --git a/common/lib/digests.ts b/common/lib/digests.ts
@@ -29,15 +29,13 @@ import type {
DigestDerivedFrom,
DigestProvenance,
} from "./digest";
+import { pageFileName } from "./manifest";
// v1: the initial shipped shape.
export const DIGESTS_MANIFEST_VERSION = 1;
-// Zero-padded to 4 digits, the convention every other page tree uses
-// (pageFileName / transcriptPageFileName / subsPageFileName / postsPageFileName).
-export function digestPageFileName(index: number): string {
- return `page-${String(index).padStart(4, "0")}.json`;
-}
+// One page-shard name for every layer; see lib/manifest.ts.
+export const digestPageFileName = pageFileName;
// A titled moment. `start` is SECONDS and is what a seek uses: the parser has
// already snapped it to a real cue boundary. `clock` is the raw HH:MM:SS string
diff --git a/common/lib/manifest.ts b/common/lib/manifest.ts
@@ -1,4 +1,5 @@
import type { ChannelGroup } from "./channelGroups";
+import { CONTRACT } from "./corpus";
export type ChannelEntry = {
name: string;
@@ -33,18 +34,18 @@ export type Manifest = {
// v3: manifests are now per-site (channels/groups are a site selection, not the
// whole pool). Bumped so a stale single-site manifest forces a rebuild.
-export const MANIFEST_VERSION = 3;
-export const SUMMARIES_PAGE_SIZE = 1000;
+export const MANIFEST_VERSION = CONTRACT.manifest;
+export const SUMMARIES_PAGE_SIZE = CONTRACT.summariesPageSize;
+// THE page-shard file name, for every layer. It was three identical copies
+// in this file (summaries, transcripts, subs) plus one per layer module;
+// `page-0.json` is a 404 on every published archive, so a copy that lost the
+// padding would 404 silently against a real site and pass every unit test.
export function pageFileName(index: number): string {
- return `page-${String(index).padStart(4, "0")}.json`;
+ return `page-${String(index).padStart(CONTRACT.pagePad, "0")}.json`;
}
-export const TRANSCRIPTS_MANIFEST_VERSION = 1;
-
-export function transcriptPageFileName(index: number): string {
- return `page-${String(index).padStart(4, "0")}.json`;
-}
+export const TRANSCRIPTS_MANIFEST_VERSION = CONTRACT.transcriptsManifest;
export type ChannelTranscriptsManifest = {
version: number;
@@ -56,7 +57,7 @@ export type ChannelTranscriptsManifest = {
};
// v4: per-site subs manifest (channels/groups are a site selection).
-export const SUBS_MANIFEST_VERSION = 4;
+export const SUBS_MANIFEST_VERSION = CONTRACT.subsManifest;
export type SubsChannelEntry = {
name: string;
@@ -82,7 +83,7 @@ export type SubsManifest = {
siteId?: string;
};
-export const SUBS_CHANNEL_MANIFEST_VERSION = 1;
+export const SUBS_CHANNEL_MANIFEST_VERSION = CONTRACT.subsChannelManifest;
export type ChannelSubsManifest = {
version: number;
@@ -94,6 +95,3 @@ export type ChannelSubsManifest = {
slugToPage: Record<string, number>;
};
-export function subsPageFileName(index: number): string {
- return `page-${String(index).padStart(4, "0")}.json`;
-}
diff --git a/common/lib/posts.ts b/common/lib/posts.ts
@@ -8,6 +8,8 @@
// Client-safe: no node imports (same convention as cookiePolicy.ts /
// availability.ts). The server-side store lives in posts-server.ts.
+import { pageFileName } from "./manifest";
+
export type PostPlatform = "twitter" | "bluesky";
export const POST_PLATFORM_VALUES: ReadonlyArray<PostPlatform> = [
@@ -161,9 +163,7 @@ export type PostsManifest = {
export const SITE_POSTS_MANIFEST_VERSION = 1;
-export function postsPageFileName(index: number): string {
- return `page-${String(index).padStart(4, "0")}.json`;
-}
+export const postsPageFileName = pageFileName;
// ---------------------------------------------------------------------------
// Derivations
diff --git a/common/lib/siteDescriptor.ts b/common/lib/siteDescriptor.ts
@@ -1,4 +1,5 @@
import type { ChannelGroup } from "./channelGroups";
+import { CONTRACT } from "./corpus";
import type { Manifest } from "./manifest";
import type { Site } from "./site";
import type { SocialLink } from "./settings";
@@ -15,7 +16,7 @@ import type { SocialLink } from "./settings";
// Deliberately excludes operational/internal config (cloudflareProject,
// relatedSites): only public presentation fields belong here.
-export const SITE_DESCRIPTOR_VERSION = 1;
+export const SITE_DESCRIPTOR_VERSION = CONTRACT.siteDescriptor;
// One channel a site exposes, with the slug that gives it a collision-free
// identity across sites (display names collide, slugs don't).
diff --git a/common/lib/stats.ts b/common/lib/stats.ts
@@ -1,4 +1,5 @@
import type { Platform } from "./platform";
+import { pageFileName } from "./manifest";
import type { VideoState } from "./availability";
// Bumping this invalidates the LMDB `statsByPath` incremental cache and forces
@@ -20,9 +21,7 @@ export type MediaType = "video" | "livestream" | "short";
// flushed by byte size rather than a fixed record count. Kept well under 25 MB.
export const STATS_MAX_PAGE_BYTES = 20 * 1024 * 1024;
-export function statsPageFileName(index: number): string {
- return `page-${String(index).padStart(4, "0")}.json`;
-}
+export const statsPageFileName = pageFileName;
// One record per video. Engagement fields are nullable: not every platform /
// metadata.info.json carries them, and older archived videos may omit them.
diff --git a/export/app/lib/offlineCache.ts b/export/app/lib/offlineCache.ts
@@ -12,7 +12,7 @@
// per (origin, slug); the site SW ignores it (same-origin only).
import {
- transcriptPageFileName,
+ pageFileName,
type ChannelTranscriptsManifest,
} from "yt-dlp-transcript-common/lib/manifest";
import { idBaseUrl, makeId } from "yt-dlp-transcript-common/components/originId";
@@ -88,7 +88,7 @@ export async function downloadChannelOffline(
const base = idBaseUrl(origin);
const urls = [`${base}/transcripts/${slug}/manifest.json`];
for (let p = 0; p < manifest.pageCount; p++) {
- urls.push(`${base}/transcripts/${slug}/${transcriptPageFileName(p)}`);
+ urls.push(`${base}/transcripts/${slug}/${pageFileName(p)}`);
}
await sendToSw(sw, { type: "CACHE_URLS", origin, slug, urls }, (data) => {
if (data.type === "progress") {
diff --git a/mcp/src/protocol.test.ts b/mcp/src/protocol.test.ts
@@ -9,7 +9,7 @@ import {
StdioClientTransport,
getDefaultEnvironment,
} from "@modelcontextprotocol/client/stdio";
-import { transcriptPageFileName } from "yt-dlp-transcript-common/lib/manifest";
+import { pageFileName } from "yt-dlp-transcript-common/lib/manifest";
// ─── Real-process protocol coverage ───
//
@@ -85,7 +85,7 @@ async function writeFixture(): Promise<string> {
cues: [{ start: 12, end: 15, text }],
});
await writeFile(
- path.join(chDir, transcriptPageFileName(0)),
+ path.join(chDir, pageFileName(0)),
JSON.stringify([
rec("a1", "Coffee one", "i love coffee"),
rec("a2", "Tea two", "i love tea"),
diff --git a/mcp/src/source.ts b/mcp/src/source.ts
@@ -1,8 +1,6 @@
import { readFile, readdir } from "node:fs/promises";
import path from "node:path";
import {
- transcriptPageFileName,
- subsPageFileName,
pageFileName,
type ChannelTranscriptsManifest,
type ChannelSubsManifest,
@@ -662,7 +660,7 @@ export class LocalSource implements ShardSource {
subsPage(ch: ChannelRef, page: number): Promise<SubsDetail[]> {
return this.subsPages.take(`${ch.slug}:${page}`, () =>
readLocalJsonSized<SubsDetail[]>(
- path.join(this.dir, "subs", ch.slug, subsPageFileName(page)),
+ path.join(this.dir, "subs", ch.slug, pageFileName(page)),
"subsPage",
),
);
@@ -781,7 +779,7 @@ export class LocalSource implements ShardSource {
transcriptPage(ch: ChannelRef, page: number): Promise<TranscriptDetail[]> {
return this.transcriptPages.take(`${ch.slug}:${page}`, () =>
readLocalJsonSized<TranscriptDetail[]>(
- path.join(this.dir, "transcripts", ch.slug, transcriptPageFileName(page)),
+ path.join(this.dir, "transcripts", ch.slug, pageFileName(page)),
"transcriptPage",
),
);
@@ -917,7 +915,7 @@ export class RemoteSource implements ShardSource {
subsPage(ch: ChannelRef, page: number): Promise<SubsDetail[]> {
return this.subsPages.take(`${ch.slug}:${page}`, () =>
this.getJsonSized<SubsDetail[]>(
- `/subs/${ch.slug}/${subsPageFileName(page)}`,
+ `/subs/${ch.slug}/${pageFileName(page)}`,
"subsPage",
),
);
@@ -1087,7 +1085,7 @@ export class RemoteSource implements ShardSource {
transcriptPage(ch: ChannelRef, page: number): Promise<TranscriptDetail[]> {
return this.transcriptPages.take(`${ch.slug}:${page}`, () =>
this.getJsonSized<TranscriptDetail[]>(
- `/transcripts/${ch.slug}/${transcriptPageFileName(page)}`,
+ `/transcripts/${ch.slug}/${pageFileName(page)}`,
"transcriptPage",
),
);
