commit 7fff691b64dc662dba14d140114f6b30747518dd
parent 9d38f625e5cf7b01a99cdba857b74aa52927477e
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 5 Oct 2026 02:11:06 -0400
common: site.json `publish` ("full" | "cited") and `reports` (ordered report ids); SITE.md regenerated
- publish: absent reads "full"; only "cited" is written. isCitedSite is the
one predicate. The cited scope is applied by the reports pipeline; until it
is, a cited site builds as a full one.
- reports: ordered report ids, each a slug ([a-z0-9][a-z0-9-]*); invalid and
repeated ids are dropped (parseSiteReports); written only when non-empty.
- Tests: defaults read and are not written, cited round-trips through
writeSite/getSite, invalid report ids dropped on read and on write.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
3 files changed, 128 insertions(+), 0 deletions(-)
diff --git a/SITE.md b/SITE.md
@@ -25,6 +25,8 @@ Regenerate this file with `pnpm --filter yt-dlp-transcript-common exec tsx bin/f
| [`siteUrl`](#siteurl) | absent |
| [`listed`](#listed) | `true` |
| [`audience`](#audience) | absent |
+| [`publish`](#publish) | `"full"` |
+| [`reports`](#reports) | `[]` |
| [`relatedSites`](#relatedsites) | `[]` |
| [`pwa`](#pwa) | `false` |
| [`archives`](#archives) | `true` |
@@ -171,6 +173,22 @@ Who this site is built for. `"public"` (the default; absent) or `"private"`: the
Default: absent
+## `publish`
+
+What this site publishes. `"full"` (the default; absent): the searchable corpus of its `channels`. `"cited"`: only the site's `reports` and the moments they cite — no search, no browse, no full transcripts, no archives; `channels` is then the pool its citations may resolve against. The cited scope is applied by the reports pipeline at build time. Only `"cited"` is written; any other value reads as `"full"`.
+
+Default: `"full"`
+
+## `reports`
+
+The site's published reports, in display order: report ids (lowercase slugs, `[a-z0-9][a-z0-9-]*`), each a directory under `sites/<siteId>/reports/`. A report directory not named here is a draft and is not published. Invalid and repeated ids are dropped. Absent/empty = no reports.
+
+Default:
+
+```json
+[]
+```
+
## `relatedSites`
Pulls specific siblings to the front of the footer's cross-site list, in named groups. Siblings not named here fall into a trailing "Other sites" group. Absent/empty = one flat list of every sibling.
diff --git a/common/lib/siteSchema.test.ts b/common/lib/siteSchema.test.ts
@@ -10,8 +10,10 @@ import {
SITE_FIELD_DOCS,
SITE_KEYS,
channelsOnlyOnUnlistedSites,
+ isCitedSite,
isListedSite,
parseSite,
+ parseSiteReports,
siteFieldsSchema,
siteToDisk,
type Site,
@@ -64,6 +66,8 @@ test("empty, null, [] and a number all read as the defaults, every key emitted",
assert.equal(want.pwa, false);
assert.equal(want.listed, true);
assert.deepEqual(want.relatedSites, []);
+ assert.equal(want.publish, "full");
+ assert.deepEqual(want.reports, []);
assert.equal(want.socialLinks, undefined);
assert.ok("socialLinks" in want);
for (const raw of [null, [], 3, "x", undefined]) {
@@ -235,6 +239,8 @@ function fixtures(): Array<[string, unknown]> {
cloudflareProject: "p",
siteUrl: "https://s.example//",
listed: false,
+ publish: "cited",
+ reports: ["r-one", "r-one", "Bad", "", 7, "r-two"],
relatedSites: [{ siteIds: ["x", "x", "BAD"] }, { label: " ", siteIds: [] }],
pwa: true,
archives: false,
@@ -310,6 +316,53 @@ test("listed: absent reads listed, only an explicit false unlists, and only fals
assert.equal(getSite("s", paths).listed, true);
});
+test("publish: absent reads full, only cited is kept, and only cited is written", () => {
+ for (const v of [undefined, "full", "FULL", "Cited", "corpus", true, 1, null]) {
+ const site = parseSite("s", v === undefined ? {} : { publish: v });
+ assert.equal(site.publish, "full", String(v));
+ assert.equal(isCitedSite(site), false, String(v));
+ assert.equal("publish" in siteToDisk(site), false, String(v));
+ }
+ const cited = parseSite("s", { publish: "cited" });
+ assert.equal(cited.publish, "cited");
+ assert.equal(isCitedSite(cited), true);
+ assert.equal(siteToDisk(cited).publish, "cited");
+ assert.equal(isCitedSite({}), false);
+});
+
+test("reports: ordered slugs, invalid and repeated ids dropped, written only when non-empty", () => {
+ assert.deepEqual(
+ parseSiteReports(["b-two", "a-one", "b-two", "Upper", "-lead", "has space", "", 3, null, "c3"]),
+ ["b-two", "a-one", "c3"],
+ );
+ for (const v of [undefined, null, "a-one", { a: 1 }, 5]) {
+ assert.deepEqual(parseSite("s", v === undefined ? {} : { reports: v }).reports, [], String(v));
+ }
+ const site = parseSite("s", { reports: ["z-last", "a-first", "a-first"] });
+ assert.deepEqual(site.reports, ["z-last", "a-first"]);
+ assert.deepEqual(siteToDisk(site).reports, ["z-last", "a-first"]);
+ assert.equal("reports" in siteToDisk(parseSite("s", { reports: [] })), false);
+ assert.equal("reports" in siteToDisk(parseSite("s", { reports: ["BAD"] })), false);
+ // siteToDisk re-parses a caller's list: a bad id handed to the writer is not persisted.
+ assert.deepEqual(siteToDisk({ ...site, reports: ["ok", "NOPE", "ok"] }).reports, ["ok"]);
+});
+
+test("writeSite → getSite round-trips a cited site and its reports; a full site writes neither key", async () => {
+ const paths = scratchPaths(await mkdtemp(path.join(os.tmpdir(), "site-")));
+ const cited = parseSite("s", { publish: "cited", reports: ["first", "second"] });
+ await writeSite(cited, paths);
+ assert.deepEqual(getSite("s", paths), cited);
+ const disk = JSON.parse(await readFile(siteConfigFile(paths, "s"), "utf8"));
+ assert.equal(disk.publish, "cited");
+ assert.deepEqual(disk.reports, ["first", "second"]);
+ await writeSite({ ...cited, publish: "full", reports: [] }, paths);
+ const plain = JSON.parse(await readFile(siteConfigFile(paths, "s"), "utf8"));
+ assert.equal("publish" in plain, false);
+ assert.equal("reports" in plain, false);
+ assert.equal(getSite("s", paths).publish, "full");
+ assert.deepEqual(getSite("s", paths).reports, []);
+});
+
test("isListedSite is the key's default; channelsOnlyOnUnlistedSites keeps a shared channel with the listed site", () => {
assert.equal(isListedSite({}), true);
assert.equal(isListedSite({ listed: true }), true);
@@ -353,4 +406,6 @@ test("writeSite → getSite round-trips, and the file holds only non-defaults",
assert.equal("transcriptDownloads" in disk, false);
assert.equal("pwa" in disk, false);
assert.equal("listed" in disk, false);
+ assert.equal("publish" in disk, false);
+ assert.equal("reports" in disk, false);
});
diff --git a/common/lib/siteSchema.ts b/common/lib/siteSchema.ts
@@ -89,6 +89,46 @@ export function isPrivateSite(site: Pick<Site, "audience">): boolean {
return site.audience === "private";
}
+// What a site publishes (report sites). "full" (the default, never written) is
+// the searchable corpus every site has always been. "cited" publishes only the
+// site's reports and the moments they cite; the reports pipeline applies that
+// scope, and until it does a cited site builds as a full one.
+export type SitePublish = "full" | "cited";
+
+export const SITE_PUBLISH_SCOPES: readonly SitePublish[] = ["full", "cited"];
+
+export function isSitePublish(v: unknown): v is SitePublish {
+ return v === "full" || v === "cited";
+}
+
+// THE ONE PREDICATE for a cited-only site. Absent or anything but "cited" reads
+// as full.
+export function isCitedSite(site: Pick<Site, "publish">): boolean {
+ return site.publish === "cited";
+}
+
+// A report id: a lowercase slug, the directory name under
+// `sites/<siteId>/reports/`. The same grammar as a site id.
+export const REPORT_ID_RE = /^[a-z0-9][a-z0-9-]*$/;
+
+export function isValidReportId(id: unknown): id is string {
+ return typeof id === "string" && REPORT_ID_RE.test(id);
+}
+
+// Parse the published report list: valid ids, first occurrence kept, order
+// preserved. Anything else (a non-array, a bad or repeated id) is dropped.
+export function parseSiteReports(input: unknown): string[] {
+ if (!Array.isArray(input)) return [];
+ const out: string[] = [];
+ const seen = new Set<string>();
+ for (const id of input) {
+ if (!isValidReportId(id) || seen.has(id)) continue;
+ seen.add(id);
+ out.push(id);
+ }
+ return out;
+}
+
// A Site is a selection + presentation layer over the single global channel
// pool. Each field is documented in SITE_FIELD_DOCS below.
export type Site = {
@@ -107,6 +147,8 @@ export type Site = {
siteUrl?: string;
listed?: boolean;
audience?: SiteAudience;
+ publish?: SitePublish;
+ reports?: string[];
relatedSites?: RelatedSiteGroup[];
pwa?: boolean;
archives?: boolean;
@@ -143,6 +185,10 @@ export const SITE_FIELD_DOCS: FieldDocs<Site> = {
"Whether the family lists this site. Opt-OUT: absent/true = listed, only an explicit `false` is written. An unlisted site still builds and deploys as before, and its own pages are unchanged; it is left out of the homepage (cards, chart, `/stats`), the hub (members, federated search, `/corpus.json`, `/llms.txt`), every other site's footer, and the published `channel-sites.json` and pooled `stats/`. A channel only unlisted sites expose is in none of the family's public totals; a channel a listed site also exposes is credited to the listed one.",
audience:
'Who this site is built for. `"public"` (the default; absent) or `"private"`: the operator\'s own reading copy, built on this machine and never deployed — every deploy path (Build & deploy, Deploy, `archilyzer deploy site`, Build & deploy all, docker/publish-site.sh) refuses it before any upload, while a build without a deploy still works. A private site is never listed (as `listed: false`, whatever `listed` says), publishes no `hubUrl`, and its `/corpus.json` says `"audience": "private"`. Content kept from the public — X posts while `social.x.visibility` is `"private"` — is built only into private sites. Only `"private"` is written.',
+ publish:
+ 'What this site publishes. `"full"` (the default; absent): the searchable corpus of its `channels`. `"cited"`: only the site\'s `reports` and the moments they cite — no search, no browse, no full transcripts, no archives; `channels` is then the pool its citations may resolve against. The cited scope is applied by the reports pipeline at build time. Only `"cited"` is written; any other value reads as `"full"`.',
+ reports:
+ "The site's published reports, in display order: report ids (lowercase slugs, `[a-z0-9][a-z0-9-]*`), each a directory under `sites/<siteId>/reports/`. A report directory not named here is a draft and is not published. Invalid and repeated ids are dropped. Absent/empty = no reports.",
relatedSites:
"Pulls specific siblings to the front of the footer's cross-site list, in named groups. Siblings not named here fall into a trailing \"Other sites\" group. Absent/empty = one flat list of every sibling.",
pwa:
@@ -312,6 +358,11 @@ export const siteFieldsSchema = z.object({
audience: settingsField((v): SiteAudience | undefined =>
v === "private" ? "private" : undefined,
).describe(d.audience),
+ // Only "cited" is kept; absent (and anything else) is the full default.
+ publish: settingsField((v): SitePublish =>
+ v === "cited" ? "cited" : "full",
+ ).describe(d.publish),
+ reports: settingsField(parseSiteReports).describe(d.reports),
relatedSites: settingsField(parseRelatedSites).describe(d.relatedSites),
pwa: settingsField((v): boolean => v === true).describe(d.pwa),
// Opt-out: only an explicit false disables. Absent/true stays on.
@@ -378,6 +429,7 @@ export function siteToDisk(site: Site): Site {
(c) => !c.groupId || groups.some((g) => g.id === c.groupId),
);
const relatedSites = parseRelatedSites(site.relatedSites);
+ const reports = parseSiteReports(site.reports);
const accent = parseAccentSetting(site.accent);
const wordmarkLead = wordmarkLeadFor(site.headerTitle, site.wordmarkLead);
const siteUrl = parseSiteUrl(site.siteUrl);
@@ -405,6 +457,9 @@ export function siteToDisk(site: Site): Site {
...(site.listed === false ? { listed: false } : {}),
// Public is the default: only the private audience is persisted.
...(isPrivateSite(site) ? { audience: "private" as const } : {}),
+ // Full is the default: only the cited scope is persisted.
+ ...(isCitedSite(site) ? { publish: "cited" as const } : {}),
+ ...(reports.length > 0 ? { reports } : {}),
...(relatedSites.length > 0 ? { relatedSites } : {}),
...(site.pwa ? { pwa: true } : {}),
// Persist only the non-default: archives is on unless explicitly disabled.
