commit 1ddf9f95a156ac050c53f24e2b4ed82d4896f642 parent 77099b41c695b06247ecb60c23e2fdfcd8c1a0b9 Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st> Date: Mon, 5 Oct 2026 12:23:58 -0400 site.json: a per-site `search` switch replaces `publish`; report-only is search off - `search` (boolean, default true; only false is written). isCitedSite is `search === false`; a search-off site with no reports is an empty report index. - Legacy `publish: "cited"` reads as `search: false`; siteToDisk never writes `publish`. SitePublish / SITE_PUBLISH_SCOPES / isSitePublish removed. - Editor site form: the Publish select is a Search checkbox; the Reports tab says "Reports only — search is off" / "Full archive — search is on". - builtScopeProblem reads the predicate; fixtures and tests use `search: false`, one legacy test kept; SITE.md regenerated. corpus.json `site.scope: "cited"` is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Diffstat:
26 files changed, 185 insertions(+), 155 deletions(-)
diff --git a/SITE.md b/SITE.md @@ -25,9 +25,9 @@ Regenerate this file with `pnpm --filter yt-dlp-transcript-common exec tsx bin/f | [`siteUrl`](#siteurl) | absent | | [`listed`](#listed) | `true` | | [`audience`](#audience) | absent | -| [`publish`](#publish) | `"full"` | | [`reports`](#reports) | `[]` | | [`relatedSites`](#relatedsites) | `[]` | +| [`search`](#search) | `true` | | [`pwa`](#pwa) | `false` | | [`archives`](#archives) | `true` | | [`duplicates`](#duplicates) | `true` | @@ -163,7 +163,7 @@ Default: absent ## `listed` -Whether the family lists this site. Opt-OUT: absent/true = listed, only an explicit `false` is written. An unlisted site still builds and deploys as before, and its own pages are unchanged; it is left out of the homepage (cards, chart, `/stats`), the hub (members, federated search, `/corpus.json`, `/llms.txt`), every other site's footer, and the published `channel-sites.json` and pooled `stats/`. A channel only unlisted sites expose is in none of the family's public totals; a channel a listed site also exposes is credited to the listed one. A private site and a cited site (`publish: "cited"`) are never listed, whatever `listed` says. +Whether the family lists this site. Opt-OUT: absent/true = listed, only an explicit `false` is written. An unlisted site still builds and deploys as before, and its own pages are unchanged; it is left out of the homepage (cards, chart, `/stats`), the hub (members, federated search, `/corpus.json`, `/llms.txt`), every other site's footer, and the published `channel-sites.json` and pooled `stats/`. A channel only unlisted sites expose is in none of the family's public totals; a channel a listed site also exposes is credited to the listed one. A private site and a report-only site (`search: false`) are never listed, whatever `listed` says. Default: `true` @@ -173,12 +173,6 @@ Who this site is built for. `"public"` (the default; absent) or `"private"`: the Default: absent -## `publish` - -What this site publishes. `"full"` (the default; absent): the searchable corpus of its `channels`. `"cited"`: only the site's `reports` and the moments they cite — no search, no browse, no full transcripts, no archives; `channels` is then the pool its citations may resolve against. The cited scope is applied by the reports pipeline at build time. A cited site is not a searchable archive, so the family does not list it (as `listed: false`, whatever `listed` says): no hub membership, no homepage card or totals, no footer link from other sites. Only `"cited"` is written; any other value reads as `"full"`. - -Default: `"full"` - ## `reports` The site's published reports, in display order: report ids (lowercase slugs, `[a-z0-9][a-z0-9-]*`), each a directory under `sites/<siteId>/reports/`. A report directory not named here is a draft and is not published. Invalid and repeated ids are dropped. Absent/empty = no reports. @@ -208,6 +202,12 @@ Default: [] ``` +## `search` + +Whether this site publishes its searchable corpus. Opt-OUT: absent/true = on, only an explicit `false` is written. Off = report-only: the site publishes only its `reports` and the moments they cite — no search, browse, transcripts or archives; `channels` is the pool its citations resolve against, and its `/corpus.json` says `"scope": "cited"`. A report-only site is never listed (as `listed: false`, whatever `listed` says). Legacy `publish: "cited"` reads as `false`. + +Default: `true` + ## `pwa` Whether this site ships an installable PWA (service worker + web manifest). Default false: a "dumb instance" that serves the CORS-enabled JSON federation contract but is not independently installable, so a visitor trusts only the hub PWA. Stored only when true. diff --git a/common/bin/compose-hub.test.ts b/common/bin/compose-hub.test.ts @@ -172,7 +172,7 @@ test("an unlisted site is in none of the hub's files; a listed one is in each", } }); -// Report sites: a cited site (`publish: "cited"`) publishes reports, not a +// Report sites: a report-only site (`search: false`) publishes reports, not a // searchable archive — no hub member, whatever `listed` says. A full site with // reports stays one. test("a cited site is no hub member; a full site with reports is", async () => { @@ -192,7 +192,7 @@ test("a cited site is no hub member; a full site with reports is", async () => { write("fixture-cited", { siteTitle: "Cited Fixture", siteUrl: "https://fixture-cited.example", - publish: "cited", + search: false, listed: true, }); console.log = () => {}; diff --git a/common/bin/compose-site.ts b/common/bin/compose-site.ts @@ -15,7 +15,7 @@ // Only the site's member channels are composed, so each deployed bundle holds // only that site's data. Checked-in static assets in public/ are left intact. // -// A CITED site (site.json `publish: "cited"`) publishes its reports and the +// A CITED site (site.json `search: false`) publishes its reports and the // moments they cite, and nothing else: its compose removes every corpus-shaped // entry (CORPUS_PUBLIC_ENTRIES) instead of composing it, then writes the // reports, site.json (no channels) and corpus.json (`site.scope: "cited"`) — diff --git a/common/controller/poolSummary.test.ts b/common/controller/poolSummary.test.ts @@ -32,7 +32,7 @@ test("channel-sites.json names listed sites only; a channel only an unlisted sit test("channel-sites.json leaves out a cited site", () => { const sites = [ parseSite("fixture-a", { channels: [{ slug: "shared" }] }), - parseSite("fixture-cited", { publish: "cited", channels: [{ slug: "shared" }, { slug: "own" }] }), + parseSite("fixture-cited", { search: false, channels: [{ slug: "shared" }, { slug: "own" }] }), ]; assert.deepEqual(channelSitesOf(sites), { shared: ["fixture-a"] }); }); diff --git a/common/lib/builtExport.test.ts b/common/lib/builtExport.test.ts @@ -376,8 +376,8 @@ test("a site configured cited with a full build is refused at deploy; a cited bu const full = bundle({ site: { siteId: "reports-site" }, corpus: { spec: 5, site: { id: "reports-site" } } }); const cited = citedOut(); try { - const site = { siteId: "reports-site", publish: "cited" }; - assert.match(builtScopeProblem(site, full.dir)!, /publishes only its reports \(publish: cited\), but .* holds a full build/); + const site = { siteId: "reports-site", search: false }; + assert.match(builtScopeProblem(site, full.dir)!, /publishes only its reports \(search: false\), but .* holds a full build/); assert.equal(deployAudienceProblem(site, full.dir), builtScopeProblem(site, full.dir)); assert.equal(builtScopeProblem(site, cited.dir), null); assert.equal(builtScopeProblem({ siteId: "reports-site" }, full.dir), null); diff --git a/common/lib/builtExport.ts b/common/lib/builtExport.ts @@ -15,6 +15,7 @@ import { existsSync, readdirSync, readFileSync, statSync, type Dirent } from "node:fs"; import path from "node:path"; +import { isCitedSite } from "./siteSchema"; /** * The site id of the build sitting in `outDir`, or null when there is no @@ -149,13 +150,13 @@ export function builtAudienceProblem(outDir: string): string | null { * logs or throws, or null. */ export function deployAudienceProblem( - site: { siteId: string; audience?: string; publish?: string }, + site: { siteId: string; audience?: string; search?: boolean }, outDir: string, ): string | null { return siteDeployProblem(site) ?? builtAudienceProblem(outDir) ?? builtScopeProblem(site, outDir); } -// ─── A CITED build (site.json `publish: "cited"`) ─── +// ─── A CITED build (a report-only site: site.json `search: false`) ─── // // A cited site publishes its reports and the moments they cite, and nothing // else (plans/report-sites.md). export/public is shared by every site's build @@ -301,18 +302,18 @@ export function citedBuildProblem(outDir: string): string | null { /** * Why `outDir` may not be deployed as `site` because of what it PUBLISHES, as - * one sentence, or null: a site configured cited (`publish: "cited"`) whose + * one sentence, or null: a site configured report-only (`search: false`) whose * build is not a cited one was built before the switch, and would ship the * whole corpus. Asked beside the audience refusals (deployAudienceProblem). */ export function builtScopeProblem( - site: { siteId: string; publish?: string }, + site: { siteId: string; search?: boolean }, outDir: string, ): string | null { - if (site.publish !== "cited" || !existsSync(path.join(outDir, "corpus.json"))) return null; + if (!isCitedSite(site) || !existsSync(path.join(outDir, "corpus.json"))) return null; if (isCitedBuild(outDir)) return null; return ( - `Site "${site.siteId}" publishes only its reports (publish: cited), but ${outDir} ` + + `Site "${site.siteId}" publishes only its reports (search: false), but ${outDir} ` + `holds a full build — build ${site.siteId} again, then deploy` ); } diff --git a/common/lib/corpus.ts b/common/lib/corpus.ts @@ -37,7 +37,7 @@ import { // // v5: reports — a site may publish cited reports, announced as `reports` // ({ index: "/reports/index.json" }, the report index the export's Reports -// pages read); and a CITED site (site.json `publish: "cited"`) publishes ONLY +// pages read); and a CITED site (site.json `search: false`) publishes ONLY // them, saying so as `site.scope: "cited"` with no channels and zero totals. // An older reader sees an empty corpus there, which is the safe reading: there // is no shard to fetch. diff --git a/common/lib/homepageSummary.test.ts b/common/lib/homepageSummary.test.ts @@ -305,10 +305,10 @@ test("an unlisted site is in no array and no total; a channel it shares is the l assert.equal(s.version, 6); }); -// Report sites: a cited site (`publish: "cited"`) is not a searchable archive, +// Report sites: a report-only site (`search: false`) is not a searchable archive, // so the homepage lists it no more than an unlisted one. test("a cited site is in no array and no total, exactly as an unlisted one", () => { - const cited = { ...site("zeta", ["q1", "a1"], "https://zeta.example"), publish: "cited" } as Site; + const cited = { ...site("zeta", ["q1", "a1"], "https://zeta.example"), search: false } as Site; const channelSites = { ...CHANNEL_SITES, a1: ["alpha", "zeta"], q1: ["zeta"] }; const own = [stat({ channelSlug: "q1", id: "q-1", uploadDate: "20251101", duration: 7200 })]; const s = buildHomepageSummary([...STATS, ...own], channelSites, [...SITES, cited], NOW); diff --git a/common/lib/homepageSummary.ts b/common/lib/homepageSummary.ts @@ -45,7 +45,7 @@ import { VIDEO_STATES, type VideoState } from "./availability"; // `availability` included. No field was added or removed; the number says the // totals' scope moved. // -// Still v6: a cited report site (site.json `publish: "cited"`) is left out +// Still v6: a cited report site (site.json `search: false`) is left out // exactly as an unlisted one (lib/siteSchema.ts isListedSite) — the same scope // rule, applied to a kind of site no summary had yet counted. export const HOMEPAGE_SUMMARY_VERSION = 6; diff --git a/common/lib/siteSchema.test.ts b/common/lib/siteSchema.test.ts @@ -66,7 +66,7 @@ test("empty, null, [] and a number all read as the defaults, every key emitted", assert.equal(want.pwa, false); assert.equal(want.listed, true); assert.deepEqual(want.relatedSites, []); - assert.equal(want.publish, "full"); + assert.equal(want.search, true); assert.deepEqual(want.reports, []); assert.equal(want.socialLinks, undefined); assert.ok("socialLinks" in want); @@ -239,7 +239,7 @@ function fixtures(): Array<[string, unknown]> { cloudflareProject: "p", siteUrl: "https://s.example//", listed: false, - publish: "cited", + search: false, reports: ["r-one", "r-one", "Bad", "", 7, "r-two"], relatedSites: [{ siteIds: ["x", "x", "BAD"] }, { label: " ", siteIds: [] }], pwa: true, @@ -316,18 +316,35 @@ test("listed: absent reads listed, only an explicit false unlists, and only fals assert.equal(getSite("s", paths).listed, true); }); -test("publish: absent reads full, only cited is kept, and only cited is written", () => { - for (const v of [undefined, "full", "FULL", "Cited", "corpus", true, 1, null]) { - const site = parseSite("s", v === undefined ? {} : { publish: v }); - assert.equal(site.publish, "full", String(v)); +test("search: absent/true is on, only false is report-only, and only false is written", () => { + for (const v of [undefined, true, "false", 0, null, "off"]) { + const site = parseSite("s", v === undefined ? {} : { search: v }); + assert.equal(site.search, true, String(v)); assert.equal(isCitedSite(site), false, String(v)); - assert.equal("publish" in siteToDisk(site), false, String(v)); + assert.equal("search" in siteToDisk(site), false, String(v)); } - const cited = parseSite("s", { publish: "cited" }); - assert.equal(cited.publish, "cited"); - assert.equal(isCitedSite(cited), true); - assert.equal(siteToDisk(cited).publish, "cited"); + const off = parseSite("s", { search: false }); + assert.equal(off.search, false); + assert.equal(isCitedSite(off), true); + assert.equal(siteToDisk(off).search, false); assert.equal(isCitedSite({}), false); + // A search-off site with no reports is still report-only (an empty index). + assert.equal(isCitedSite(parseSite("s", { search: false, reports: [] })), true); +}); + +test("legacy publish: \"cited\" reads as search off and is rewritten as search: false", () => { + const legacy = parseSite("s", { publish: "cited" }); + assert.equal(legacy.search, false); + assert.equal(isCitedSite(legacy), true); + assert.equal("publish" in legacy, false); + const disk = siteToDisk(legacy); + assert.equal(disk.search, false); + assert.equal("publish" in disk, false); + // Any other publish value is nothing; an explicit search wins over it. + for (const v of ["full", "Cited", true, 1]) { + assert.equal(parseSite("s", { publish: v }).search, true, String(v)); + } + assert.equal(parseSite("s", { publish: "cited", search: true }).search, true); }); test("reports: ordered slugs, invalid and repeated ids dropped, written only when non-empty", () => { @@ -347,19 +364,20 @@ test("reports: ordered slugs, invalid and repeated ids dropped, written only whe assert.deepEqual(siteToDisk({ ...site, reports: ["ok", "NOPE", "ok"] }).reports, ["ok"]); }); -test("writeSite → getSite round-trips a cited site and its reports; a full site writes neither key", async () => { +test("writeSite → getSite round-trips a report-only site and its reports; a full site writes neither key", async () => { const paths = scratchPaths(await mkdtemp(path.join(os.tmpdir(), "site-"))); - const cited = parseSite("s", { publish: "cited", reports: ["first", "second"] }); + const cited = parseSite("s", { search: false, reports: ["first", "second"] }); await writeSite(cited, paths); assert.deepEqual(getSite("s", paths), cited); const disk = JSON.parse(await readFile(siteConfigFile(paths, "s"), "utf8")); - assert.equal(disk.publish, "cited"); + assert.equal(disk.search, false); + assert.equal("publish" in disk, false); assert.deepEqual(disk.reports, ["first", "second"]); - await writeSite({ ...cited, publish: "full", reports: [] }, paths); + await writeSite({ ...cited, search: true, reports: [] }, paths); const plain = JSON.parse(await readFile(siteConfigFile(paths, "s"), "utf8")); - assert.equal("publish" in plain, false); + assert.equal("search" in plain, false); assert.equal("reports" in plain, false); - assert.equal(getSite("s", paths).publish, "full"); + assert.equal(getSite("s", paths).search, true); assert.deepEqual(getSite("s", paths).reports, []); }); @@ -376,14 +394,14 @@ test("isListedSite is the key's default; channelsOnlyOnUnlistedSites keeps a sha assert.deepEqual([...channelsOnlyOnUnlistedSites([sites[0]])], []); }); -test("a cited site is never listed, whatever `listed` says; a full site with reports is listed as before", () => { - assert.equal(isListedSite({ publish: "cited" }), false); - assert.equal(isListedSite({ publish: "cited", listed: true }), false); - assert.equal(isListedSite({ publish: "full" }), true); +test("a report-only site is never listed, whatever `listed` says; a full site with reports is listed as before", () => { + assert.equal(isListedSite({ search: false }), false); + assert.equal(isListedSite({ search: false, listed: true }), false); + assert.equal(isListedSite({ search: true }), true); assert.equal(isListedSite(parseSite("full-with-reports", { reports: ["demo-report"] })), true); const sites = [ parseSite("shown", { channels: [{ slug: "shared" }] }), - parseSite("reports", { publish: "cited", channels: [{ slug: "shared" }, { slug: "pool-only" }] }), + parseSite("reports", { search: false, channels: [{ slug: "shared" }, { slug: "pool-only" }] }), ]; // A cited site's channels are its citations' pool, not a published archive. assert.deepEqual([...channelsOnlyOnUnlistedSites(sites)], ["pool-only"]); @@ -419,6 +437,7 @@ test("writeSite → getSite round-trips, and the file holds only non-defaults", assert.equal("transcriptDownloads" in disk, false); assert.equal("pwa" in disk, false); assert.equal("listed" in disk, false); + assert.equal("search" in disk, false); assert.equal("publish" in disk, false); assert.equal("reports" in disk, false); }); @@ -431,7 +450,7 @@ test("patchSite applies a patch to the site on disk now, keeps every other key, const site = parseSite("s", { siteTitle: "Mine", archives: false, - publish: "cited", + search: false, reports: ["one", "two"], }); await writeSite(site, paths); diff --git a/common/lib/siteSchema.ts b/common/lib/siteSchema.ts @@ -90,22 +90,21 @@ export function isPrivateSite(site: Pick<Site, "audience">): boolean { return site.audience === "private"; } -// What a site publishes (report sites). "full" (the default, never written) is -// the searchable corpus every site has always been. "cited" publishes only the -// site's reports and the moments they cite; the reports pipeline applies that -// scope, and until it does a cited site builds as a full one. -export type SitePublish = "full" | "cited"; - -export const SITE_PUBLISH_SCOPES: readonly SitePublish[] = ["full", "cited"]; - -export function isSitePublish(v: unknown): v is SitePublish { - return v === "full" || v === "cited"; +// THE ONE PREDICATE for a report-only ("cited") site: search is off. Such a +// site publishes only its reports and the moments they cite (corpus.json +// `site.scope: "cited"`). Absent or true is a full, searchable site. +export function isCitedSite(site: Pick<Site, "search">): boolean { + return site.search === false; } -// THE ONE PREDICATE for a cited-only site. Absent or anything but "cited" reads -// as full. -export function isCitedSite(site: Pick<Site, "publish">): boolean { - return site.publish === "cited"; +// LEGACY: `publish: "cited"` (the pre-`search` report-only switch) reads as +// `search: false` when the file sets no `search`. siteToDisk never writes +// `publish`, so the next save migrates the file. +function migrateLegacyPublish(raw: unknown): unknown { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) return raw; + const r = raw as Record<string, unknown>; + if (r.publish !== "cited" || r.search !== undefined) return raw; + return { ...r, search: false }; } // A report id: a lowercase slug, the directory name under @@ -149,9 +148,9 @@ export type Site = { siteUrl?: string; listed?: boolean; audience?: SiteAudience; - publish?: SitePublish; reports?: string[]; relatedSites?: RelatedSiteGroup[]; + search?: boolean; pwa?: boolean; archives?: boolean; duplicates?: boolean; @@ -184,15 +183,15 @@ export const SITE_FIELD_DOCS: FieldDocs<Site> = { siteUrl: "Absolute public URL of this site's deployment, e.g. `https://jeralyzer.pages.dev` (trimmed, trailing slashes removed; anything not absolute http(s) is dropped). Drives the cross-site footer: a site with no siteUrl is omitted from every other site's list.", listed: - "Whether the family lists this site. Opt-OUT: absent/true = listed, only an explicit `false` is written. An unlisted site still builds and deploys as before, and its own pages are unchanged; it is left out of the homepage (cards, chart, `/stats`), the hub (members, federated search, `/corpus.json`, `/llms.txt`), every other site's footer, and the published `channel-sites.json` and pooled `stats/`. A channel only unlisted sites expose is in none of the family's public totals; a channel a listed site also exposes is credited to the listed one. A private site and a cited site (`publish: \"cited\"`) are never listed, whatever `listed` says.", + "Whether the family lists this site. Opt-OUT: absent/true = listed, only an explicit `false` is written. An unlisted site still builds and deploys as before, and its own pages are unchanged; it is left out of the homepage (cards, chart, `/stats`), the hub (members, federated search, `/corpus.json`, `/llms.txt`), every other site's footer, and the published `channel-sites.json` and pooled `stats/`. A channel only unlisted sites expose is in none of the family's public totals; a channel a listed site also exposes is credited to the listed one. A private site and a report-only site (`search: false`) are never listed, whatever `listed` says.", audience: 'Who this site is built for. `"public"` (the default; absent) or `"private"`: the operator\'s own reading copy, built on this machine and never deployed — every deploy path (Build & deploy, Deploy, `archilyzer deploy site`, Build & deploy all, docker/publish-site.sh) refuses it before any upload, while a build without a deploy still works. A private site is never listed (as `listed: false`, whatever `listed` says), publishes no `hubUrl`, and its `/corpus.json` says `"audience": "private"`. Content kept from the public — X posts while `social.x.visibility` is `"private"` — is built only into private sites. Only `"private"` is written.', - publish: - 'What this site publishes. `"full"` (the default; absent): the searchable corpus of its `channels`. `"cited"`: only the site\'s `reports` and the moments they cite — no search, no browse, no full transcripts, no archives; `channels` is then the pool its citations may resolve against. The cited scope is applied by the reports pipeline at build time. A cited site is not a searchable archive, so the family does not list it (as `listed: false`, whatever `listed` says): no hub membership, no homepage card or totals, no footer link from other sites. Only `"cited"` is written; any other value reads as `"full"`.', reports: "The site's published reports, in display order: report ids (lowercase slugs, `[a-z0-9][a-z0-9-]*`), each a directory under `sites/<siteId>/reports/`. A report directory not named here is a draft and is not published. Invalid and repeated ids are dropped. Absent/empty = no reports.", relatedSites: "Pulls specific siblings to the front of the footer's cross-site list, in named groups. Siblings not named here fall into a trailing \"Other sites\" group. Absent/empty = one flat list of every sibling.", + search: + "Whether this site publishes its searchable corpus. Opt-OUT: absent/true = on, only an explicit `false` is written. Off = report-only: the site publishes only its `reports` and the moments they cite — no search, browse, transcripts or archives; `channels` is the pool its citations resolve against, and its `/corpus.json` says `\"scope\": \"cited\"`. A report-only site is never listed (as `listed: false`, whatever `listed` says). Legacy `publish: \"cited\"` reads as `false`.", pwa: "Whether this site ships an installable PWA (service worker + web manifest). Default false: a \"dumb instance\" that serves the CORS-enabled JSON federation contract but is not independently installable, so a visitor trusts only the hub PWA. Stored only when true.", archives: @@ -225,11 +224,11 @@ export function isValidSiteId(id: unknown): id is string { // // A PRIVATE site (`audience: "private"`) is never listed, whatever `listed` // says: it is never deployed, so there is nothing at its URL to list. Nor is a -// CITED site (`publish: "cited"`): it publishes reports and the moments they +// report-only site (`search: false`): it publishes reports and the moments they // cite, not a searchable archive, so it is no hub member (federated search // would find no channel there), no homepage card and in no family total. A // full site with reports is listed as before. -export function isListedSite(site: Pick<Site, "listed" | "audience" | "publish">): boolean { +export function isListedSite(site: Pick<Site, "listed" | "audience" | "search">): boolean { return site.listed !== false && !isPrivateSite(site) && !isCitedSite(site); } @@ -239,7 +238,7 @@ export function isListedSite(site: Pick<Site, "listed" | "audience" | "publish"> // and a channel no site exposes (pool-only) is not here either — the family's // instance-wide totals have always counted it. export function channelsOnlyOnUnlistedSites( - sites: readonly Pick<Site, "listed" | "audience" | "publish" | "channels">[], + sites: readonly Pick<Site, "listed" | "audience" | "search" | "channels">[], ): Set<string> { const onListed = new Set<string>(); const onUnlisted = new Set<string>(); @@ -364,12 +363,10 @@ export const siteFieldsSchema = z.object({ audience: settingsField((v): SiteAudience | undefined => v === "private" ? "private" : undefined, ).describe(d.audience), - // Only "cited" is kept; absent (and anything else) is the full default. - publish: settingsField((v): SitePublish => - v === "cited" ? "cited" : "full", - ).describe(d.publish), reports: settingsField(parseSiteReports).describe(d.reports), relatedSites: settingsField(parseRelatedSites).describe(d.relatedSites), + // Opt-out: only an explicit false is report-only. Absent/true stays on. + search: settingsField((v): boolean => v !== false).describe(d.search), pwa: settingsField((v): boolean => v === true).describe(d.pwa), // Opt-out: only an explicit false disables. Absent/true stays on. archives: settingsField((v): boolean => v !== false).describe(d.archives), @@ -389,7 +386,7 @@ export const siteFieldsSchema = z.object({ // output lacks `socialLinks`, `accent`, … on a file that does not spell them — // where parseSite has always emitted them, present and `undefined`. Rebuilding // from SITE_KEYS keeps that shape (and the key order) exactly. -export const siteSchema = siteFieldsSchema.transform((s): Site => { +export const siteSchema = z.preprocess(migrateLegacyPublish, siteFieldsSchema).transform((s): Site => { const groups = s.groups; const resolved: Partial<Record<keyof Site, unknown>> = { ...s, @@ -463,10 +460,10 @@ export function siteToDisk(site: Site): Site { ...(site.listed === false ? { listed: false } : {}), // Public is the default: only the private audience is persisted. ...(isPrivateSite(site) ? { audience: "private" as const } : {}), - // Full is the default: only the cited scope is persisted. - ...(isCitedSite(site) ? { publish: "cited" as const } : {}), ...(reports.length > 0 ? { reports } : {}), ...(relatedSites.length > 0 ? { relatedSites } : {}), + // Search is the default: only report-only is persisted (never `publish`). + ...(isCitedSite(site) ? { search: false } : {}), ...(site.pwa ? { pwa: true } : {}), // Persist only the non-default: archives is on unless explicitly disabled. ...(site.archives === false ? { archives: false } : {}), diff --git a/common/publish/composeReports.test.ts b/common/publish/composeReports.test.ts @@ -278,7 +278,7 @@ async function seedCorpus() { }, ]); - seedSite("cited", { publish: "cited", reports: [REPORT] }); + seedSite("cited", { search: false, reports: [REPORT] }); seedSite("full", { reports: [REPORT] }); seedSite("plain", {}, false); await buildIndex({ paths, onLog: () => {} }); diff --git a/common/publish/reportMedia.test.ts b/common/publish/reportMedia.test.ts @@ -108,7 +108,7 @@ for (const id of ["111", "222"]) { writeJson(path.join(paths.sitesDir, SITE, "site.json"), { title: "Demo", channels: [{ slug: CH }, { slug: X }], - publish: "cited", + search: false, reports: ["r1", "r2", "r-gone"], }); writeJson( diff --git a/editor/app/sites/[siteId]/reports/page.tsx b/editor/app/sites/[siteId]/reports/page.tsx @@ -58,8 +58,8 @@ export default async function SiteReportsPage({ <p className="text-sm"> <span className="font-medium">Publishes:</span>{" "} {cited - ? "Cited moments only — its reports and the moments they cite" - : "Full archive — the searchable corpus of its channels"}{" "} + ? "Reports only — search is off" + : "Full archive — search is on"}{" "} <span className="text-muted-foreground"> (changed on{" "} <Link href={`/sites/${siteId}`} className="underline"> @@ -70,8 +70,8 @@ export default async function SiteReportsPage({ </p> {cited && publishedCount === 0 && ( <p role="status" className="text-sm text-warning"> - This site publishes only cited moments and has no published report: - its build would have nothing to show. + Search is off and no report is published: the site is an empty + report index. </p> )} </section> diff --git a/editor/app/sites/actions.ts b/editor/app/sites/actions.ts @@ -118,8 +118,8 @@ export async function saveSiteAction( return { ok: false, error: `Unknown audience "${audienceRaw}".`, values }; } const isPrivate = audienceRaw === "private"; - // What the site publishes (report sites): only "cited" is kept. The report - // list is not on this form; the stored one is carried over. + // Search (opt-out; off = report-only) and the report list, which is not on + // this form: the stored one is carried over. let storedReports: Pick<Site, "reports"> | undefined; try { storedReports = getSite(siteId, getPaths()); @@ -127,7 +127,6 @@ export async function saveSiteAction( storedReports = undefined; } const reportKeys = reportSiteKeysFromForm(formData, storedReports); - if (!reportKeys.ok) return { ok: false, error: reportKeys.error, values }; // Hub parent (per-site override of the family default) + PWA opt-in. const hubUrlRaw = String(formData.get("hubUrl") ?? "").trim(); @@ -266,7 +265,7 @@ export async function saveSiteAction( // The Site is rebuilt from the form: a key missing here is dropped on save. ...(listed ? {} : { listed: false }), ...(isPrivate ? { audience: "private" as const } : {}), - ...reportKeys.keys, + ...reportKeys, ...(hubUrl ? { hubUrl } : {}), ...(pwa ? { pwa: true } : {}), ...(archives ? {} : { archives: false }), diff --git a/editor/app/sites/components/SiteForm.tsx b/editor/app/sites/components/SiteForm.tsx @@ -361,26 +361,18 @@ export function SiteForm({ initial, channels, allSites, isNew }: Props) { into while Settings keeps X posts private. </span> </label> - <label className="flex flex-col gap-1 text-sm"> - <span className="font-medium">Publish</span> - <SeededSelect - state={state} - name="publish" - aria-label="Publish" - initial={initial.publish === "cited" ? "cited" : "full"} - className="w-fit rounded border border-border bg-card px-2 py-1 text-sm" - > - <option value="full">Full archive — the searchable corpus of its channels</option> - <option value="cited">Cited moments only — its reports and the moments they cite</option> - </SeededSelect> - <span className="text-xs text-muted-foreground"> - A cited-only site publishes no search, browse, full transcripts or - archives; its channels are the pool its reports' citations resolve - against. The reports pipeline applies this scope when the site is built. - It is never listed on the homepage or the hub: it is not a searchable - archive. - </span> + <label className="flex items-center gap-2 text-sm"> + <input + type="checkbox" + name="search" + defaultChecked={seedChecked(state, "search", initial.search !== false)} + className="accent-brand" + /> + Search </label> + <p className="-mt-2 text-xs text-muted-foreground"> + Off: the site is its reports only. + </p> <div className="flex flex-col gap-1 text-sm"> <span className="font-medium">Reports</span> {initial.reports && initial.reports.length > 0 ? ( diff --git a/editor/app/sites/lib/reportSiteKeys.test.ts b/editor/app/sites/lib/reportSiteKeys.test.ts @@ -10,36 +10,22 @@ function form(entries: Record<string, string>): FormData { return fd; } -test("full (or no field) writes no publish key; cited is kept", () => { - assert.deepEqual(reportSiteKeysFromForm(form({ publish: "full" }), undefined), { - ok: true, - keys: {}, - }); - assert.deepEqual(reportSiteKeysFromForm(form({}), undefined), { ok: true, keys: {} }); - assert.deepEqual(reportSiteKeysFromForm(form({ publish: "cited" }), undefined), { - ok: true, - keys: { publish: "cited" }, - }); -}); - -test("an unknown publish scope is refused, not read as full", () => { - for (const v of ["Cited", "corpus", ""]) { - const r = reportSiteKeysFromForm(form({ publish: v }), undefined); - assert.equal(r.ok, false, v); - if (!r.ok) assert.match(r.error, /Unknown publish scope/); - } +test("search checked writes no key; unchecked (no field) writes search: false", () => { + assert.deepEqual(reportSiteKeysFromForm(form({ search: "on" }), undefined), {}); + assert.deepEqual(reportSiteKeysFromForm(form({}), undefined), { search: false }); + // The old select's field is not read: only the checkbox decides. + assert.deepEqual(reportSiteKeysFromForm(form({ search: "on", publish: "cited" }), undefined), {}); }); test("the stored report list is carried over in order; the form cannot set it", () => { const stored = { reports: ["second", "first"] }; assert.deepEqual( - reportSiteKeysFromForm(form({ publish: "full", reports: "injected" }), stored), - { ok: true, keys: { reports: ["second", "first"] } }, + reportSiteKeysFromForm(form({ search: "on", reports: "injected" }), stored), + { reports: ["second", "first"] }, ); // An empty or missing stored list writes no key; a bad stored id is dropped. - assert.deepEqual(reportSiteKeysFromForm(form({}), { reports: [] }), { ok: true, keys: {} }); - assert.deepEqual(reportSiteKeysFromForm(form({}), { reports: ["ok", "NOT OK"] }), { - ok: true, - keys: { reports: ["ok"] }, + assert.deepEqual(reportSiteKeysFromForm(form({ search: "on" }), { reports: [] }), {}); + assert.deepEqual(reportSiteKeysFromForm(form({ search: "on" }), { reports: ["ok", "NOT OK"] }), { + reports: ["ok"], }); }); diff --git a/editor/app/sites/lib/reportSiteKeys.ts b/editor/app/sites/lib/reportSiteKeys.ts @@ -1,35 +1,23 @@ -// The site form's two report-site keys (site.json `publish` and `reports`), +// The site form's two report-site keys (site.json `search` and `reports`), // pure so the parse is tested without a server (saveSiteAction calls it). // -// `publish` is the form's select; only "cited" is kept, as the file writes it. +// `search` is the form's checkbox, opt-out like `archives`: an unchecked box +// sends no field and is written as `search: false` (report-only). A legacy +// `publish: "cited"` read from the file is never written back. // `reports` is NOT edited by this form (the site's Reports tab manages it): the // Site is rebuilt from the form on save, so the list already stored is carried // over, and saving any other field never drops a published report. -import { - isSitePublish, - parseSiteReports, - type Site, -} from "yt-dlp-transcript-common/lib/site"; - -export type ReportSiteKeys = - | { ok: true; keys: Pick<Site, "publish" | "reports"> } - | { ok: false; error: string }; +import { parseSiteReports, type Site } from "yt-dlp-transcript-common/lib/site"; export function reportSiteKeysFromForm( formData: FormData, stored: Pick<Site, "reports"> | undefined, -): ReportSiteKeys { - const raw = String(formData.get("publish") ?? "full"); - if (!isSitePublish(raw)) { - return { ok: false, error: `Unknown publish scope "${raw}".` }; - } +): Pick<Site, "search" | "reports"> { + const search = formData.get("search") === "on"; const reports = parseSiteReports(stored?.reports); return { - ok: true, - keys: { - ...(raw === "cited" ? { publish: "cited" as const } : {}), - ...(reports.length > 0 ? { reports } : {}), - }, + ...(search ? {} : { search: false }), + ...(reports.length > 0 ? { reports } : {}), }; } diff --git a/editor/e2e/helpers.ts b/editor/e2e/helpers.ts @@ -153,6 +153,9 @@ export async function writeSite( : {}), ...(site.siteUrl ? { siteUrl: site.siteUrl } : {}), ...(site.listed === false ? { listed: false } : {}), + ...(site.search === false ? { search: false } : {}), + // The legacy report-only key, written as given (siteSchema migrates it). + ...(site.publish !== undefined ? { publish: site.publish } : {}), ...(site.relatedSites ? { relatedSites: site.relatedSites } : {}), }; await writeFile( diff --git a/editor/e2e/sites-crud.spec.ts b/editor/e2e/sites-crud.spec.ts @@ -345,6 +345,50 @@ test("archives + per-video transcript downloads opt-outs round-trip", async ({ await expect(downloads).toBeChecked(); }); +test("the search switch round-trips; a legacy publish: cited reads as off and saves as search: false", async ({ + page, +}) => { + await resetData("empty"); + await writeSite("reportonly", { siteTitle: "Report Only", publish: "cited" }); + + type SearchSiteFile = { search?: boolean; publish?: string }; + const file = "test-transcripts/sites/reportonly/site.json"; + const search = page.getByRole("checkbox", { name: "Search", exact: true }); + const save = async () => { + await page.getByRole("button", { name: /save site/i }).click(); + await expect( + page.getByRole("status").filter({ hasText: "Saved" }), + ).toBeVisible(); + }; + + // The legacy key reads as search off; saving rewrites it as search: false. + await page.goto("/sites/reportonly"); + await expect(search).not.toBeChecked(); + await save(); + await expect(async () => { + const site = await readJson<SearchSiteFile>(file); + expect(site.search).toBe(false); + expect("publish" in site).toBe(false); + }).toPass({ timeout: 10_000 }); + + await page.goto("/sites/reportonly/reports"); + await expect(page.getByText("Reports only — search is off")).toBeVisible(); + + // Ticking it back on removes the key — only the non-default is persisted. + await page.goto("/sites/reportonly"); + await search.check(); + await save(); + await expect(async () => { + const site = await readJson<SearchSiteFile>(file); + expect("search" in site).toBe(false); + }).toPass({ timeout: 10_000 }); + + await page.goto("/sites/reportonly"); + await expect(search).toBeChecked(); + await page.goto("/sites/reportonly/reports"); + await expect(page.getByText("Full archive — search is on")).toBeVisible(); +}); + test("the listed opt-out round-trips, and a save of another field keeps it", async ({ page, }) => { diff --git a/export/app/(workspace)/layout.tsx b/export/app/(workspace)/layout.tsx @@ -10,7 +10,7 @@ import SiteWorkspace from "./SiteWorkspace"; // // Hub mode keeps its own self-contained pages (<HubHome/>, <AskHub/>), so here // it's a straight pass-through — the fork is preserved exactly as before. So -// is a CITED site (site.json `publish: "cited"`): it has no corpus to search, +// is a CITED site (site.json `search: false`): it has no corpus to search, // and its home is the report index. export default function WorkspaceLayout({ children, diff --git a/export/app/components/Footer.tsx b/export/app/components/Footer.tsx @@ -19,7 +19,7 @@ import { isCitedSite } from "yt-dlp-transcript-common/lib/siteSchema"; export default function Footer() { const site = currentSite(); - // A cited site (site.json `publish: "cited"`) has no archives to download + // A cited site (site.json `search: false`) has no archives to download // and nothing for an AI to read; its footer keeps the rest. const cited = instanceMode() === "site" && isCitedSite(site); const showArchives = !cited && hasArchives(); diff --git a/export/app/lib/nav.ts b/export/app/lib/nav.ts @@ -9,7 +9,7 @@ export type NavLink = { href: string; label: string; external?: boolean }; // site, so a plain <a> in the same tab, not a client navigation. Reports // shows when this build has a report (public/reports/index.json). // -// A CITED site (site.json `publish: "cited"`) publishes only its reports and +// A CITED site (site.json `search: false`) publishes only its reports and // their moments: no search, no chat, no downloads or duplicates, and nothing // for an AI to read — its nav is Reports alone. export function headerNavLinks(has: { diff --git a/export/app/lib/reports.test.ts b/export/app/lib/reports.test.ts @@ -22,7 +22,7 @@ fs.mkdirSync(publicDir, { recursive: true }); fs.mkdirSync(path.join(sitesDir, "demo-cited"), { recursive: true }); fs.writeFileSync( path.join(sitesDir, "demo-cited", "site.json"), - JSON.stringify({ siteTitle: "Demo Reports", publish: "cited", reports: ["demo-factcheck"] }), + JSON.stringify({ siteTitle: "Demo Reports", search: false, reports: ["demo-factcheck"] }), ); process.env.EXPORT_PUBLIC_DIR = publicDir; process.env.TRANSCRIPTS_DIR = path.join(root, "transcripts"); diff --git a/export/e2e-report/fixtures/sites/reportsite/site.json b/export/e2e-report/fixtures/sites/reportsite/site.json @@ -4,6 +4,6 @@ "siteDescription": "A cited fixture site: one fact-check and the moments it cites.", "headerTitle": "Demo Reports", "siteUrl": "https://reports.example.org", - "publish": "cited", + "search": false, "reports": ["demo-factcheck"] } diff --git a/mcp/README.md b/mcp/README.md @@ -331,7 +331,8 @@ verdict tally, then each claim with its verdict, its findings and, per citation, the quote, the original (the platform at the cited second, the post, the document) and the site's moment page (`/m/<channel>/<id>/<start>-<end>/`). -A **cited-only** site (`corpus.json` `site.scope: "cited"`) publishes its reports +A **cited-only** site (`corpus.json` `site.scope: "cited"`; built from a site with +search off, `site.json` `search: false`) publishes its reports and the moments they cite and nothing else: no channels, no transcripts, nothing to search. `list_channels`, `list_sources` and `resolve_source` say "cited-only site: N report(s)" there rather than reporting an empty corpus. Reports are per