commit 521a327e919f623be547ce529c4aa894f2a6436d
parent dd12b36fc97903cc72311f16efaa21942b57c9e0
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 5 Oct 2026 02:16:13 -0400
Merge report-r1-site-keys (site.json publish: full | cited and an ordered reports list; the site form's Publish control)
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
8 files changed, 259 insertions(+), 0 deletions(-)
diff --git a/SITE.md b/SITE.md
@@ -25,6 +25,8 @@ Regenerate this file with `pnpm --filter yt-dlp-transcript-common exec tsx bin/f
| [`siteUrl`](#siteurl) | absent |
| [`listed`](#listed) | `true` |
| [`audience`](#audience) | absent |
+| [`publish`](#publish) | `"full"` |
+| [`reports`](#reports) | `[]` |
| [`relatedSites`](#relatedsites) | `[]` |
| [`pwa`](#pwa) | `false` |
| [`archives`](#archives) | `true` |
@@ -171,6 +173,22 @@ Who this site is built for. `"public"` (the default; absent) or `"private"`: the
Default: absent
+## `publish`
+
+What this site publishes. `"full"` (the default; absent): the searchable corpus of its `channels`. `"cited"`: only the site's `reports` and the moments they cite — no search, no browse, no full transcripts, no archives; `channels` is then the pool its citations may resolve against. The cited scope is applied by the reports pipeline at build time. Only `"cited"` is written; any other value reads as `"full"`.
+
+Default: `"full"`
+
+## `reports`
+
+The site's published reports, in display order: report ids (lowercase slugs, `[a-z0-9][a-z0-9-]*`), each a directory under `sites/<siteId>/reports/`. A report directory not named here is a draft and is not published. Invalid and repeated ids are dropped. Absent/empty = no reports.
+
+Default:
+
+```json
+[]
+```
+
## `relatedSites`
Pulls specific siblings to the front of the footer's cross-site list, in named groups. Siblings not named here fall into a trailing "Other sites" group. Absent/empty = one flat list of every sibling.
diff --git a/common/lib/siteSchema.test.ts b/common/lib/siteSchema.test.ts
@@ -10,8 +10,10 @@ import {
SITE_FIELD_DOCS,
SITE_KEYS,
channelsOnlyOnUnlistedSites,
+ isCitedSite,
isListedSite,
parseSite,
+ parseSiteReports,
siteFieldsSchema,
siteToDisk,
type Site,
@@ -64,6 +66,8 @@ test("empty, null, [] and a number all read as the defaults, every key emitted",
assert.equal(want.pwa, false);
assert.equal(want.listed, true);
assert.deepEqual(want.relatedSites, []);
+ assert.equal(want.publish, "full");
+ assert.deepEqual(want.reports, []);
assert.equal(want.socialLinks, undefined);
assert.ok("socialLinks" in want);
for (const raw of [null, [], 3, "x", undefined]) {
@@ -235,6 +239,8 @@ function fixtures(): Array<[string, unknown]> {
cloudflareProject: "p",
siteUrl: "https://s.example//",
listed: false,
+ publish: "cited",
+ reports: ["r-one", "r-one", "Bad", "", 7, "r-two"],
relatedSites: [{ siteIds: ["x", "x", "BAD"] }, { label: " ", siteIds: [] }],
pwa: true,
archives: false,
@@ -310,6 +316,53 @@ test("listed: absent reads listed, only an explicit false unlists, and only fals
assert.equal(getSite("s", paths).listed, true);
});
+test("publish: absent reads full, only cited is kept, and only cited is written", () => {
+ for (const v of [undefined, "full", "FULL", "Cited", "corpus", true, 1, null]) {
+ const site = parseSite("s", v === undefined ? {} : { publish: v });
+ assert.equal(site.publish, "full", String(v));
+ assert.equal(isCitedSite(site), false, String(v));
+ assert.equal("publish" in siteToDisk(site), false, String(v));
+ }
+ const cited = parseSite("s", { publish: "cited" });
+ assert.equal(cited.publish, "cited");
+ assert.equal(isCitedSite(cited), true);
+ assert.equal(siteToDisk(cited).publish, "cited");
+ assert.equal(isCitedSite({}), false);
+});
+
+test("reports: ordered slugs, invalid and repeated ids dropped, written only when non-empty", () => {
+ assert.deepEqual(
+ parseSiteReports(["b-two", "a-one", "b-two", "Upper", "-lead", "has space", "", 3, null, "c3"]),
+ ["b-two", "a-one", "c3"],
+ );
+ for (const v of [undefined, null, "a-one", { a: 1 }, 5]) {
+ assert.deepEqual(parseSite("s", v === undefined ? {} : { reports: v }).reports, [], String(v));
+ }
+ const site = parseSite("s", { reports: ["z-last", "a-first", "a-first"] });
+ assert.deepEqual(site.reports, ["z-last", "a-first"]);
+ assert.deepEqual(siteToDisk(site).reports, ["z-last", "a-first"]);
+ assert.equal("reports" in siteToDisk(parseSite("s", { reports: [] })), false);
+ assert.equal("reports" in siteToDisk(parseSite("s", { reports: ["BAD"] })), false);
+ // siteToDisk re-parses a caller's list: a bad id handed to the writer is not persisted.
+ assert.deepEqual(siteToDisk({ ...site, reports: ["ok", "NOPE", "ok"] }).reports, ["ok"]);
+});
+
+test("writeSite → getSite round-trips a cited site and its reports; a full site writes neither key", async () => {
+ const paths = scratchPaths(await mkdtemp(path.join(os.tmpdir(), "site-")));
+ const cited = parseSite("s", { publish: "cited", reports: ["first", "second"] });
+ await writeSite(cited, paths);
+ assert.deepEqual(getSite("s", paths), cited);
+ const disk = JSON.parse(await readFile(siteConfigFile(paths, "s"), "utf8"));
+ assert.equal(disk.publish, "cited");
+ assert.deepEqual(disk.reports, ["first", "second"]);
+ await writeSite({ ...cited, publish: "full", reports: [] }, paths);
+ const plain = JSON.parse(await readFile(siteConfigFile(paths, "s"), "utf8"));
+ assert.equal("publish" in plain, false);
+ assert.equal("reports" in plain, false);
+ assert.equal(getSite("s", paths).publish, "full");
+ assert.deepEqual(getSite("s", paths).reports, []);
+});
+
test("isListedSite is the key's default; channelsOnlyOnUnlistedSites keeps a shared channel with the listed site", () => {
assert.equal(isListedSite({}), true);
assert.equal(isListedSite({ listed: true }), true);
@@ -353,4 +406,6 @@ test("writeSite → getSite round-trips, and the file holds only non-defaults",
assert.equal("transcriptDownloads" in disk, false);
assert.equal("pwa" in disk, false);
assert.equal("listed" in disk, false);
+ assert.equal("publish" in disk, false);
+ assert.equal("reports" in disk, false);
});
diff --git a/common/lib/siteSchema.ts b/common/lib/siteSchema.ts
@@ -89,6 +89,46 @@ export function isPrivateSite(site: Pick<Site, "audience">): boolean {
return site.audience === "private";
}
+// What a site publishes (report sites). "full" (the default, never written) is
+// the searchable corpus every site has always been. "cited" publishes only the
+// site's reports and the moments they cite; the reports pipeline applies that
+// scope, and until it does a cited site builds as a full one.
+export type SitePublish = "full" | "cited";
+
+export const SITE_PUBLISH_SCOPES: readonly SitePublish[] = ["full", "cited"];
+
+export function isSitePublish(v: unknown): v is SitePublish {
+ return v === "full" || v === "cited";
+}
+
+// THE ONE PREDICATE for a cited-only site. Absent or anything but "cited" reads
+// as full.
+export function isCitedSite(site: Pick<Site, "publish">): boolean {
+ return site.publish === "cited";
+}
+
+// A report id: a lowercase slug, the directory name under
+// `sites/<siteId>/reports/`. The same grammar as a site id.
+export const REPORT_ID_RE = /^[a-z0-9][a-z0-9-]*$/;
+
+export function isValidReportId(id: unknown): id is string {
+ return typeof id === "string" && REPORT_ID_RE.test(id);
+}
+
+// Parse the published report list: valid ids, first occurrence kept, order
+// preserved. Anything else (a non-array, a bad or repeated id) is dropped.
+export function parseSiteReports(input: unknown): string[] {
+ if (!Array.isArray(input)) return [];
+ const out: string[] = [];
+ const seen = new Set<string>();
+ for (const id of input) {
+ if (!isValidReportId(id) || seen.has(id)) continue;
+ seen.add(id);
+ out.push(id);
+ }
+ return out;
+}
+
// A Site is a selection + presentation layer over the single global channel
// pool. Each field is documented in SITE_FIELD_DOCS below.
export type Site = {
@@ -107,6 +147,8 @@ export type Site = {
siteUrl?: string;
listed?: boolean;
audience?: SiteAudience;
+ publish?: SitePublish;
+ reports?: string[];
relatedSites?: RelatedSiteGroup[];
pwa?: boolean;
archives?: boolean;
@@ -143,6 +185,10 @@ export const SITE_FIELD_DOCS: FieldDocs<Site> = {
"Whether the family lists this site. Opt-OUT: absent/true = listed, only an explicit `false` is written. An unlisted site still builds and deploys as before, and its own pages are unchanged; it is left out of the homepage (cards, chart, `/stats`), the hub (members, federated search, `/corpus.json`, `/llms.txt`), every other site's footer, and the published `channel-sites.json` and pooled `stats/`. A channel only unlisted sites expose is in none of the family's public totals; a channel a listed site also exposes is credited to the listed one.",
audience:
'Who this site is built for. `"public"` (the default; absent) or `"private"`: the operator\'s own reading copy, built on this machine and never deployed — every deploy path (Build & deploy, Deploy, `archilyzer deploy site`, Build & deploy all, docker/publish-site.sh) refuses it before any upload, while a build without a deploy still works. A private site is never listed (as `listed: false`, whatever `listed` says), publishes no `hubUrl`, and its `/corpus.json` says `"audience": "private"`. Content kept from the public — X posts while `social.x.visibility` is `"private"` — is built only into private sites. Only `"private"` is written.',
+ publish:
+ 'What this site publishes. `"full"` (the default; absent): the searchable corpus of its `channels`. `"cited"`: only the site\'s `reports` and the moments they cite — no search, no browse, no full transcripts, no archives; `channels` is then the pool its citations may resolve against. The cited scope is applied by the reports pipeline at build time. Only `"cited"` is written; any other value reads as `"full"`.',
+ reports:
+ "The site's published reports, in display order: report ids (lowercase slugs, `[a-z0-9][a-z0-9-]*`), each a directory under `sites/<siteId>/reports/`. A report directory not named here is a draft and is not published. Invalid and repeated ids are dropped. Absent/empty = no reports.",
relatedSites:
"Pulls specific siblings to the front of the footer's cross-site list, in named groups. Siblings not named here fall into a trailing \"Other sites\" group. Absent/empty = one flat list of every sibling.",
pwa:
@@ -312,6 +358,11 @@ export const siteFieldsSchema = z.object({
audience: settingsField((v): SiteAudience | undefined =>
v === "private" ? "private" : undefined,
).describe(d.audience),
+ // Only "cited" is kept; absent (and anything else) is the full default.
+ publish: settingsField((v): SitePublish =>
+ v === "cited" ? "cited" : "full",
+ ).describe(d.publish),
+ reports: settingsField(parseSiteReports).describe(d.reports),
relatedSites: settingsField(parseRelatedSites).describe(d.relatedSites),
pwa: settingsField((v): boolean => v === true).describe(d.pwa),
// Opt-out: only an explicit false disables. Absent/true stays on.
@@ -378,6 +429,7 @@ export function siteToDisk(site: Site): Site {
(c) => !c.groupId || groups.some((g) => g.id === c.groupId),
);
const relatedSites = parseRelatedSites(site.relatedSites);
+ const reports = parseSiteReports(site.reports);
const accent = parseAccentSetting(site.accent);
const wordmarkLead = wordmarkLeadFor(site.headerTitle, site.wordmarkLead);
const siteUrl = parseSiteUrl(site.siteUrl);
@@ -405,6 +457,9 @@ export function siteToDisk(site: Site): Site {
...(site.listed === false ? { listed: false } : {}),
// Public is the default: only the private audience is persisted.
...(isPrivateSite(site) ? { audience: "private" as const } : {}),
+ // Full is the default: only the cited scope is persisted.
+ ...(isCitedSite(site) ? { publish: "cited" as const } : {}),
+ ...(reports.length > 0 ? { reports } : {}),
...(relatedSites.length > 0 ? { relatedSites } : {}),
...(site.pwa ? { pwa: true } : {}),
// Persist only the non-default: archives is on unless explicitly disabled.
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,7 @@
# Changelog
## [Unreleased]
+- **A site can say what it publishes, and which reports.** `site.json` takes `publish` — `"full"`, the searchable corpus every site has been (the default, never written), or `"cited"`, only the site's reports and the moments they cite — and `reports`, the ordered ids of its published reports (each a slug; invalid and repeated ids are dropped). The site form has a Publish control and lists the site's reports read-only; saving the form keeps the stored list. A cited site still builds as a full one until the reports pipeline applies the scope. SITE.md documents both keys.
- **A long report video no longer runs out of memory while its clips are crossfaded.** `build-video.mjs` used to join every segment of a cut in one ffmpeg command, which grows with the number of segments: a cut of a few hundred clips could use more memory than the machine had and be stopped. Past 24 segments the build now crossfades them in batches of consecutive segments, each into a file under `out/<variant>/xfade-batches/`, then crossfades those files together with the same transition and lays the on-screen deck, the rail and the dips over them. Every transition, the deck's schedule and the chapters land on the same frames as before, at the cost of one more video encode on such a cut. The batch size is `render.xfadeBatch` in the manifest or `REPORT_VIDEO_XFADE_BATCH` in the environment (which wins); `0` never batches. A batch file is reused while its segments, their holds and moves and the encode settings are unchanged, so a `--chrome-only` run that moves no footage redoes only the last pass.
- **Auto-download no longer tries a video the metadata scan already found members-only or private.** The scan records why it could not read a video, but only a failed download used to take a video out of the auto-download queue, so each members-only video the scan had found was still downloaded once: four yt-dlp requests, two with browser cookies, about 30 seconds each. A members-only or private answer from the scan now keeps the video out of the queue and counts it under the channel's members-only or private exclusions, and it stays in **Needs cookies** for a manual cookie run. A scan that reads the video later lifts this. A video the scan saw only as "Video unavailable" is still tried, since YouTube gives that answer when it is throttling too.
- **X post fetches stop on Drain, and an account with no posts is not searched.** Draining a `fetch-posts` job used to do nothing until gallery-dl finished its whole run. Now the timeline fetch stops at the next page boundary (at once when gallery-dl is between pages or waiting out a rate limit), the older-posts walk stops its current window's search at once and never starts the 45–120 second pause between windows, and both keep their resume point: the job ends done, not failed, and the log says "Drained; the next run resumes …". An older-posts walk is refused when nothing is archived and the last timeline fetch finished having read no posts, since it would only repeat empty searches; `"force": true` (`--force` on `archilyzer posts fetch`) walks anyway. A walk with nothing archived that finds nothing ends after two empty three-month windows instead of four, and records why; a walk that has posts keeps the year-of-empty-windows rule. Capture-posts already stopped between posts on Drain.
diff --git a/editor/app/sites/actions.ts b/editor/app/sites/actions.ts
@@ -31,6 +31,7 @@ import {
import { migrateToSites } from "yt-dlp-transcript-common/controller/migrateToSites";
import { socialLinksForSave } from "yt-dlp-transcript-common/lib/socialLinks";
import { formValues, type FormState } from "../lib/formState";
+import { reportSiteKeysFromForm } from "./lib/reportSiteKeys";
// A failure carries what was submitted, so SiteForm keeps what was typed
// (lib/formState.ts).
@@ -117,6 +118,16 @@ export async function saveSiteAction(
return { ok: false, error: `Unknown audience "${audienceRaw}".`, values };
}
const isPrivate = audienceRaw === "private";
+ // What the site publishes (report sites): only "cited" is kept. The report
+ // list is not on this form; the stored one is carried over.
+ let storedReports: Pick<Site, "reports"> | undefined;
+ try {
+ storedReports = getSite(siteId, getPaths());
+ } catch {
+ storedReports = undefined;
+ }
+ const reportKeys = reportSiteKeysFromForm(formData, storedReports);
+ if (!reportKeys.ok) return { ok: false, error: reportKeys.error, values };
// Hub parent (per-site override of the family default) + PWA opt-in.
const hubUrlRaw = String(formData.get("hubUrl") ?? "").trim();
@@ -255,6 +266,7 @@ export async function saveSiteAction(
// The Site is rebuilt from the form: a key missing here is dropped on save.
...(listed ? {} : { listed: false }),
...(isPrivate ? { audience: "private" as const } : {}),
+ ...reportKeys.keys,
...(hubUrl ? { hubUrl } : {}),
...(pwa ? { pwa: true } : {}),
...(archives ? {} : { archives: false }),
diff --git a/editor/app/sites/components/SiteForm.tsx b/editor/app/sites/components/SiteForm.tsx
@@ -360,6 +360,44 @@ export function SiteForm({ initial, channels, allSites, isNew }: Props) {
into while Settings keeps X posts private.
</span>
</label>
+ <label className="flex flex-col gap-1 text-sm">
+ <span className="font-medium">Publish</span>
+ <SeededSelect
+ state={state}
+ name="publish"
+ aria-label="Publish"
+ initial={initial.publish === "cited" ? "cited" : "full"}
+ className="w-fit rounded border border-border bg-card px-2 py-1 text-sm"
+ >
+ <option value="full">Full archive — the searchable corpus of its channels</option>
+ <option value="cited">Cited moments only — its reports and the moments they cite</option>
+ </SeededSelect>
+ <span className="text-xs text-muted-foreground">
+ A cited-only site publishes no search, browse, full transcripts or
+ archives; its channels are the pool its reports' citations resolve
+ against. The reports pipeline applies this scope when the site is built.
+ </span>
+ </label>
+ <div className="flex flex-col gap-1 text-sm">
+ <span className="font-medium">Reports</span>
+ {initial.reports && initial.reports.length > 0 ? (
+ <ol
+ aria-label="Published reports"
+ className="list-decimal pl-6 font-mono text-xs"
+ >
+ {initial.reports.map((id) => (
+ <li key={id}>{id}</li>
+ ))}
+ </ol>
+ ) : (
+ <span className="text-xs text-muted-foreground">None</span>
+ )}
+ <span className="text-xs text-muted-foreground">
+ The published reports, in order (report ids under this site's
+ reports/ directory). Not edited here: saving this form keeps the list
+ as it is.
+ </span>
+ </div>
<label className="flex items-center gap-2 text-sm">
<input
type="checkbox"
diff --git a/editor/app/sites/lib/reportSiteKeys.test.ts b/editor/app/sites/lib/reportSiteKeys.test.ts
@@ -0,0 +1,45 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { reportSiteKeysFromForm } from "./reportSiteKeys";
+
+// Run with: pnpm -C editor exec tsx --test "app/**/*.test.ts"
+
+function form(entries: Record<string, string>): FormData {
+ const fd = new FormData();
+ for (const [k, v] of Object.entries(entries)) fd.set(k, v);
+ return fd;
+}
+
+test("full (or no field) writes no publish key; cited is kept", () => {
+ assert.deepEqual(reportSiteKeysFromForm(form({ publish: "full" }), undefined), {
+ ok: true,
+ keys: {},
+ });
+ assert.deepEqual(reportSiteKeysFromForm(form({}), undefined), { ok: true, keys: {} });
+ assert.deepEqual(reportSiteKeysFromForm(form({ publish: "cited" }), undefined), {
+ ok: true,
+ keys: { publish: "cited" },
+ });
+});
+
+test("an unknown publish scope is refused, not read as full", () => {
+ for (const v of ["Cited", "corpus", ""]) {
+ const r = reportSiteKeysFromForm(form({ publish: v }), undefined);
+ assert.equal(r.ok, false, v);
+ if (!r.ok) assert.match(r.error, /Unknown publish scope/);
+ }
+});
+
+test("the stored report list is carried over in order; the form cannot set it", () => {
+ const stored = { reports: ["second", "first"] };
+ assert.deepEqual(
+ reportSiteKeysFromForm(form({ publish: "full", reports: "injected" }), stored),
+ { ok: true, keys: { reports: ["second", "first"] } },
+ );
+ // An empty or missing stored list writes no key; a bad stored id is dropped.
+ assert.deepEqual(reportSiteKeysFromForm(form({}), { reports: [] }), { ok: true, keys: {} });
+ assert.deepEqual(reportSiteKeysFromForm(form({}), { reports: ["ok", "NOT OK"] }), {
+ ok: true,
+ keys: { reports: ["ok"] },
+ });
+});
diff --git a/editor/app/sites/lib/reportSiteKeys.ts b/editor/app/sites/lib/reportSiteKeys.ts
@@ -0,0 +1,35 @@
+// The site form's two report-site keys (site.json `publish` and `reports`),
+// pure so the parse is tested without a server (saveSiteAction calls it).
+//
+// `publish` is the form's select; only "cited" is kept, as the file writes it.
+// `reports` is NOT edited by this form (the site's Reports tab manages it): the
+// Site is rebuilt from the form on save, so the list already stored is carried
+// over, and saving any other field never drops a published report.
+
+import {
+ isSitePublish,
+ parseSiteReports,
+ type Site,
+} from "yt-dlp-transcript-common/lib/site";
+
+export type ReportSiteKeys =
+ | { ok: true; keys: Pick<Site, "publish" | "reports"> }
+ | { ok: false; error: string };
+
+export function reportSiteKeysFromForm(
+ formData: FormData,
+ stored: Pick<Site, "reports"> | undefined,
+): ReportSiteKeys {
+ const raw = String(formData.get("publish") ?? "full");
+ if (!isSitePublish(raw)) {
+ return { ok: false, error: `Unknown publish scope "${raw}".` };
+ }
+ const reports = parseSiteReports(stored?.reports);
+ return {
+ ok: true,
+ keys: {
+ ...(raw === "cited" ? { publish: "cited" as const } : {}),
+ ...(reports.length > 0 ? { reports } : {}),
+ },
+ };
+}
