commit 6196a978a90fb7364ef06a88b722ddf4ede02d71
parent 7fff691b64dc662dba14d140114f6b30747518dd
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 5 Oct 2026 02:14:58 -0400
editor: the site form's Publish control and a read-only Reports list; the save keeps the stored reports; an [Unreleased] bullet
- SiteForm: a Publish select (Full archive | Cited moments only) with one
line of help, and the site's report ids listed read-only (or "None").
- saveSiteAction: reportSiteKeysFromForm (sites/lib/reportSiteKeys.ts, pure,
tested) keeps only "cited", refuses an unknown scope, and carries the
report list already on disk into the rebuilt Site, so a save never drops
a published report.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
5 files changed, 131 insertions(+), 0 deletions(-)
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,7 @@
# Changelog
## [Unreleased]
+- **A site can say what it publishes, and which reports.** `site.json` takes `publish` — `"full"`, the searchable corpus every site has been (the default, never written), or `"cited"`, only the site's reports and the moments they cite — and `reports`, the ordered ids of its published reports (each a slug; invalid and repeated ids are dropped). The site form has a Publish control and lists the site's reports read-only; saving the form keeps the stored list. A cited site still builds as a full one until the reports pipeline applies the scope. SITE.md documents both keys.
- **A long report video no longer runs out of memory while its clips are crossfaded.** `build-video.mjs` used to join every segment of a cut in one ffmpeg command, which grows with the number of segments: a cut of a few hundred clips could use more memory than the machine had and be stopped. Past 24 segments the build now crossfades them in batches of consecutive segments, each into a file under `out/<variant>/xfade-batches/`, then crossfades those files together with the same transition and lays the on-screen deck, the rail and the dips over them. Every transition, the deck's schedule and the chapters land on the same frames as before, at the cost of one more video encode on such a cut. The batch size is `render.xfadeBatch` in the manifest or `REPORT_VIDEO_XFADE_BATCH` in the environment (which wins); `0` never batches. A batch file is reused while its segments, their holds and moves and the encode settings are unchanged, so a `--chrome-only` run that moves no footage redoes only the last pass.
- **Auto-download no longer tries a video the metadata scan already found members-only or private.** The scan records why it could not read a video, but only a failed download used to take a video out of the auto-download queue, so each members-only video the scan had found was still downloaded once: four yt-dlp requests, two with browser cookies, about 30 seconds each. A members-only or private answer from the scan now keeps the video out of the queue and counts it under the channel's members-only or private exclusions, and it stays in **Needs cookies** for a manual cookie run. A scan that reads the video later lifts this. A video the scan saw only as "Video unavailable" is still tried, since YouTube gives that answer when it is throttling too.
- **X post fetches stop on Drain, and an account with no posts is not searched.** Draining a `fetch-posts` job used to do nothing until gallery-dl finished its whole run. Now the timeline fetch stops at the next page boundary (at once when gallery-dl is between pages or waiting out a rate limit), the older-posts walk stops its current window's search at once and never starts the 45–120 second pause between windows, and both keep their resume point: the job ends done, not failed, and the log says "Drained; the next run resumes …". An older-posts walk is refused when nothing is archived and the last timeline fetch finished having read no posts, since it would only repeat empty searches; `"force": true` (`--force` on `archilyzer posts fetch`) walks anyway. A walk with nothing archived that finds nothing ends after two empty three-month windows instead of four, and records why; a walk that has posts keeps the year-of-empty-windows rule. Capture-posts already stopped between posts on Drain.
diff --git a/editor/app/sites/actions.ts b/editor/app/sites/actions.ts
@@ -31,6 +31,7 @@ import {
import { migrateToSites } from "yt-dlp-transcript-common/controller/migrateToSites";
import { socialLinksForSave } from "yt-dlp-transcript-common/lib/socialLinks";
import { formValues, type FormState } from "../lib/formState";
+import { reportSiteKeysFromForm } from "./lib/reportSiteKeys";
// A failure carries what was submitted, so SiteForm keeps what was typed
// (lib/formState.ts).
@@ -117,6 +118,16 @@ export async function saveSiteAction(
return { ok: false, error: `Unknown audience "${audienceRaw}".`, values };
}
const isPrivate = audienceRaw === "private";
+ // What the site publishes (report sites): only "cited" is kept. The report
+ // list is not on this form; the stored one is carried over.
+ let storedReports: Pick<Site, "reports"> | undefined;
+ try {
+ storedReports = getSite(siteId, getPaths());
+ } catch {
+ storedReports = undefined;
+ }
+ const reportKeys = reportSiteKeysFromForm(formData, storedReports);
+ if (!reportKeys.ok) return { ok: false, error: reportKeys.error, values };
// Hub parent (per-site override of the family default) + PWA opt-in.
const hubUrlRaw = String(formData.get("hubUrl") ?? "").trim();
@@ -255,6 +266,7 @@ export async function saveSiteAction(
// The Site is rebuilt from the form: a key missing here is dropped on save.
...(listed ? {} : { listed: false }),
...(isPrivate ? { audience: "private" as const } : {}),
+ ...reportKeys.keys,
...(hubUrl ? { hubUrl } : {}),
...(pwa ? { pwa: true } : {}),
...(archives ? {} : { archives: false }),
diff --git a/editor/app/sites/components/SiteForm.tsx b/editor/app/sites/components/SiteForm.tsx
@@ -360,6 +360,44 @@ export function SiteForm({ initial, channels, allSites, isNew }: Props) {
into while Settings keeps X posts private.
</span>
</label>
+ <label className="flex flex-col gap-1 text-sm">
+ <span className="font-medium">Publish</span>
+ <SeededSelect
+ state={state}
+ name="publish"
+ aria-label="Publish"
+ initial={initial.publish === "cited" ? "cited" : "full"}
+ className="w-fit rounded border border-border bg-card px-2 py-1 text-sm"
+ >
+ <option value="full">Full archive — the searchable corpus of its channels</option>
+ <option value="cited">Cited moments only — its reports and the moments they cite</option>
+ </SeededSelect>
+ <span className="text-xs text-muted-foreground">
+ A cited-only site publishes no search, browse, full transcripts or
+ archives; its channels are the pool its reports' citations resolve
+ against. The reports pipeline applies this scope when the site is built.
+ </span>
+ </label>
+ <div className="flex flex-col gap-1 text-sm">
+ <span className="font-medium">Reports</span>
+ {initial.reports && initial.reports.length > 0 ? (
+ <ol
+ aria-label="Published reports"
+ className="list-decimal pl-6 font-mono text-xs"
+ >
+ {initial.reports.map((id) => (
+ <li key={id}>{id}</li>
+ ))}
+ </ol>
+ ) : (
+ <span className="text-xs text-muted-foreground">None</span>
+ )}
+ <span className="text-xs text-muted-foreground">
+ The published reports, in order (report ids under this site's
+ reports/ directory). Not edited here: saving this form keeps the list
+ as it is.
+ </span>
+ </div>
<label className="flex items-center gap-2 text-sm">
<input
type="checkbox"
diff --git a/editor/app/sites/lib/reportSiteKeys.test.ts b/editor/app/sites/lib/reportSiteKeys.test.ts
@@ -0,0 +1,45 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { reportSiteKeysFromForm } from "./reportSiteKeys";
+
+// Run with: pnpm -C editor exec tsx --test "app/**/*.test.ts"
+
+function form(entries: Record<string, string>): FormData {
+ const fd = new FormData();
+ for (const [k, v] of Object.entries(entries)) fd.set(k, v);
+ return fd;
+}
+
+test("full (or no field) writes no publish key; cited is kept", () => {
+ assert.deepEqual(reportSiteKeysFromForm(form({ publish: "full" }), undefined), {
+ ok: true,
+ keys: {},
+ });
+ assert.deepEqual(reportSiteKeysFromForm(form({}), undefined), { ok: true, keys: {} });
+ assert.deepEqual(reportSiteKeysFromForm(form({ publish: "cited" }), undefined), {
+ ok: true,
+ keys: { publish: "cited" },
+ });
+});
+
+test("an unknown publish scope is refused, not read as full", () => {
+ for (const v of ["Cited", "corpus", ""]) {
+ const r = reportSiteKeysFromForm(form({ publish: v }), undefined);
+ assert.equal(r.ok, false, v);
+ if (!r.ok) assert.match(r.error, /Unknown publish scope/);
+ }
+});
+
+test("the stored report list is carried over in order; the form cannot set it", () => {
+ const stored = { reports: ["second", "first"] };
+ assert.deepEqual(
+ reportSiteKeysFromForm(form({ publish: "full", reports: "injected" }), stored),
+ { ok: true, keys: { reports: ["second", "first"] } },
+ );
+ // An empty or missing stored list writes no key; a bad stored id is dropped.
+ assert.deepEqual(reportSiteKeysFromForm(form({}), { reports: [] }), { ok: true, keys: {} });
+ assert.deepEqual(reportSiteKeysFromForm(form({}), { reports: ["ok", "NOT OK"] }), {
+ ok: true,
+ keys: { reports: ["ok"] },
+ });
+});
diff --git a/editor/app/sites/lib/reportSiteKeys.ts b/editor/app/sites/lib/reportSiteKeys.ts
@@ -0,0 +1,35 @@
+// The site form's two report-site keys (site.json `publish` and `reports`),
+// pure so the parse is tested without a server (saveSiteAction calls it).
+//
+// `publish` is the form's select; only "cited" is kept, as the file writes it.
+// `reports` is NOT edited by this form (the site's Reports tab manages it): the
+// Site is rebuilt from the form on save, so the list already stored is carried
+// over, and saving any other field never drops a published report.
+
+import {
+ isSitePublish,
+ parseSiteReports,
+ type Site,
+} from "yt-dlp-transcript-common/lib/site";
+
+export type ReportSiteKeys =
+ | { ok: true; keys: Pick<Site, "publish" | "reports"> }
+ | { ok: false; error: string };
+
+export function reportSiteKeysFromForm(
+ formData: FormData,
+ stored: Pick<Site, "reports"> | undefined,
+): ReportSiteKeys {
+ const raw = String(formData.get("publish") ?? "full");
+ if (!isSitePublish(raw)) {
+ return { ok: false, error: `Unknown publish scope "${raw}".` };
+ }
+ const reports = parseSiteReports(stored?.reports);
+ return {
+ ok: true,
+ keys: {
+ ...(raw === "cited" ? { publish: "cited" as const } : {}),
+ ...(reports.length > 0 ? { reports } : {}),
+ },
+ };
+}
